You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2015/03/04 16:35:03 UTC
[01/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Repository: incubator-ignite
Updated Branches:
refs/heads/ignite-386-tmp [created] 498dcfab5
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopHashMapSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopHashMapSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopHashMapSelfTest.java
new file mode 100644
index 0000000..5b1b6a8
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopHashMapSelfTest.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
+
+import com.google.common.collect.*;
+import org.apache.hadoop.io.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+import org.apache.ignite.internal.util.typedef.*;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+/**
+ *
+ */
+public class HadoopHashMapSelfTest extends HadoopAbstractMapTest {
+
+ public void _testAllocation() throws Exception {
+ final GridUnsafeMemory mem = new GridUnsafeMemory(0);
+
+ long size = 3L * 1024 * 1024 * 1024;
+
+ final long chunk = 16;// * 1024;
+
+ final int page = 4 * 1024;
+
+ final int writes = chunk < page ? 1 : (int)(chunk / page);
+
+ final long cnt = size / chunk;
+
+ assert cnt < Integer.MAX_VALUE;
+
+ final int threads = 4;
+
+ long start = System.currentTimeMillis();
+
+ multithreaded(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ int cnt0 = (int)(cnt / threads);
+
+ for (int i = 0; i < cnt0; i++) {
+ long ptr = mem.allocate(chunk);
+
+ for (int j = 0; j < writes; j++)
+ mem.writeInt(ptr + j * page, 100500);
+ }
+
+ return null;
+ }
+ }, threads);
+
+ X.println("End: " + (System.currentTimeMillis() - start) + " mem: " + mem.allocatedSize() + " cnt: " + cnt);
+
+ Thread.sleep(30000);
+ }
+
+
+ /** */
+ public void testMapSimple() throws Exception {
+ GridUnsafeMemory mem = new GridUnsafeMemory(0);
+
+// mem.listen(new GridOffHeapEventListener() {
+// @Override public void onEvent(GridOffHeapEvent evt) {
+// if (evt == GridOffHeapEvent.ALLOCATE)
+// U.dumpStack();
+// }
+// });
+
+ Random rnd = new Random();
+
+ int mapSize = 16 << rnd.nextInt(3);
+
+ HadoopTaskContext taskCtx = new TaskContext();
+
+ final HadoopHashMultimap m = new HadoopHashMultimap(new JobInfo(), mem, mapSize);
+
+ HadoopMultimap.Adder a = m.startAdding(taskCtx);
+
+ Multimap<Integer, Integer> mm = ArrayListMultimap.create();
+
+ for (int i = 0, vals = 4 * mapSize + rnd.nextInt(25); i < vals; i++) {
+ int key = rnd.nextInt(mapSize);
+ int val = rnd.nextInt();
+
+ a.write(new IntWritable(key), new IntWritable(val));
+ mm.put(key, val);
+
+ X.println("k: " + key + " v: " + val);
+
+ a.close();
+
+ check(m, mm, taskCtx);
+
+ a = m.startAdding(taskCtx);
+ }
+
+// a.add(new IntWritable(10), new IntWritable(2));
+// mm.put(10, 2);
+// check(m, mm);
+
+ a.close();
+
+ X.println("Alloc: " + mem.allocatedSize());
+
+ m.close();
+
+ assertEquals(0, mem.allocatedSize());
+ }
+
+ private void check(HadoopHashMultimap m, Multimap<Integer, Integer> mm, HadoopTaskContext taskCtx) throws Exception {
+ final HadoopTaskInput in = m.input(taskCtx);
+
+ Map<Integer, Collection<Integer>> mmm = mm.asMap();
+
+ int keys = 0;
+
+ while (in.next()) {
+ keys++;
+
+ IntWritable k = (IntWritable)in.key();
+
+ assertNotNull(k);
+
+ ArrayList<Integer> vs = new ArrayList<>();
+
+ Iterator<?> it = in.values();
+
+ while (it.hasNext())
+ vs.add(((IntWritable) it.next()).get());
+
+ Collection<Integer> exp = mmm.get(k.get());
+
+ assertEquals(sorted(exp), sorted(vs));
+ }
+
+ X.println("keys: " + keys + " cap: " + m.capacity());
+
+ assertEquals(mmm.size(), keys);
+
+ assertEquals(m.keys(), keys);
+
+ in.close();
+ }
+
+ private GridLongList sorted(Collection<Integer> col) {
+ GridLongList lst = new GridLongList(col.size());
+
+ for (Integer i : col)
+ lst.add(i);
+
+ return lst.sort();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipListSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipListSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipListSelfTest.java
new file mode 100644
index 0000000..8a046e0
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipListSelfTest.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
+
+import com.google.common.collect.*;
+import org.apache.hadoop.io.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.io.*;
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+import org.apache.ignite.internal.util.typedef.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+import static java.lang.Math.*;
+import static org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory.*;
+
+/**
+ * Skip list tests.
+ */
+public class HadoopSkipListSelfTest extends HadoopAbstractMapTest {
+ /**
+ *
+ */
+ public void testLevel() {
+ Random rnd = new GridRandom();
+
+ int[] levelsCnts = new int[32];
+
+ int all = 10000;
+
+ for (int i = 0; i < all; i++) {
+ int level = HadoopSkipList.randomLevel(rnd);
+
+ levelsCnts[level]++;
+ }
+
+ X.println("Distribution: " + Arrays.toString(levelsCnts));
+
+ for (int level = 0; level < levelsCnts.length; level++) {
+ int exp = (level + 1) == levelsCnts.length ? 0 : all >>> (level + 1);
+
+ double precission = 0.72 / Math.max(32 >>> level, 1);
+
+ int sigma = max((int)ceil(precission * exp), 5);
+
+ X.println("Level: " + level + " exp: " + exp + " act: " + levelsCnts[level] + " precission: " + precission +
+ " sigma: " + sigma);
+
+ assertTrue(abs(exp - levelsCnts[level]) <= sigma);
+ }
+ }
+
+ public void testMapSimple() throws Exception {
+ GridUnsafeMemory mem = new GridUnsafeMemory(0);
+
+// mem.listen(new GridOffHeapEventListener() {
+// @Override public void onEvent(GridOffHeapEvent evt) {
+// if (evt == GridOffHeapEvent.ALLOCATE)
+// U.dumpStack();
+// }
+// });
+
+ Random rnd = new Random();
+
+ int mapSize = 16 << rnd.nextInt(6);
+
+ HadoopJobInfo job = new JobInfo();
+
+ HadoopTaskContext taskCtx = new TaskContext();
+
+ HadoopMultimap m = new HadoopSkipList(job, mem);
+
+ HadoopConcurrentHashMultimap.Adder a = m.startAdding(taskCtx);
+
+ Multimap<Integer, Integer> mm = ArrayListMultimap.create();
+ Multimap<Integer, Integer> vis = ArrayListMultimap.create();
+
+ for (int i = 0, vals = 4 * mapSize + rnd.nextInt(25); i < vals; i++) {
+ int key = rnd.nextInt(mapSize);
+ int val = rnd.nextInt();
+
+ a.write(new IntWritable(key), new IntWritable(val));
+ mm.put(key, val);
+
+ X.println("k: " + key + " v: " + val);
+
+ a.close();
+
+ check(m, mm, vis, taskCtx);
+
+ a = m.startAdding(taskCtx);
+ }
+
+// a.add(new IntWritable(10), new IntWritable(2));
+// mm.put(10, 2);
+// check(m, mm);
+
+ a.close();
+
+ X.println("Alloc: " + mem.allocatedSize());
+
+ m.close();
+
+ assertEquals(0, mem.allocatedSize());
+ }
+
+ private void check(HadoopMultimap m, Multimap<Integer, Integer> mm, final Multimap<Integer, Integer> vis, HadoopTaskContext taskCtx)
+ throws Exception {
+ final HadoopTaskInput in = m.input(taskCtx);
+
+ Map<Integer, Collection<Integer>> mmm = mm.asMap();
+
+ int keys = 0;
+
+ int prevKey = Integer.MIN_VALUE;
+
+ while (in.next()) {
+ keys++;
+
+ IntWritable k = (IntWritable)in.key();
+
+ assertNotNull(k);
+
+ assertTrue(k.get() > prevKey);
+
+ prevKey = k.get();
+
+ Deque<Integer> vs = new LinkedList<>();
+
+ Iterator<?> it = in.values();
+
+ while (it.hasNext())
+ vs.addFirst(((IntWritable) it.next()).get());
+
+ Collection<Integer> exp = mmm.get(k.get());
+
+ assertEquals(exp, vs);
+ }
+
+ assertEquals(mmm.size(), keys);
+
+//! assertEquals(m.keys(), keys);
+
+ // Check visitor.
+
+ final byte[] buf = new byte[4];
+
+ final GridDataInput dataInput = new GridUnsafeDataInput();
+
+ m.visit(false, new HadoopConcurrentHashMultimap.Visitor() {
+ /** */
+ IntWritable key = new IntWritable();
+
+ /** */
+ IntWritable val = new IntWritable();
+
+ @Override public void onKey(long keyPtr, int keySize) {
+ read(keyPtr, keySize, key);
+ }
+
+ @Override public void onValue(long valPtr, int valSize) {
+ read(valPtr, valSize, val);
+
+ vis.put(key.get(), val.get());
+ }
+
+ private void read(long ptr, int size, Writable w) {
+ assert size == 4 : size;
+
+ UNSAFE.copyMemory(null, ptr, buf, BYTE_ARR_OFF, size);
+
+ dataInput.bytes(buf, size);
+
+ try {
+ w.readFields(dataInput);
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+
+// X.println("vis: " + vis);
+
+ assertEquals(mm, vis);
+
+ in.close();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testMultiThreaded() throws Exception {
+ GridUnsafeMemory mem = new GridUnsafeMemory(0);
+
+ X.println("___ Started");
+
+ Random rnd = new GridRandom();
+
+ for (int i = 0; i < 20; i++) {
+ HadoopJobInfo job = new JobInfo();
+
+ final HadoopTaskContext taskCtx = new TaskContext();
+
+ final HadoopMultimap m = new HadoopSkipList(job, mem);
+
+ final ConcurrentMap<Integer, Collection<Integer>> mm = new ConcurrentHashMap<>();
+
+ X.println("___ MT");
+
+ multithreaded(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ X.println("___ TH in");
+
+ Random rnd = new GridRandom();
+
+ IntWritable key = new IntWritable();
+ IntWritable val = new IntWritable();
+
+ HadoopMultimap.Adder a = m.startAdding(taskCtx);
+
+ for (int i = 0; i < 50000; i++) {
+ int k = rnd.nextInt(32000);
+ int v = rnd.nextInt();
+
+ key.set(k);
+ val.set(v);
+
+ a.write(key, val);
+
+ Collection<Integer> list = mm.get(k);
+
+ if (list == null) {
+ list = new ConcurrentLinkedQueue<>();
+
+ Collection<Integer> old = mm.putIfAbsent(k, list);
+
+ if (old != null)
+ list = old;
+ }
+
+ list.add(v);
+ }
+
+ a.close();
+
+ X.println("___ TH out");
+
+ return null;
+ }
+ }, 3 + rnd.nextInt(27));
+
+ HadoopTaskInput in = m.input(taskCtx);
+
+ int prevKey = Integer.MIN_VALUE;
+
+ while (in.next()) {
+ IntWritable key = (IntWritable)in.key();
+
+ assertTrue(key.get() > prevKey);
+
+ prevKey = key.get();
+
+ Iterator<?> valsIter = in.values();
+
+ Collection<Integer> vals = mm.remove(key.get());
+
+ assertNotNull(vals);
+
+ while (valsIter.hasNext()) {
+ IntWritable val = (IntWritable) valsIter.next();
+
+ assertTrue(vals.remove(val.get()));
+ }
+
+ assertTrue(vals.isEmpty());
+ }
+
+ in.close();
+ m.close();
+
+ assertEquals(0, mem.allocatedSize());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopDataStreamSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopDataStreamSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopDataStreamSelfTest.java
deleted file mode 100644
index 39a537b..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopDataStreamSelfTest.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.streams;
-
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-import org.apache.ignite.testframework.junits.common.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- *
- */
-public class GridHadoopDataStreamSelfTest extends GridCommonAbstractTest {
-
- public void testStreams() throws IOException {
- GridUnsafeMemory mem = new GridUnsafeMemory(0);
-
- GridHadoopDataOutStream out = new GridHadoopDataOutStream(mem);
-
- int size = 4 * 1024;
-
- final long ptr = mem.allocate(size);
-
- out.buffer().set(ptr, size);
-
- out.writeBoolean(false);
- out.writeBoolean(true);
- out.writeBoolean(false);
- out.write(17);
- out.write(121);
- out.write(0xfafa);
- out.writeByte(17);
- out.writeByte(121);
- out.writeByte(0xfafa);
- out.writeChar('z');
- out.writeChar('o');
- out.writeChar('r');
- out.writeShort(100);
- out.writeShort(Short.MIN_VALUE);
- out.writeShort(Short.MAX_VALUE);
- out.writeShort(65535);
- out.writeShort(65536); // 0
- out.writeInt(Integer.MAX_VALUE);
- out.writeInt(Integer.MIN_VALUE);
- out.writeInt(-1);
- out.writeInt(0);
- out.writeInt(1);
- out.writeFloat(0.33f);
- out.writeFloat(0.5f);
- out.writeFloat(-0.7f);
- out.writeFloat(Float.MAX_VALUE);
- out.writeFloat(Float.MIN_VALUE);
- out.writeFloat(Float.MIN_NORMAL);
- out.writeFloat(Float.POSITIVE_INFINITY);
- out.writeFloat(Float.NEGATIVE_INFINITY);
- out.writeFloat(Float.NaN);
- out.writeDouble(-12312312.3333333336666779);
- out.writeDouble(123123.234);
- out.writeDouble(Double.MAX_VALUE);
- out.writeDouble(Double.MIN_VALUE);
- out.writeDouble(Double.MIN_NORMAL);
- out.writeDouble(Double.NEGATIVE_INFINITY);
- out.writeDouble(Double.POSITIVE_INFINITY);
- out.writeDouble(Double.NaN);
- out.writeLong(Long.MAX_VALUE);
- out.writeLong(Long.MIN_VALUE);
- out.writeLong(0);
- out.writeLong(-1L);
- out.write(new byte[]{1,2,3});
- out.write(new byte[]{0,1,2,3}, 1, 2);
- out.writeUTF("mom washes rum");
-
- GridHadoopDataInStream in = new GridHadoopDataInStream(mem);
-
- in.buffer().set(ptr, out.buffer().pointer());
-
- assertEquals(false, in.readBoolean());
- assertEquals(true, in.readBoolean());
- assertEquals(false, in.readBoolean());
- assertEquals(17, in.read());
- assertEquals(121, in.read());
- assertEquals(0xfa, in.read());
- assertEquals(17, in.readByte());
- assertEquals(121, in.readByte());
- assertEquals((byte)0xfa, in.readByte());
- assertEquals('z', in.readChar());
- assertEquals('o', in.readChar());
- assertEquals('r', in.readChar());
- assertEquals(100, in.readShort());
- assertEquals(Short.MIN_VALUE, in.readShort());
- assertEquals(Short.MAX_VALUE, in.readShort());
- assertEquals(-1, in.readShort());
- assertEquals(0, in.readShort());
- assertEquals(Integer.MAX_VALUE, in.readInt());
- assertEquals(Integer.MIN_VALUE, in.readInt());
- assertEquals(-1, in.readInt());
- assertEquals(0, in.readInt());
- assertEquals(1, in.readInt());
- assertEquals(0.33f, in.readFloat());
- assertEquals(0.5f, in.readFloat());
- assertEquals(-0.7f, in.readFloat());
- assertEquals(Float.MAX_VALUE, in.readFloat());
- assertEquals(Float.MIN_VALUE, in.readFloat());
- assertEquals(Float.MIN_NORMAL, in.readFloat());
- assertEquals(Float.POSITIVE_INFINITY, in.readFloat());
- assertEquals(Float.NEGATIVE_INFINITY, in.readFloat());
- assertEquals(Float.NaN, in.readFloat());
- assertEquals(-12312312.3333333336666779, in.readDouble());
- assertEquals(123123.234, in.readDouble());
- assertEquals(Double.MAX_VALUE, in.readDouble());
- assertEquals(Double.MIN_VALUE, in.readDouble());
- assertEquals(Double.MIN_NORMAL, in.readDouble());
- assertEquals(Double.NEGATIVE_INFINITY, in.readDouble());
- assertEquals(Double.POSITIVE_INFINITY, in.readDouble());
- assertEquals(Double.NaN, in.readDouble());
- assertEquals(Long.MAX_VALUE, in.readLong());
- assertEquals(Long.MIN_VALUE, in.readLong());
- assertEquals(0, in.readLong());
- assertEquals(-1, in.readLong());
-
- byte[] b = new byte[3];
-
- in.read(b);
-
- assertTrue(Arrays.equals(new byte[]{1,2,3}, b));
-
- b = new byte[4];
-
- in.read(b, 1, 2);
-
- assertTrue(Arrays.equals(new byte[]{0, 1, 2, 0}, b));
-
- assertEquals("mom washes rum", in.readUTF());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataStreamSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataStreamSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataStreamSelfTest.java
new file mode 100644
index 0000000..48b99ab
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataStreamSelfTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.streams;
+
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+import org.apache.ignite.testframework.junits.common.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ *
+ */
+public class HadoopDataStreamSelfTest extends GridCommonAbstractTest {
+
+ public void testStreams() throws IOException {
+ GridUnsafeMemory mem = new GridUnsafeMemory(0);
+
+ HadoopDataOutStream out = new HadoopDataOutStream(mem);
+
+ int size = 4 * 1024;
+
+ final long ptr = mem.allocate(size);
+
+ out.buffer().set(ptr, size);
+
+ out.writeBoolean(false);
+ out.writeBoolean(true);
+ out.writeBoolean(false);
+ out.write(17);
+ out.write(121);
+ out.write(0xfafa);
+ out.writeByte(17);
+ out.writeByte(121);
+ out.writeByte(0xfafa);
+ out.writeChar('z');
+ out.writeChar('o');
+ out.writeChar('r');
+ out.writeShort(100);
+ out.writeShort(Short.MIN_VALUE);
+ out.writeShort(Short.MAX_VALUE);
+ out.writeShort(65535);
+ out.writeShort(65536); // 0
+ out.writeInt(Integer.MAX_VALUE);
+ out.writeInt(Integer.MIN_VALUE);
+ out.writeInt(-1);
+ out.writeInt(0);
+ out.writeInt(1);
+ out.writeFloat(0.33f);
+ out.writeFloat(0.5f);
+ out.writeFloat(-0.7f);
+ out.writeFloat(Float.MAX_VALUE);
+ out.writeFloat(Float.MIN_VALUE);
+ out.writeFloat(Float.MIN_NORMAL);
+ out.writeFloat(Float.POSITIVE_INFINITY);
+ out.writeFloat(Float.NEGATIVE_INFINITY);
+ out.writeFloat(Float.NaN);
+ out.writeDouble(-12312312.3333333336666779);
+ out.writeDouble(123123.234);
+ out.writeDouble(Double.MAX_VALUE);
+ out.writeDouble(Double.MIN_VALUE);
+ out.writeDouble(Double.MIN_NORMAL);
+ out.writeDouble(Double.NEGATIVE_INFINITY);
+ out.writeDouble(Double.POSITIVE_INFINITY);
+ out.writeDouble(Double.NaN);
+ out.writeLong(Long.MAX_VALUE);
+ out.writeLong(Long.MIN_VALUE);
+ out.writeLong(0);
+ out.writeLong(-1L);
+ out.write(new byte[]{1,2,3});
+ out.write(new byte[]{0,1,2,3}, 1, 2);
+ out.writeUTF("mom washes rum");
+
+ HadoopDataInStream in = new HadoopDataInStream(mem);
+
+ in.buffer().set(ptr, out.buffer().pointer());
+
+ assertEquals(false, in.readBoolean());
+ assertEquals(true, in.readBoolean());
+ assertEquals(false, in.readBoolean());
+ assertEquals(17, in.read());
+ assertEquals(121, in.read());
+ assertEquals(0xfa, in.read());
+ assertEquals(17, in.readByte());
+ assertEquals(121, in.readByte());
+ assertEquals((byte)0xfa, in.readByte());
+ assertEquals('z', in.readChar());
+ assertEquals('o', in.readChar());
+ assertEquals('r', in.readChar());
+ assertEquals(100, in.readShort());
+ assertEquals(Short.MIN_VALUE, in.readShort());
+ assertEquals(Short.MAX_VALUE, in.readShort());
+ assertEquals(-1, in.readShort());
+ assertEquals(0, in.readShort());
+ assertEquals(Integer.MAX_VALUE, in.readInt());
+ assertEquals(Integer.MIN_VALUE, in.readInt());
+ assertEquals(-1, in.readInt());
+ assertEquals(0, in.readInt());
+ assertEquals(1, in.readInt());
+ assertEquals(0.33f, in.readFloat());
+ assertEquals(0.5f, in.readFloat());
+ assertEquals(-0.7f, in.readFloat());
+ assertEquals(Float.MAX_VALUE, in.readFloat());
+ assertEquals(Float.MIN_VALUE, in.readFloat());
+ assertEquals(Float.MIN_NORMAL, in.readFloat());
+ assertEquals(Float.POSITIVE_INFINITY, in.readFloat());
+ assertEquals(Float.NEGATIVE_INFINITY, in.readFloat());
+ assertEquals(Float.NaN, in.readFloat());
+ assertEquals(-12312312.3333333336666779, in.readDouble());
+ assertEquals(123123.234, in.readDouble());
+ assertEquals(Double.MAX_VALUE, in.readDouble());
+ assertEquals(Double.MIN_VALUE, in.readDouble());
+ assertEquals(Double.MIN_NORMAL, in.readDouble());
+ assertEquals(Double.NEGATIVE_INFINITY, in.readDouble());
+ assertEquals(Double.POSITIVE_INFINITY, in.readDouble());
+ assertEquals(Double.NaN, in.readDouble());
+ assertEquals(Long.MAX_VALUE, in.readLong());
+ assertEquals(Long.MIN_VALUE, in.readLong());
+ assertEquals(0, in.readLong());
+ assertEquals(-1, in.readLong());
+
+ byte[] b = new byte[3];
+
+ in.read(b);
+
+ assertTrue(Arrays.equals(new byte[]{1,2,3}, b));
+
+ b = new byte[4];
+
+ in.read(b, 1, 2);
+
+ assertTrue(Arrays.equals(new byte[]{0, 1, 2, 0}, b));
+
+ assertEquals("mom washes rum", in.readUTF());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopExecutorServiceTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopExecutorServiceTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopExecutorServiceTest.java
deleted file mode 100644
index c97b6ab..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopExecutorServiceTest.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor;
-
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.testframework.junits.common.*;
-import org.jdk8.backport.*;
-
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-
-/**
- *
- */
-public class GridHadoopExecutorServiceTest extends GridCommonAbstractTest {
- /**
- * @throws Exception If failed.
- */
- public void testExecutesAll() throws Exception {
- final GridHadoopExecutorService exec = new GridHadoopExecutorService(log, "_GRID_NAME_", 10, 5);
-
- for (int i = 0; i < 5; i++) {
- final int loops = 5000;
- int threads = 17;
-
- final LongAdder sum = new LongAdder();
-
- multithreaded(new Callable<Object>() {
- @Override public Object call() throws Exception {
- for (int i = 0; i < loops; i++) {
- exec.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- sum.increment();
-
- return null;
- }
- });
- }
-
- return null;
- }
- }, threads);
-
- while (exec.active() != 0) {
- X.println("__ active: " + exec.active());
-
- Thread.sleep(200);
- }
-
- assertEquals(threads * loops, sum.sum());
-
- X.println("_ ok");
- }
-
- assertTrue(exec.shutdown(0));
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testShutdown() throws Exception {
- for (int i = 0; i < 5; i++) {
- final GridHadoopExecutorService exec = new GridHadoopExecutorService(log, "_GRID_NAME_", 10, 5);
-
- final LongAdder sum = new LongAdder();
-
- final AtomicBoolean finish = new AtomicBoolean();
-
- IgniteInternalFuture<?> fut = multithreadedAsync(new Callable<Object>() {
- @Override public Object call() throws Exception {
- while (!finish.get()) {
- exec.submit(new Callable<Void>() {
- @Override public Void call() throws Exception {
- sum.increment();
-
- return null;
- }
- });
- }
-
- return null;
- }
- }, 19);
-
- Thread.sleep(200);
-
- assertTrue(exec.shutdown(50));
-
- long res = sum.sum();
-
- assertTrue(res > 0);
-
- finish.set(true);
-
- fut.get();
-
- assertEquals(res, sum.sum()); // Nothing was executed after shutdown.
-
- X.println("_ ok");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopExecutorServiceTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopExecutorServiceTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopExecutorServiceTest.java
new file mode 100644
index 0000000..aa50fa9
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopExecutorServiceTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor;
+
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.testframework.junits.common.*;
+import org.jdk8.backport.*;
+
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+
+/**
+ *
+ */
+public class HadoopExecutorServiceTest extends GridCommonAbstractTest {
+ /**
+ * @throws Exception If failed.
+ */
+ public void testExecutesAll() throws Exception {
+ final HadoopExecutorService exec = new HadoopExecutorService(log, "_GRID_NAME_", 10, 5);
+
+ for (int i = 0; i < 5; i++) {
+ final int loops = 5000;
+ int threads = 17;
+
+ final LongAdder sum = new LongAdder();
+
+ multithreaded(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ for (int i = 0; i < loops; i++) {
+ exec.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ sum.increment();
+
+ return null;
+ }
+ });
+ }
+
+ return null;
+ }
+ }, threads);
+
+ while (exec.active() != 0) {
+ X.println("__ active: " + exec.active());
+
+ Thread.sleep(200);
+ }
+
+ assertEquals(threads * loops, sum.sum());
+
+ X.println("_ ok");
+ }
+
+ assertTrue(exec.shutdown(0));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testShutdown() throws Exception {
+ for (int i = 0; i < 5; i++) {
+ final HadoopExecutorService exec = new HadoopExecutorService(log, "_GRID_NAME_", 10, 5);
+
+ final LongAdder sum = new LongAdder();
+
+ final AtomicBoolean finish = new AtomicBoolean();
+
+ IgniteInternalFuture<?> fut = multithreadedAsync(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ while (!finish.get()) {
+ exec.submit(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ sum.increment();
+
+ return null;
+ }
+ });
+ }
+
+ return null;
+ }
+ }, 19);
+
+ Thread.sleep(200);
+
+ assertTrue(exec.shutdown(50));
+
+ long res = sum.sum();
+
+ assertTrue(res > 0);
+
+ finish.set(true);
+
+ fut.get();
+
+ assertEquals(res, sum.sum()); // Nothing was executed after shutdown.
+
+ X.println("_ ok");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopExternalTaskExecutionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopExternalTaskExecutionSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopExternalTaskExecutionSelfTest.java
deleted file mode 100644
index 52bfa98..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopExternalTaskExecutionSelfTest.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.hadoop.mapreduce.lib.output.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.typedef.*;
-
-import java.io.*;
-import java.util.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Job tracker self test.
- */
-public class GridHadoopExternalTaskExecutionSelfTest extends GridHadoopAbstractSelfTest {
- /** {@inheritDoc} */
- @Override protected boolean igfsEnabled() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- startGrids(gridCount());
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopConfiguration hadoopConfiguration(String gridName) {
- GridHadoopConfiguration cfg = super.hadoopConfiguration(gridName);
-
- cfg.setExternalExecution(true);
-
- return cfg;
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testSimpleTaskSubmit() throws Exception {
- String testInputFile = "/test";
-
- prepareTestFile(testInputFile);
-
- Configuration cfg = new Configuration();
-
- setupFileSystems(cfg);
-
- Job job = Job.getInstance(cfg);
-
- job.setMapperClass(TestMapper.class);
- job.setCombinerClass(TestReducer.class);
- job.setReducerClass(TestReducer.class);
-
- job.setMapOutputKeyClass(Text.class);
- job.setMapOutputValueClass(IntWritable.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- job.setNumReduceTasks(1);
-
- FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/" + testInputFile));
- FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output"));
-
- job.setJarByClass(getClass());
-
- IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 1),
- createJobInfo(job.getConfiguration()));
-
- fut.get();
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testMapperException() throws Exception {
- String testInputFile = "/test";
-
- prepareTestFile(testInputFile);
-
- Configuration cfg = new Configuration();
-
- setupFileSystems(cfg);
-
- Job job = Job.getInstance(cfg);
-
- job.setMapperClass(TestFailingMapper.class);
- job.setCombinerClass(TestReducer.class);
- job.setReducerClass(TestReducer.class);
-
- job.setMapOutputKeyClass(Text.class);
- job.setMapOutputValueClass(IntWritable.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- job.setNumReduceTasks(1);
-
- FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/" + testInputFile));
- FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output"));
-
- job.setJarByClass(getClass());
-
- IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 1),
- createJobInfo(job.getConfiguration()));
-
- try {
- fut.get();
- }
- catch (IgniteCheckedException e) {
- IOException exp = X.cause(e, IOException.class);
-
- assertNotNull(exp);
- assertEquals("Test failure", exp.getMessage());
- }
- }
-
- /**
- * @param filePath File path to prepare.
- * @throws Exception If failed.
- */
- private void prepareTestFile(String filePath) throws Exception {
- IgniteFs igfs = grid(0).fileSystem(igfsName);
-
- try (IgfsOutputStream out = igfs.create(new IgfsPath(filePath), true)) {
- PrintWriter wr = new PrintWriter(new OutputStreamWriter(out));
-
- for (int i = 0; i < 1000; i++)
- wr.println("Hello, world: " + i);
-
- wr.flush();
- }
- }
-
- /**
- *
- */
- private static class TestMapper extends Mapper<Object, Text, Text, IntWritable> {
- /** One constant. */
- private IntWritable one = new IntWritable(1);
-
- /** Line constant. */
- private Text line = new Text("line");
-
- @Override protected void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
- ctx.write(line, one);
- }
- }
-
- /**
- * Failing mapper.
- */
- private static class TestFailingMapper extends Mapper<Object, Text, Text, IntWritable> {
- @Override protected void map(Object key, Text val, Context c) throws IOException, InterruptedException {
- throw new IOException("Test failure");
- }
- }
-
- /**
- *
- */
- private static class TestReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
- /** Line constant. */
- private Text line = new Text("line");
-
- @Override protected void setup(Context ctx) throws IOException, InterruptedException {
- super.setup(ctx);
- }
-
- /** {@inheritDoc} */
- @Override protected void reduce(Text key, Iterable<IntWritable> values, Context ctx)
- throws IOException, InterruptedException {
- int s = 0;
-
- for (IntWritable val : values)
- s += val.get();
-
- System.out.println(">>>> Reduced: " + s);
-
- ctx.write(line, new IntWritable(s));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
new file mode 100644
index 0000000..59ac445
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.*;
+import org.apache.hadoop.mapreduce.lib.output.*;
+import org.apache.ignite.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.typedef.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Job tracker self test.
+ */
+public class HadoopExternalTaskExecutionSelfTest extends HadoopAbstractSelfTest {
+ /** {@inheritDoc} */
+ @Override protected boolean igfsEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ startGrids(gridCount());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopConfiguration hadoopConfiguration(String gridName) {
+ HadoopConfiguration cfg = super.hadoopConfiguration(gridName);
+
+ cfg.setExternalExecution(true);
+
+ return cfg;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSimpleTaskSubmit() throws Exception {
+ String testInputFile = "/test";
+
+ prepareTestFile(testInputFile);
+
+ Configuration cfg = new Configuration();
+
+ setupFileSystems(cfg);
+
+ Job job = Job.getInstance(cfg);
+
+ job.setMapperClass(TestMapper.class);
+ job.setCombinerClass(TestReducer.class);
+ job.setReducerClass(TestReducer.class);
+
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(IntWritable.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ job.setNumReduceTasks(1);
+
+ FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/" + testInputFile));
+ FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output"));
+
+ job.setJarByClass(getClass());
+
+ IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1),
+ createJobInfo(job.getConfiguration()));
+
+ fut.get();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMapperException() throws Exception {
+ String testInputFile = "/test";
+
+ prepareTestFile(testInputFile);
+
+ Configuration cfg = new Configuration();
+
+ setupFileSystems(cfg);
+
+ Job job = Job.getInstance(cfg);
+
+ job.setMapperClass(TestFailingMapper.class);
+ job.setCombinerClass(TestReducer.class);
+ job.setReducerClass(TestReducer.class);
+
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(IntWritable.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ job.setNumReduceTasks(1);
+
+ FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/" + testInputFile));
+ FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output"));
+
+ job.setJarByClass(getClass());
+
+ IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1),
+ createJobInfo(job.getConfiguration()));
+
+ try {
+ fut.get();
+ }
+ catch (IgniteCheckedException e) {
+ IOException exp = X.cause(e, IOException.class);
+
+ assertNotNull(exp);
+ assertEquals("Test failure", exp.getMessage());
+ }
+ }
+
+ /**
+ * @param filePath File path to prepare.
+ * @throws Exception If failed.
+ */
+ private void prepareTestFile(String filePath) throws Exception {
+ IgniteFileSystem igfs = grid(0).fileSystem(igfsName);
+
+ try (IgfsOutputStream out = igfs.create(new IgfsPath(filePath), true)) {
+ PrintWriter wr = new PrintWriter(new OutputStreamWriter(out));
+
+ for (int i = 0; i < 1000; i++)
+ wr.println("Hello, world: " + i);
+
+ wr.flush();
+ }
+ }
+
+ /**
+ *
+ */
+ private static class TestMapper extends Mapper<Object, Text, Text, IntWritable> {
+ /** One constant. */
+ private IntWritable one = new IntWritable(1);
+
+ /** Line constant. */
+ private Text line = new Text("line");
+
+ @Override protected void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
+ ctx.write(line, one);
+ }
+ }
+
+ /**
+ * Failing mapper.
+ */
+ private static class TestFailingMapper extends Mapper<Object, Text, Text, IntWritable> {
+ @Override protected void map(Object key, Text val, Context c) throws IOException, InterruptedException {
+ throw new IOException("Test failure");
+ }
+ }
+
+ /**
+ *
+ */
+ private static class TestReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
+ /** Line constant. */
+ private Text line = new Text("line");
+
+ @Override protected void setup(Context ctx) throws IOException, InterruptedException {
+ super.setup(ctx);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void reduce(Text key, Iterable<IntWritable> values, Context ctx)
+ throws IOException, InterruptedException {
+ int s = 0;
+
+ for (IntWritable val : values)
+ s += val.get();
+
+ System.out.println(">>>> Reduced: " + s);
+
+ ctx.write(line, new IntWritable(s));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopExternalCommunicationSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopExternalCommunicationSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopExternalCommunicationSelfTest.java
deleted file mode 100644
index a725ddc..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopExternalCommunicationSelfTest.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.marshaller.*;
-import org.apache.ignite.marshaller.optimized.*;
-import org.apache.ignite.testframework.junits.common.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-/**
- * Tests Hadoop external communication component.
- */
-public class GridHadoopExternalCommunicationSelfTest extends GridCommonAbstractTest {
- /**
- * @throws Exception If failed.
- */
- public void testSimpleMessageSendingTcp() throws Exception {
- checkSimpleMessageSending(false);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testSimpleMessageSendingShmem() throws Exception {
- checkSimpleMessageSending(true);
- }
-
- /**
- * @throws Exception If failed.
- */
- private void checkSimpleMessageSending(boolean useShmem) throws Exception {
- UUID parentNodeId = UUID.randomUUID();
-
- Marshaller marsh = new OptimizedMarshaller();
-
- IgniteLogger log = log();
-
- GridHadoopExternalCommunication[] comms = new GridHadoopExternalCommunication[4];
-
- try {
- String name = "grid";
-
- TestHadoopListener[] lsnrs = new TestHadoopListener[4];
-
- int msgs = 10;
-
- for (int i = 0; i < comms.length; i++) {
- comms[i] = new GridHadoopExternalCommunication(parentNodeId, UUID.randomUUID(), marsh, log,
- Executors.newFixedThreadPool(1), name + i);
-
- if (useShmem)
- comms[i].setSharedMemoryPort(14000);
-
- lsnrs[i] = new TestHadoopListener(msgs);
-
- comms[i].setListener(lsnrs[i]);
-
- comms[i].start();
- }
-
- for (int r = 0; r < msgs; r++) {
- for (int from = 0; from < comms.length; from++) {
- for (int to = 0; to < comms.length; to++) {
- if (from == to)
- continue;
-
- comms[from].sendMessage(comms[to].localProcessDescriptor(), new TestMessage(from, to));
- }
- }
- }
-
- U.sleep(1000);
-
- for (TestHadoopListener lsnr : lsnrs) {
- lsnr.await(3_000);
-
- assertEquals(String.valueOf(lsnr.messages()), msgs * (comms.length - 1), lsnr.messages().size());
- }
- }
- finally {
- for (GridHadoopExternalCommunication comm : comms) {
- if (comm != null)
- comm.stop();
- }
- }
- }
-
- /**
- *
- */
- private static class TestHadoopListener implements GridHadoopMessageListener {
- /** Received messages (array list is safe because executor has one thread). */
- private Collection<TestMessage> msgs = new ArrayList<>();
-
- /** Await latch. */
- private CountDownLatch receiveLatch;
-
- /**
- * @param msgs Number of messages to await.
- */
- private TestHadoopListener(int msgs) {
- receiveLatch = new CountDownLatch(msgs);
- }
-
- /** {@inheritDoc} */
- @Override public void onMessageReceived(GridHadoopProcessDescriptor desc, GridHadoopMessage msg) {
- assert msg instanceof TestMessage;
-
- msgs.add((TestMessage)msg);
-
- receiveLatch.countDown();
- }
-
- /** {@inheritDoc} */
- @Override public void onConnectionLost(GridHadoopProcessDescriptor desc) {
- // No-op.
- }
-
- /**
- * @return Received messages.
- */
- public Collection<TestMessage> messages() {
- return msgs;
- }
-
- /**
- * @param millis Time to await.
- * @throws InterruptedException If wait interrupted.
- */
- public void await(int millis) throws InterruptedException {
- receiveLatch.await(millis, TimeUnit.MILLISECONDS);
- }
- }
-
- /**
- *
- */
- private static class TestMessage implements GridHadoopMessage {
- /** From index. */
- private int from;
-
- /** To index. */
- private int to;
-
- /**
- * @param from From index.
- * @param to To index.
- */
- private TestMessage(int from, int to) {
- this.from = from;
- this.to = to;
- }
-
- /**
- * Required by {@link Externalizable}.
- */
- public TestMessage() {
- // No-op.
- }
-
- /**
- * @return From index.
- */
- public int from() {
- return from;
- }
-
- /**
- * @return To index.
- */
- public int to() {
- return to;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeInt(from);
- out.writeInt(to);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- from = in.readInt();
- to = in.readInt();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunicationSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunicationSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunicationSelfTest.java
new file mode 100644
index 0000000..a21633d
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunicationSelfTest.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.marshaller.*;
+import org.apache.ignite.marshaller.optimized.*;
+import org.apache.ignite.testframework.junits.common.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+/**
+ * Tests Hadoop external communication component.
+ */
+public class HadoopExternalCommunicationSelfTest extends GridCommonAbstractTest {
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSimpleMessageSendingTcp() throws Exception {
+ checkSimpleMessageSending(false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSimpleMessageSendingShmem() throws Exception {
+ checkSimpleMessageSending(true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void checkSimpleMessageSending(boolean useShmem) throws Exception {
+ UUID parentNodeId = UUID.randomUUID();
+
+ Marshaller marsh = new OptimizedMarshaller();
+
+ IgniteLogger log = log();
+
+ HadoopExternalCommunication[] comms = new HadoopExternalCommunication[4];
+
+ try {
+ String name = "grid";
+
+ TestHadoopListener[] lsnrs = new TestHadoopListener[4];
+
+ int msgs = 10;
+
+ for (int i = 0; i < comms.length; i++) {
+ comms[i] = new HadoopExternalCommunication(parentNodeId, UUID.randomUUID(), marsh, log,
+ Executors.newFixedThreadPool(1), name + i);
+
+ if (useShmem)
+ comms[i].setSharedMemoryPort(14000);
+
+ lsnrs[i] = new TestHadoopListener(msgs);
+
+ comms[i].setListener(lsnrs[i]);
+
+ comms[i].start();
+ }
+
+ for (int r = 0; r < msgs; r++) {
+ for (int from = 0; from < comms.length; from++) {
+ for (int to = 0; to < comms.length; to++) {
+ if (from == to)
+ continue;
+
+ comms[from].sendMessage(comms[to].localProcessDescriptor(), new TestMessage(from, to));
+ }
+ }
+ }
+
+ U.sleep(1000);
+
+ for (TestHadoopListener lsnr : lsnrs) {
+ lsnr.await(3_000);
+
+ assertEquals(String.valueOf(lsnr.messages()), msgs * (comms.length - 1), lsnr.messages().size());
+ }
+ }
+ finally {
+ for (HadoopExternalCommunication comm : comms) {
+ if (comm != null)
+ comm.stop();
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private static class TestHadoopListener implements HadoopMessageListener {
+ /** Received messages (array list is safe because executor has one thread). */
+ private Collection<TestMessage> msgs = new ArrayList<>();
+
+ /** Await latch. */
+ private CountDownLatch receiveLatch;
+
+ /**
+ * @param msgs Number of messages to await.
+ */
+ private TestHadoopListener(int msgs) {
+ receiveLatch = new CountDownLatch(msgs);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onMessageReceived(HadoopProcessDescriptor desc, HadoopMessage msg) {
+ assert msg instanceof TestMessage;
+
+ msgs.add((TestMessage)msg);
+
+ receiveLatch.countDown();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onConnectionLost(HadoopProcessDescriptor desc) {
+ // No-op.
+ }
+
+ /**
+ * @return Received messages.
+ */
+ public Collection<TestMessage> messages() {
+ return msgs;
+ }
+
+ /**
+ * @param millis Time to await.
+ * @throws InterruptedException If wait interrupted.
+ */
+ public void await(int millis) throws InterruptedException {
+ receiveLatch.await(millis, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ /**
+ *
+ */
+ private static class TestMessage implements HadoopMessage {
+ /** From index. */
+ private int from;
+
+ /** To index. */
+ private int to;
+
+ /**
+ * @param from From index.
+ * @param to To index.
+ */
+ private TestMessage(int from, int to) {
+ this.from = from;
+ this.to = to;
+ }
+
+ /**
+ * Required by {@link Externalizable}.
+ */
+ public TestMessage() {
+ // No-op.
+ }
+
+ /**
+ * @return From index.
+ */
+ public int from() {
+ return from;
+ }
+
+ /**
+ * @return To index.
+ */
+ public int to() {
+ return to;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeInt(from);
+ out.writeInt(to);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ from = in.readInt();
+ to = in.readInt();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
index 836cdaa..4790e63 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
@@ -50,75 +50,75 @@ public class IgniteHadoopTestSuite extends TestSuite {
downloadHadoop();
downloadHive();
- GridHadoopClassLoader ldr = new GridHadoopClassLoader(null);
+ HadoopClassLoader ldr = new HadoopClassLoader(null);
TestSuite suite = new TestSuite("Ignite Hadoop MR Test Suite");
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemLoopbackExternalPrimarySelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemLoopbackExternalSecondarySelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemLoopbackExternalDualSyncSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemLoopbackExternalDualAsyncSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemSecondaryModeSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemSecondaryModeSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemClientSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemClientSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemLoggerStateSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemLoggerSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoggerStateSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoggerSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemHandshakeSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemHandshakeSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoop20FileSystemLoopbackPrimarySelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfs20FileSystemLoopbackPrimarySelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopDualSyncSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopDualAsyncSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfsDualSyncSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfsDualAsyncSelfTest.class.getName())));
suite.addTest(IgfsEventsTestSuite.suiteNoarchOnly());
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopFileSystemsTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopFileSystemsTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopValidationSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopValidationSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopDefaultMapReducePlannerSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopJobTrackerSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopDefaultMapReducePlannerSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopJobTrackerSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopHashMapSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopDataStreamSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopConcurrentHashMultimapSelftest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopHashMapSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopDataStreamSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopConcurrentHashMultimapSelftest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopSkipListSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopSkipListSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopTaskExecutionSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopTaskExecutionSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopV2JobSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopV2JobSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopSerializationWrapperSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopSplitWrapperSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopSerializationWrapperSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopSplitWrapperSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopTasksV1Test.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopTasksV2Test.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopTasksV1Test.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopTasksV2Test.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopMapReduceTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopMapReduceTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopMapReduceEmbeddedSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopMapReduceEmbeddedSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopExternalTaskExecutionSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopExternalCommunicationSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopExternalTaskExecutionSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopExternalCommunicationSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopSortingTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopSortingTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopSortingExternalTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopSortingExternalTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopGroupingTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopGroupingTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopClientProtocolSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopClientProtocolEmbeddedSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopClientProtocolSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopClientProtocolEmbeddedSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(GridHadoopCommandLineTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopCommandLineTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopSecondaryFileSystemConfigurationTest.class.getName())));
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java
index 6055db9..87233fc 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java
@@ -36,25 +36,25 @@ public class IgniteIgfsLinuxAndMacOSTestSuite extends TestSuite {
public static TestSuite suite() throws Exception {
downloadHadoop();
- GridHadoopClassLoader ldr = new GridHadoopClassLoader(null);
+ HadoopClassLoader ldr = new HadoopClassLoader(null);
TestSuite suite = new TestSuite("Ignite IGFS Test Suite For Linux And Mac OS");
suite.addTest(new TestSuite(ldr.loadClass(IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemShmemExternalPrimarySelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemShmemExternalSecondarySelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemShmemExternalDualSyncSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemShmemExternalDualAsyncSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemExternalPrimarySelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemExternalSecondarySelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemShmemEmbeddedPrimarySelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemShmemEmbeddedSecondarySelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemShmemEmbeddedDualSyncSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoopFileSystemIpcCacheSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemIpcCacheSelfTest.class.getName())));
- suite.addTest(new TestSuite(ldr.loadClass(IgfsHadoop20FileSystemShmemPrimarySelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfs20FileSystemShmemPrimarySelfTest.class.getName())));
suite.addTest(IgfsEventsTestSuite.suite());
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java
----------------------------------------------------------------------
diff --git a/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java b/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java
index 31aa9e5..0f0a93f 100644
--- a/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java
+++ b/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java
@@ -254,14 +254,14 @@ public class IgniteSpringBean implements Ignite, DisposableBean, InitializingBea
}
/** {@inheritDoc} */
- @Override public IgniteFs fileSystem(String name) {
+ @Override public IgniteFileSystem fileSystem(String name) {
assert g != null;
return g.fileSystem(name);
}
/** {@inheritDoc} */
- @Override public Collection<IgniteFs> fileSystems() {
+ @Override public Collection<IgniteFileSystem> fileSystems() {
assert g != null;
return g.fileSystems();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala
----------------------------------------------------------------------
diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala
index 964de1d..f0e5eba 100644
--- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala
@@ -1520,8 +1520,8 @@ object visor extends VisorTag {
val cfgs =
try
// Cache, IGFS, streamer and DR configurations should be excluded from daemon node config.
- spring.loadConfigurations(url, "cacheConfiguration", "igfsConfiguration", "streamerConfiguration",
- "drSenderHubConfiguration", "drReceiverHubConfiguration").get1()
+ spring.loadConfigurations(url, "cacheConfiguration", "fileSystemConfiguration",
+ "streamerConfiguration", "drSenderHubConfiguration", "drReceiverHubConfiguration").get1()
finally {
if (log4jTup != null)
U.removeLog4jNoOpLogger(log4jTup)
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b561aab..9c6ef75 100644
--- a/pom.xml
+++ b/pom.xml
@@ -731,12 +731,12 @@
<packages>org.apache.ignite.services</packages>
</group>
<group>
- <title>Ignite File System</title>
- <packages>org.apache.ignite.igfs</packages>
+ <title>File System APIs</title>
+ <packages>org.apache.ignite.igfs:org.apache.ignite.igfs.mapreduce:org.apache.ignite.igfs.mapreduce.records:org.apache.ignite.igfs.secondary</packages>
</group>
<group>
<title>Hadoop Accelerator APIs</title>
- <packages>org.apache.ignite.igfs.hadoop:org.apache.ignite.igfs.hadoop.v1:org.apache.ignite.igfs.hadoop.v2:org.apache.ignite.igfs.mapreduce:org.apache.ignite.igfs.mapreduce.records:org.apache.ignite.hadoop</packages>
+ <packages>org.apache.ignite.hadoop:org.apache.ignite.hadoop.fs:org.apache.ignite.hadoop.fs.v1:org.apache.ignite.hadoop.fs.v2:org.apache.ignite.hadoop.mapreduce</packages>
</group>
<group>
<title>Streaming APIs</title>
@@ -926,12 +926,12 @@
<packages>org.apache.ignite.services</packages>
</group>
<group>
- <title>Ignite File System</title>
- <packages>org.apache.ignite.igfs</packages>
+ <title>File System APIs</title>
+ <packages>org.apache.ignite.igfs:org.apache.ignite.igfs.mapreduce:org.apache.ignite.igfs.mapreduce.records:org.apache.ignite.igfs.secondary</packages>
</group>
<group>
<title>Hadoop Accelerator APIs</title>
- <packages>org.apache.ignite.igfs.hadoop:org.apache.ignite.igfs.hadoop.v1:org.apache.ignite.igfs.hadoop.v2:org.apache.ignite.igfs.mapreduce:org.apache.ignite.igfs.mapreduce.records:org.apache.ignite.hadoop</packages>
+ <packages>org.apache.ignite.hadoop:org.apache.ignite.hadoop.fs:org.apache.ignite.hadoop.fs.v1:org.apache.ignite.hadoop.fs.v2:org.apache.ignite.hadoop.mapreduce</packages>
</group>
<group>
<title>Streaming APIs</title>
[31/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopPerformanceCounter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopPerformanceCounter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopPerformanceCounter.java
new file mode 100644
index 0000000..351839a
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopPerformanceCounter.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.counter;
+
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Counter for the job statistics accumulation.
+ */
+public class HadoopPerformanceCounter extends HadoopCounterAdapter {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** The group name for this counter. */
+ private static final String GROUP_NAME = "SYSTEM";
+
+ /** The counter name for this counter. */
+ private static final String COUNTER_NAME = "PERFORMANCE";
+
+ /** Events collections. */
+ private Collection<T2<String,Long>> evts = new ArrayList<>();
+
+ /** Node id to insert into the event info. */
+ private UUID nodeId;
+
+ /** */
+ private int reducerNum;
+
+ /** */
+ private volatile Long firstShuffleMsg;
+
+ /** */
+ private volatile Long lastShuffleMsg;
+
+ /**
+ * Default constructor required by {@link Externalizable}.
+ */
+ public HadoopPerformanceCounter() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param grp Group name.
+ * @param name Counter name.
+ */
+ public HadoopPerformanceCounter(String grp, String name) {
+ super(grp, name);
+ }
+
+ /**
+ * Constructor to create instance to use this as helper.
+ *
+ * @param nodeId Id of the work node.
+ */
+ public HadoopPerformanceCounter(UUID nodeId) {
+ this.nodeId = nodeId;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void writeValue(ObjectOutput out) throws IOException {
+ U.writeCollection(out, evts);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void readValue(ObjectInput in) throws IOException {
+ try {
+ evts = U.readCollection(in);
+ }
+ catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void merge(HadoopCounter cntr) {
+ evts.addAll(((HadoopPerformanceCounter)cntr).evts);
+ }
+
+ /**
+ * Gets the events collection.
+ *
+ * @return Collection of event.
+ */
+ public Collection<T2<String, Long>> evts() {
+ return evts;
+ }
+
+ /**
+ * Generate name that consists of some event information.
+ *
+ * @param info Task info.
+ * @param evtType The type of the event.
+ * @return String contains necessary event information.
+ */
+ private String eventName(HadoopTaskInfo info, String evtType) {
+ return eventName(info.type().toString(), info.taskNumber(), evtType);
+ }
+
+ /**
+ * Generate name that consists of some event information.
+ *
+ * @param taskType Task type.
+ * @param taskNum Number of the task.
+ * @param evtType The type of the event.
+ * @return String contains necessary event information.
+ */
+ private String eventName(String taskType, int taskNum, String evtType) {
+ assert nodeId != null;
+
+ return taskType + " " + taskNum + " " + evtType + " " + nodeId;
+ }
+
+ /**
+ * Adds event of the task submission (task instance creation).
+ *
+ * @param info Task info.
+ * @param ts Timestamp of the event.
+ */
+ public void onTaskSubmit(HadoopTaskInfo info, long ts) {
+ evts.add(new T2<>(eventName(info, "submit"), ts));
+ }
+
+ /**
+ * Adds event of the task preparation.
+ *
+ * @param info Task info.
+ * @param ts Timestamp of the event.
+ */
+ public void onTaskPrepare(HadoopTaskInfo info, long ts) {
+ evts.add(new T2<>(eventName(info, "prepare"), ts));
+ }
+
+ /**
+ * Adds event of the task finish.
+ *
+ * @param info Task info.
+ * @param ts Timestamp of the event.
+ */
+ public void onTaskFinish(HadoopTaskInfo info, long ts) {
+ if (info.type() == HadoopTaskType.REDUCE && lastShuffleMsg != null) {
+ evts.add(new T2<>(eventName("SHUFFLE", reducerNum, "start"), firstShuffleMsg));
+ evts.add(new T2<>(eventName("SHUFFLE", reducerNum, "finish"), lastShuffleMsg));
+
+ lastShuffleMsg = null;
+ }
+
+ evts.add(new T2<>(eventName(info, "finish"), ts));
+ }
+
+ /**
+ * Adds event of the task run.
+ *
+ * @param info Task info.
+ * @param ts Timestamp of the event.
+ */
+ public void onTaskStart(HadoopTaskInfo info, long ts) {
+ evts.add(new T2<>(eventName(info, "start"), ts));
+ }
+
+ /**
+ * Adds event of the job preparation.
+ *
+ * @param ts Timestamp of the event.
+ */
+ public void onJobPrepare(long ts) {
+ assert nodeId != null;
+
+ evts.add(new T2<>("JOB prepare " + nodeId, ts));
+ }
+
+ /**
+ * Adds event of the job start.
+ *
+ * @param ts Timestamp of the event.
+ */
+ public void onJobStart(long ts) {
+ assert nodeId != null;
+
+ evts.add(new T2<>("JOB start " + nodeId, ts));
+ }
+
+ /**
+ * Adds client submission events from job info.
+ *
+ * @param info Job info.
+ */
+ public void clientSubmissionEvents(HadoopJobInfo info) {
+ assert nodeId != null;
+
+ addEventFromProperty("JOB requestId", info, REQ_NEW_JOBID_TS_PROPERTY);
+ addEventFromProperty("JOB responseId", info, RESPONSE_NEW_JOBID_TS_PROPERTY);
+ addEventFromProperty("JOB submit", info, JOB_SUBMISSION_START_TS_PROPERTY);
+ }
+
+ /**
+ * Adds event with timestamp from some property in job info.
+ *
+ * @param evt Event type and phase.
+ * @param info Job info.
+ * @param propName Property name to get timestamp.
+ */
+ private void addEventFromProperty(String evt, HadoopJobInfo info, String propName) {
+ String val = info.property(propName);
+
+ if (!F.isEmpty(val)) {
+ try {
+ evts.add(new T2<>(evt + " " + nodeId, Long.parseLong(val)));
+ }
+ catch (NumberFormatException e) {
+ throw new IllegalStateException("Invalid value '" + val + "' of property '" + propName + "'", e);
+ }
+ }
+ }
+
+ /**
+ * Registers shuffle message event.
+ *
+ * @param reducerNum Number of reducer that receives the data.
+ * @param ts Timestamp of the event.
+ */
+ public void onShuffleMessage(int reducerNum, long ts) {
+ this.reducerNum = reducerNum;
+
+ if (firstShuffleMsg == null)
+ firstShuffleMsg = ts;
+
+ lastShuffleMsg = ts;
+ }
+
+ /**
+ * Gets system predefined performance counter from the HadoopCounters object.
+ *
+ * @param cntrs HadoopCounters object.
+ * @param nodeId Node id for methods that adds events. It may be null if you don't use ones.
+ * @return Predefined performance counter.
+ */
+ public static HadoopPerformanceCounter getCounter(HadoopCounters cntrs, @Nullable UUID nodeId) {
+ HadoopPerformanceCounter cntr = cntrs.counter(GROUP_NAME, COUNTER_NAME, HadoopPerformanceCounter.class);
+
+ if (nodeId != null)
+ cntr.nodeId(nodeId);
+
+ return cntrs.counter(GROUP_NAME, COUNTER_NAME, HadoopPerformanceCounter.class);
+ }
+
+ /**
+ * Sets the nodeId field.
+ *
+ * @param nodeId Node id.
+ */
+ private void nodeId(UUID nodeId) {
+ this.nodeId = nodeId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopDistributedFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopDistributedFileSystem.java
deleted file mode 100644
index e9461e2..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopDistributedFileSystem.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.fs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.hdfs.*;
-import org.apache.hadoop.mapreduce.*;
-
-import java.io.*;
-import java.net.*;
-
-import static org.apache.ignite.configuration.IgfsConfiguration.*;
-
-/**
- * Wrapper of HDFS for support of separated working directory.
- */
-public class GridHadoopDistributedFileSystem extends DistributedFileSystem {
- /** User name for each thread. */
- private final ThreadLocal<String> userName = new ThreadLocal<String>() {
- /** {@inheritDoc} */
- @Override protected String initialValue() {
- return DFLT_USER_NAME;
- }
- };
-
- /** Working directory for each thread. */
- private final ThreadLocal<Path> workingDir = new ThreadLocal<Path>() {
- /** {@inheritDoc} */
- @Override protected Path initialValue() {
- return getHomeDirectory();
- }
- };
-
- /** {@inheritDoc} */
- @Override public void initialize(URI uri, Configuration conf) throws IOException {
- super.initialize(uri, conf);
-
- setUser(conf.get(MRJobConfig.USER_NAME, DFLT_USER_NAME));
- }
-
- /**
- * Set user name and default working directory for current thread.
- *
- * @param userName User name.
- */
- public void setUser(String userName) {
- this.userName.set(userName);
-
- setWorkingDirectory(getHomeDirectory());
- }
-
- /** {@inheritDoc} */
- @Override public Path getHomeDirectory() {
- Path path = new Path("/user/" + userName.get());
-
- return path.makeQualified(getUri(), null);
- }
-
- /** {@inheritDoc} */
- @Override public void setWorkingDirectory(Path dir) {
- Path fixedDir = fixRelativePart(dir);
-
- String res = fixedDir.toUri().getPath();
-
- if (!DFSUtil.isValidName(res))
- throw new IllegalArgumentException("Invalid DFS directory name " + res);
-
- workingDir.set(fixedDir);
- }
-
- /** {@inheritDoc} */
- @Override public Path getWorkingDirectory() {
- return workingDir.get();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopFileSystemsUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopFileSystemsUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopFileSystemsUtils.java
deleted file mode 100644
index 52e7d29..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopFileSystemsUtils.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.fs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.hdfs.protocol.*;
-import org.apache.ignite.igfs.hadoop.v1.*;
-
-/**
- * Utilities for configuring file systems to support the separate working directory per each thread.
- */
-public class GridHadoopFileSystemsUtils {
- /** Name of the property for setting working directory on create new local FS instance. */
- public static final String LOC_FS_WORK_DIR_PROP = "fs." + FsConstants.LOCAL_FS_URI.getScheme() + ".workDir";
-
- /**
- * Set user name and default working directory for current thread if it's supported by file system.
- *
- * @param fs File system.
- * @param userName User name.
- */
- public static void setUser(FileSystem fs, String userName) {
- if (fs instanceof IgfsHadoopFileSystem)
- ((IgfsHadoopFileSystem)fs).setUser(userName);
- else if (fs instanceof GridHadoopDistributedFileSystem)
- ((GridHadoopDistributedFileSystem)fs).setUser(userName);
- }
-
- /**
- * Setup wrappers of filesystems to support the separate working directory.
- *
- * @param cfg Config for setup.
- */
- public static void setupFileSystems(Configuration cfg) {
- cfg.set("fs." + FsConstants.LOCAL_FS_URI.getScheme() + ".impl", GridHadoopLocalFileSystemV1.class.getName());
- cfg.set("fs.AbstractFileSystem." + FsConstants.LOCAL_FS_URI.getScheme() + ".impl",
- GridHadoopLocalFileSystemV2.class.getName());
-
- cfg.set("fs." + HdfsConstants.HDFS_URI_SCHEME + ".impl", GridHadoopDistributedFileSystem.class.getName());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopLocalFileSystemV1.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopLocalFileSystemV1.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopLocalFileSystemV1.java
deleted file mode 100644
index 28834d4..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopLocalFileSystemV1.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.fs;
-
-import org.apache.hadoop.fs.*;
-
-import java.io.*;
-
-/**
- * Local file system replacement for Hadoop jobs.
- */
-public class GridHadoopLocalFileSystemV1 extends LocalFileSystem {
- /**
- * Creates new local file system.
- */
- public GridHadoopLocalFileSystemV1() {
- super(new GridHadoopRawLocalFileSystem());
- }
-
- /** {@inheritDoc} */
- @Override public File pathToFile(Path path) {
- return ((GridHadoopRawLocalFileSystem)getRaw()).convert(path);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopLocalFileSystemV2.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopLocalFileSystemV2.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopLocalFileSystemV2.java
deleted file mode 100644
index 62d7cea..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopLocalFileSystemV2.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.fs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.local.*;
-
-import java.io.*;
-import java.net.*;
-
-import static org.apache.hadoop.fs.FsConstants.*;
-
-/**
- * Local file system replacement for Hadoop jobs.
- */
-public class GridHadoopLocalFileSystemV2 extends ChecksumFs {
- /**
- * Creates new local file system.
- *
- * @param cfg Configuration.
- * @throws IOException If failed.
- * @throws URISyntaxException If failed.
- */
- public GridHadoopLocalFileSystemV2(Configuration cfg) throws IOException, URISyntaxException {
- super(new DelegateFS(cfg));
- }
-
- /**
- * Creates new local file system.
- *
- * @param uri URI.
- * @param cfg Configuration.
- * @throws IOException If failed.
- * @throws URISyntaxException If failed.
- */
- public GridHadoopLocalFileSystemV2(URI uri, Configuration cfg) throws IOException, URISyntaxException {
- this(cfg);
- }
-
- /**
- * Delegate file system.
- */
- private static class DelegateFS extends DelegateToFileSystem {
- /**
- * Creates new local file system.
- *
- * @param cfg Configuration.
- * @throws IOException If failed.
- * @throws URISyntaxException If failed.
- */
- public DelegateFS(Configuration cfg) throws IOException, URISyntaxException {
- super(LOCAL_FS_URI, new GridHadoopRawLocalFileSystem(), cfg, LOCAL_FS_URI.getScheme(), false);
- }
-
- /** {@inheritDoc} */
- @Override public int getUriDefaultPort() {
- return -1;
- }
-
- /** {@inheritDoc} */
- @Override public FsServerDefaults getServerDefaults() throws IOException {
- return LocalConfigKeys.getServerDefaults();
- }
-
- /** {@inheritDoc} */
- @Override public boolean isValidName(String src) {
- return true;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopRawLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopRawLocalFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopRawLocalFileSystem.java
deleted file mode 100644
index 29645f8..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/GridHadoopRawLocalFileSystem.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.fs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.*;
-import org.apache.hadoop.util.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.net.*;
-import java.nio.file.*;
-
-/**
- * Local file system implementation for Hadoop.
- */
-public class GridHadoopRawLocalFileSystem extends FileSystem {
- /** Working directory for each thread. */
- private final ThreadLocal<Path> workDir = new ThreadLocal<Path>() {
- @Override protected Path initialValue() {
- return getInitialWorkingDirectory();
- }
- };
-
- /**
- * Converts Hadoop path to local path.
- *
- * @param path Hadoop path.
- * @return Local path.
- */
- File convert(Path path) {
- checkPath(path);
-
- if (path.isAbsolute())
- return new File(path.toUri().getPath());
-
- return new File(getWorkingDirectory().toUri().getPath(), path.toUri().getPath());
- }
-
- /** {@inheritDoc} */
- @Override public Path getHomeDirectory() {
- return makeQualified(new Path(System.getProperty("user.home")));
- }
-
- /** {@inheritDoc} */
- @Override public Path getInitialWorkingDirectory() {
- File f = new File(System.getProperty("user.dir"));
-
- return new Path(f.getAbsoluteFile().toURI()).makeQualified(getUri(), null);
- }
-
- /** {@inheritDoc} */
- @Override public void initialize(URI uri, Configuration conf) throws IOException {
- super.initialize(uri, conf);
-
- setConf(conf);
-
- String initWorkDir = conf.get(GridHadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP);
-
- if (initWorkDir != null)
- setWorkingDirectory(new Path(initWorkDir));
- }
-
- /** {@inheritDoc} */
- @Override public URI getUri() {
- return FsConstants.LOCAL_FS_URI;
- }
-
- /** {@inheritDoc} */
- @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException {
- return new FSDataInputStream(new InStream(checkExists(convert(f))));
- }
-
- /** {@inheritDoc} */
- @Override public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufSize,
- short replication, long blockSize, Progressable progress) throws IOException {
- File file = convert(f);
-
- if (!overwrite && !file.createNewFile())
- throw new IOException("Failed to create new file: " + f.toUri());
-
- return out(file, false, bufSize);
- }
-
- /**
- * @param file File.
- * @param append Append flag.
- * @return Output stream.
- * @throws IOException If failed.
- */
- private FSDataOutputStream out(File file, boolean append, int bufSize) throws IOException {
- return new FSDataOutputStream(new BufferedOutputStream(new FileOutputStream(file, append),
- bufSize < 32 * 1024 ? 32 * 1024 : bufSize), new Statistics(getUri().getScheme()));
- }
-
- /** {@inheritDoc} */
- @Override public FSDataOutputStream append(Path f, int bufSize, Progressable progress) throws IOException {
- return out(convert(f), true, bufSize);
- }
-
- /** {@inheritDoc} */
- @Override public boolean rename(Path src, Path dst) throws IOException {
- return convert(src).renameTo(convert(dst));
- }
-
- /** {@inheritDoc} */
- @Override public boolean delete(Path f, boolean recursive) throws IOException {
- File file = convert(f);
-
- if (file.isDirectory() && !recursive)
- throw new IOException("Failed to remove directory in non recursive mode: " + f.toUri());
-
- return U.delete(file);
- }
-
- /** {@inheritDoc} */
- @Override public void setWorkingDirectory(Path dir) {
- workDir.set(fixRelativePart(dir));
-
- checkPath(dir);
- }
-
- /** {@inheritDoc} */
- @Override public Path getWorkingDirectory() {
- return workDir.get();
- }
-
- /** {@inheritDoc} */
- @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException {
- if(f == null)
- throw new IllegalArgumentException("mkdirs path arg is null");
-
- Path parent = f.getParent();
-
- File p2f = convert(f);
-
- if(parent != null) {
- File parent2f = convert(parent);
-
- if(parent2f != null && parent2f.exists() && !parent2f.isDirectory())
- throw new FileAlreadyExistsException("Parent path is not a directory: " + parent);
-
- }
-
- return (parent == null || mkdirs(parent)) && (p2f.mkdir() || p2f.isDirectory());
- }
-
- /** {@inheritDoc} */
- @Override public FileStatus getFileStatus(Path f) throws IOException {
- return fileStatus(checkExists(convert(f)));
- }
-
- /**
- * @return File status.
- */
- private FileStatus fileStatus(File file) throws IOException {
- boolean dir = file.isDirectory();
-
- java.nio.file.Path path = dir ? null : file.toPath();
-
- return new FileStatus(dir ? 0 : file.length(), dir, 1, 4 * 1024, file.lastModified(), file.lastModified(),
- /*permission*/null, /*owner*/null, /*group*/null, dir ? null : Files.isSymbolicLink(path) ?
- new Path(Files.readSymbolicLink(path).toUri()) : null, new Path(file.toURI()));
- }
-
- /**
- * @param file File.
- * @return Same file.
- * @throws FileNotFoundException If does not exist.
- */
- private static File checkExists(File file) throws FileNotFoundException {
- if (!file.exists())
- throw new FileNotFoundException("File " + file.getAbsolutePath() + " does not exist.");
-
- return file;
- }
-
- /** {@inheritDoc} */
- @Override public FileStatus[] listStatus(Path f) throws IOException {
- File file = convert(f);
-
- if (checkExists(file).isFile())
- return new FileStatus[] {fileStatus(file)};
-
- File[] files = file.listFiles();
-
- FileStatus[] res = new FileStatus[files.length];
-
- for (int i = 0; i < res.length; i++)
- res[i] = fileStatus(files[i]);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public boolean supportsSymlinks() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public void createSymlink(Path target, Path link, boolean createParent) throws IOException {
- Files.createSymbolicLink(convert(link).toPath(), convert(target).toPath());
- }
-
- /** {@inheritDoc} */
- @Override public FileStatus getFileLinkStatus(Path f) throws IOException {
- return getFileStatus(getLinkTarget(f));
- }
-
- /** {@inheritDoc} */
- @Override public Path getLinkTarget(Path f) throws IOException {
- File file = Files.readSymbolicLink(convert(f).toPath()).toFile();
-
- return new Path(file.toURI());
- }
-
- /**
- * Input stream.
- */
- private static class InStream extends InputStream implements Seekable, PositionedReadable {
- /** */
- private final RandomAccessFile file;
-
- /**
- * @param f File.
- * @throws IOException If failed.
- */
- public InStream(File f) throws IOException {
- file = new RandomAccessFile(f, "r");
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int read() throws IOException {
- return file.read();
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int read(byte[] b, int off, int len) throws IOException {
- return file.read(b, off, len);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void close() throws IOException {
- file.close();
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int read(long pos, byte[] buf, int off, int len) throws IOException {
- long pos0 = file.getFilePointer();
-
- file.seek(pos);
- int res = file.read(buf, off, len);
-
- file.seek(pos0);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public void readFully(long pos, byte[] buf, int off, int len) throws IOException {
- if (read(pos, buf, off, len) != len)
- throw new IOException();
- }
-
- /** {@inheritDoc} */
- @Override public void readFully(long pos, byte[] buf) throws IOException {
- readFully(pos, buf, 0, buf.length);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void seek(long pos) throws IOException {
- file.seek(pos);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized long getPos() throws IOException {
- return file.getFilePointer();
- }
-
- /** {@inheritDoc} */
- @Override public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopDistributedFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopDistributedFileSystem.java
new file mode 100644
index 0000000..509f443
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopDistributedFileSystem.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.fs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.hdfs.*;
+import org.apache.hadoop.mapreduce.*;
+
+import java.io.*;
+import java.net.*;
+
+import static org.apache.ignite.configuration.FileSystemConfiguration.*;
+
+/**
+ * Wrapper of HDFS for support of separated working directory.
+ */
+public class HadoopDistributedFileSystem extends DistributedFileSystem {
+ /** User name for each thread. */
+ private final ThreadLocal<String> userName = new ThreadLocal<String>() {
+ /** {@inheritDoc} */
+ @Override protected String initialValue() {
+ return DFLT_USER_NAME;
+ }
+ };
+
+ /** Working directory for each thread. */
+ private final ThreadLocal<Path> workingDir = new ThreadLocal<Path>() {
+ /** {@inheritDoc} */
+ @Override protected Path initialValue() {
+ return getHomeDirectory();
+ }
+ };
+
+ /** {@inheritDoc} */
+ @Override public void initialize(URI uri, Configuration conf) throws IOException {
+ super.initialize(uri, conf);
+
+ setUser(conf.get(MRJobConfig.USER_NAME, DFLT_USER_NAME));
+ }
+
+ /**
+ * Set user name and default working directory for current thread.
+ *
+ * @param userName User name.
+ */
+ public void setUser(String userName) {
+ this.userName.set(userName);
+
+ setWorkingDirectory(getHomeDirectory());
+ }
+
+ /** {@inheritDoc} */
+ @Override public Path getHomeDirectory() {
+ Path path = new Path("/user/" + userName.get());
+
+ return path.makeQualified(getUri(), null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setWorkingDirectory(Path dir) {
+ Path fixedDir = fixRelativePart(dir);
+
+ String res = fixedDir.toUri().getPath();
+
+ if (!DFSUtil.isValidName(res))
+ throw new IllegalArgumentException("Invalid DFS directory name " + res);
+
+ workingDir.set(fixedDir);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Path getWorkingDirectory() {
+ return workingDir.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopFileSystemsUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopFileSystemsUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopFileSystemsUtils.java
new file mode 100644
index 0000000..f3f51d4
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopFileSystemsUtils.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.fs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.hdfs.protocol.*;
+import org.apache.ignite.hadoop.fs.v1.*;
+
+/**
+ * Utilities for configuring file systems to support the separate working directory per each thread.
+ */
+public class HadoopFileSystemsUtils {
+ /** Name of the property for setting working directory on create new local FS instance. */
+ public static final String LOC_FS_WORK_DIR_PROP = "fs." + FsConstants.LOCAL_FS_URI.getScheme() + ".workDir";
+
+ /**
+ * Set user name and default working directory for current thread if it's supported by file system.
+ *
+ * @param fs File system.
+ * @param userName User name.
+ */
+ public static void setUser(FileSystem fs, String userName) {
+ if (fs instanceof IgniteHadoopFileSystem)
+ ((IgniteHadoopFileSystem)fs).setUser(userName);
+ else if (fs instanceof HadoopDistributedFileSystem)
+ ((HadoopDistributedFileSystem)fs).setUser(userName);
+ }
+
+ /**
+ * Setup wrappers of filesystems to support the separate working directory.
+ *
+ * @param cfg Config for setup.
+ */
+ public static void setupFileSystems(Configuration cfg) {
+ cfg.set("fs." + FsConstants.LOCAL_FS_URI.getScheme() + ".impl", HadoopLocalFileSystemV1.class.getName());
+ cfg.set("fs.AbstractFileSystem." + FsConstants.LOCAL_FS_URI.getScheme() + ".impl",
+ HadoopLocalFileSystemV2.class.getName());
+
+ cfg.set("fs." + HdfsConstants.HDFS_URI_SCHEME + ".impl", HadoopDistributedFileSystem.class.getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLocalFileSystemV1.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLocalFileSystemV1.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLocalFileSystemV1.java
new file mode 100644
index 0000000..9cc5881
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLocalFileSystemV1.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.fs;
+
+import org.apache.hadoop.fs.*;
+
+import java.io.*;
+
+/**
+ * Local file system replacement for Hadoop jobs.
+ */
+public class HadoopLocalFileSystemV1 extends LocalFileSystem {
+ /**
+ * Creates new local file system.
+ */
+ public HadoopLocalFileSystemV1() {
+ super(new HadoopRawLocalFileSystem());
+ }
+
+ /** {@inheritDoc} */
+ @Override public File pathToFile(Path path) {
+ return ((HadoopRawLocalFileSystem)getRaw()).convert(path);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLocalFileSystemV2.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLocalFileSystemV2.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLocalFileSystemV2.java
new file mode 100644
index 0000000..15ddc5a
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLocalFileSystemV2.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.fs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.local.*;
+
+import java.io.*;
+import java.net.*;
+
+import static org.apache.hadoop.fs.FsConstants.*;
+
+/**
+ * Local file system replacement for Hadoop jobs.
+ */
+public class HadoopLocalFileSystemV2 extends ChecksumFs {
+ /**
+ * Creates new local file system.
+ *
+ * @param cfg Configuration.
+ * @throws IOException If failed.
+ * @throws URISyntaxException If failed.
+ */
+ public HadoopLocalFileSystemV2(Configuration cfg) throws IOException, URISyntaxException {
+ super(new DelegateFS(cfg));
+ }
+
+ /**
+ * Creates new local file system.
+ *
+ * @param uri URI.
+ * @param cfg Configuration.
+ * @throws IOException If failed.
+ * @throws URISyntaxException If failed.
+ */
+ public HadoopLocalFileSystemV2(URI uri, Configuration cfg) throws IOException, URISyntaxException {
+ this(cfg);
+ }
+
+ /**
+ * Delegate file system.
+ */
+ private static class DelegateFS extends DelegateToFileSystem {
+ /**
+ * Creates new local file system.
+ *
+ * @param cfg Configuration.
+ * @throws IOException If failed.
+ * @throws URISyntaxException If failed.
+ */
+ public DelegateFS(Configuration cfg) throws IOException, URISyntaxException {
+ super(LOCAL_FS_URI, new HadoopRawLocalFileSystem(), cfg, LOCAL_FS_URI.getScheme(), false);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getUriDefaultPort() {
+ return -1;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FsServerDefaults getServerDefaults() throws IOException {
+ return LocalConfigKeys.getServerDefaults();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isValidName(String src) {
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopParameters.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopParameters.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopParameters.java
new file mode 100644
index 0000000..7edcec0
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopParameters.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.fs;
+
+/**
+ * This class lists parameters that can be specified in Hadoop configuration.
+ * Hadoop configuration can be specified in {@code core-site.xml} file
+ * or passed to map-reduce task directly when using Hadoop driver for IGFS file system:
+ * <ul>
+ * <li>
+ * {@code fs.igfs.[name].open.sequential_reads_before_prefetch} - this parameter overrides
+ * the one specified in {@link org.apache.ignite.configuration.FileSystemConfiguration#getSequentialReadsBeforePrefetch()}
+ * IGFS data node configuration property.
+ * </li>
+ * <li>
+ * {@code fs.igfs.[name].log.enabled} - specifies whether IGFS sampling logger is enabled. If
+ * {@code true}, then all file system operations will be logged to a file.
+ * </li>
+ * <li>{@code fs.igfs.[name].log.dir} - specifies log directory where sampling log files should be placed.</li>
+ * <li>
+ * {@code fs.igfs.[name].log.batch_size} - specifies how many log entries are accumulated in a batch before
+ * it gets flushed to log file. Higher values will imply greater performance, but will increase delay
+ * before record appears in the log file.
+ * </li>
+ * <li>
+ * {@code fs.igfs.[name].colocated.writes} - specifies whether written files should be colocated on data
+ * node to which client is connected. If {@code true}, file will not be distributed and will be written
+ * to a single data node. Default value is {@code true}.
+ * </li>
+ * <li>
+ * {@code fs.igfs.prefer.local.writes} - specifies whether file preferably should be written to
+ * local data node if it has enough free space. After some time it can be redistributed across nodes though.
+ * </li>
+ * </ul>
+ * Where {@code [name]} is file system endpoint which you specify in file system URI authority part. E.g. in
+ * case your file system URI is {@code igfs://127.0.0.1:10500} then {@code name} will be {@code 127.0.0.1:10500}.
+ * <p>
+ * Sample configuration that can be placed to {@code core-site.xml} file:
+ * <pre name="code" class="xml">
+ * <property>
+ * <name>fs.igfs.127.0.0.1:10500.log.enabled</name>
+ * <value>true</value>
+ * </property>
+ * <property>
+ * <name>fs.igfs.127.0.0.1:10500.log.dir</name>
+ * <value>/home/apache/ignite/log/sampling</value>
+ * </property>
+ * <property>
+ * <name>fs.igfs.127.0.0.1:10500.log.batch_size</name>
+ * <value>16</value>
+ * </property>
+ * </pre>
+ * Parameters could also be specified per mapreduce job, e.g.
+ * <pre name="code" class="bash">
+ * hadoop jar myjarfile.jar MyMapReduceJob -Dfs.igfs.open.sequential_reads_before_prefetch=4
+ * </pre>
+ * If you want to use these parameters in code, then you have to substitute you file system name in it. The easiest
+ * way to do that is {@code String.format(PARAM_IGFS_COLOCATED_WRITES, [name])}.
+ */
+public class HadoopParameters {
+ /** Parameter name for control over file colocation write mode. */
+ public static final String PARAM_IGFS_COLOCATED_WRITES = "fs.igfs.%s.colocated.writes";
+
+ /** Parameter name for custom sequential reads before prefetch value. */
+ public static final String PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH =
+ "fs.igfs.%s.open.sequential_reads_before_prefetch";
+
+ /** Parameter name for client logger directory. */
+ public static final String PARAM_IGFS_LOG_DIR = "fs.igfs.%s.log.dir";
+
+ /** Parameter name for log batch size. */
+ public static final String PARAM_IGFS_LOG_BATCH_SIZE = "fs.igfs.%s.log.batch_size";
+
+ /** Parameter name for log enabled flag. */
+ public static final String PARAM_IGFS_LOG_ENABLED = "fs.igfs.%s.log.enabled";
+
+ /** Parameter name for prefer local writes flag. */
+ public static final String PARAM_IGFS_PREFER_LOCAL_WRITES = "fs.igfs.prefer.local.writes";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopRawLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopRawLocalFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopRawLocalFileSystem.java
new file mode 100644
index 0000000..e5ec3f7
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopRawLocalFileSystem.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.fs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.*;
+import org.apache.hadoop.util.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.net.*;
+import java.nio.file.*;
+
+/**
+ * Local file system implementation for Hadoop.
+ */
+public class HadoopRawLocalFileSystem extends FileSystem {
+ /** Working directory for each thread. */
+ private final ThreadLocal<Path> workDir = new ThreadLocal<Path>() {
+ @Override protected Path initialValue() {
+ return getInitialWorkingDirectory();
+ }
+ };
+
+ /**
+ * Converts Hadoop path to local path.
+ *
+ * @param path Hadoop path.
+ * @return Local path.
+ */
+ File convert(Path path) {
+ checkPath(path);
+
+ if (path.isAbsolute())
+ return new File(path.toUri().getPath());
+
+ return new File(getWorkingDirectory().toUri().getPath(), path.toUri().getPath());
+ }
+
+ /** {@inheritDoc} */
+ @Override public Path getHomeDirectory() {
+ return makeQualified(new Path(System.getProperty("user.home")));
+ }
+
+ /** {@inheritDoc} */
+ @Override public Path getInitialWorkingDirectory() {
+ File f = new File(System.getProperty("user.dir"));
+
+ return new Path(f.getAbsoluteFile().toURI()).makeQualified(getUri(), null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initialize(URI uri, Configuration conf) throws IOException {
+ super.initialize(uri, conf);
+
+ setConf(conf);
+
+ String initWorkDir = conf.get(HadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP);
+
+ if (initWorkDir != null)
+ setWorkingDirectory(new Path(initWorkDir));
+ }
+
+ /** {@inheritDoc} */
+ @Override public URI getUri() {
+ return FsConstants.LOCAL_FS_URI;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException {
+ return new FSDataInputStream(new InStream(checkExists(convert(f))));
+ }
+
+ /** {@inheritDoc} */
+ @Override public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufSize,
+ short replication, long blockSize, Progressable progress) throws IOException {
+ File file = convert(f);
+
+ if (!overwrite && !file.createNewFile())
+ throw new IOException("Failed to create new file: " + f.toUri());
+
+ return out(file, false, bufSize);
+ }
+
+ /**
+ * @param file File.
+ * @param append Append flag.
+ * @return Output stream.
+ * @throws IOException If failed.
+ */
+ private FSDataOutputStream out(File file, boolean append, int bufSize) throws IOException {
+ return new FSDataOutputStream(new BufferedOutputStream(new FileOutputStream(file, append),
+ bufSize < 32 * 1024 ? 32 * 1024 : bufSize), new Statistics(getUri().getScheme()));
+ }
+
+ /** {@inheritDoc} */
+ @Override public FSDataOutputStream append(Path f, int bufSize, Progressable progress) throws IOException {
+ return out(convert(f), true, bufSize);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean rename(Path src, Path dst) throws IOException {
+ return convert(src).renameTo(convert(dst));
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean delete(Path f, boolean recursive) throws IOException {
+ File file = convert(f);
+
+ if (file.isDirectory() && !recursive)
+ throw new IOException("Failed to remove directory in non recursive mode: " + f.toUri());
+
+ return U.delete(file);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setWorkingDirectory(Path dir) {
+ workDir.set(fixRelativePart(dir));
+
+ checkPath(dir);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Path getWorkingDirectory() {
+ return workDir.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ if(f == null)
+ throw new IllegalArgumentException("mkdirs path arg is null");
+
+ Path parent = f.getParent();
+
+ File p2f = convert(f);
+
+ if(parent != null) {
+ File parent2f = convert(parent);
+
+ if(parent2f != null && parent2f.exists() && !parent2f.isDirectory())
+ throw new FileAlreadyExistsException("Parent path is not a directory: " + parent);
+
+ }
+
+ return (parent == null || mkdirs(parent)) && (p2f.mkdir() || p2f.isDirectory());
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileStatus getFileStatus(Path f) throws IOException {
+ return fileStatus(checkExists(convert(f)));
+ }
+
+ /**
+ * @return File status.
+ */
+ private FileStatus fileStatus(File file) throws IOException {
+ boolean dir = file.isDirectory();
+
+ java.nio.file.Path path = dir ? null : file.toPath();
+
+ return new FileStatus(dir ? 0 : file.length(), dir, 1, 4 * 1024, file.lastModified(), file.lastModified(),
+ /*permission*/null, /*owner*/null, /*group*/null, dir ? null : Files.isSymbolicLink(path) ?
+ new Path(Files.readSymbolicLink(path).toUri()) : null, new Path(file.toURI()));
+ }
+
+ /**
+ * @param file File.
+ * @return Same file.
+ * @throws FileNotFoundException If does not exist.
+ */
+ private static File checkExists(File file) throws FileNotFoundException {
+ if (!file.exists())
+ throw new FileNotFoundException("File " + file.getAbsolutePath() + " does not exist.");
+
+ return file;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileStatus[] listStatus(Path f) throws IOException {
+ File file = convert(f);
+
+ if (checkExists(file).isFile())
+ return new FileStatus[] {fileStatus(file)};
+
+ File[] files = file.listFiles();
+
+ FileStatus[] res = new FileStatus[files.length];
+
+ for (int i = 0; i < res.length; i++)
+ res[i] = fileStatus(files[i]);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean supportsSymlinks() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void createSymlink(Path target, Path link, boolean createParent) throws IOException {
+ Files.createSymbolicLink(convert(link).toPath(), convert(target).toPath());
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileStatus getFileLinkStatus(Path f) throws IOException {
+ return getFileStatus(getLinkTarget(f));
+ }
+
+ /** {@inheritDoc} */
+ @Override public Path getLinkTarget(Path f) throws IOException {
+ File file = Files.readSymbolicLink(convert(f).toPath()).toFile();
+
+ return new Path(file.toURI());
+ }
+
+ /**
+ * Input stream.
+ */
+ private static class InStream extends InputStream implements Seekable, PositionedReadable {
+ /** */
+ private final RandomAccessFile file;
+
+ /**
+ * @param f File.
+ * @throws IOException If failed.
+ */
+ public InStream(File f) throws IOException {
+ file = new RandomAccessFile(f, "r");
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int read() throws IOException {
+ return file.read();
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int read(byte[] b, int off, int len) throws IOException {
+ return file.read(b, off, len);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void close() throws IOException {
+ file.close();
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int read(long pos, byte[] buf, int off, int len) throws IOException {
+ long pos0 = file.getFilePointer();
+
+ file.seek(pos);
+ int res = file.read(buf, off, len);
+
+ file.seek(pos0);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFully(long pos, byte[] buf, int off, int len) throws IOException {
+ if (read(pos, buf, off, len) != len)
+ throw new IOException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFully(long pos, byte[] buf) throws IOException {
+ readFully(pos, buf, 0, buf.length);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void seek(long pos) throws IOException {
+ file.seek(pos);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized long getPos() throws IOException {
+ return file.getFilePointer();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean seekToNewSource(long targetPos) throws IOException {
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfs.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfs.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfs.java
new file mode 100644
index 0000000..b3cb235
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfs.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Facade for communication with grid.
+ */
+public interface HadoopIgfs {
+ /**
+ * Perform handshake.
+ *
+ * @param logDir Log directory.
+ * @return Future with handshake result.
+ * @throws IgniteCheckedException If failed.
+ */
+ public IgfsHandshakeResponse handshake(String logDir) throws IgniteCheckedException, IOException;
+
+ /**
+ * Close connection.
+ *
+ * @param force Force flag.
+ */
+ public void close(boolean force);
+
+ /**
+ * Command to retrieve file info for some IGFS path.
+ *
+ * @param path Path to get file info for.
+ * @return Future for info operation.
+ * @throws IgniteCheckedException If failed.
+ */
+ public IgfsFile info(IgfsPath path) throws IgniteCheckedException, IOException;
+
+ /**
+ * Command to update file properties.
+ *
+ * @param path IGFS path to update properties.
+ * @param props Properties to update.
+ * @return Future for update operation.
+ * @throws IgniteCheckedException If failed.
+ */
+ public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteCheckedException, IOException;
+
+ /**
+ * Sets last access time and last modification time for a file.
+ *
+ * @param path Path to update times.
+ * @param accessTime Last access time to set.
+ * @param modificationTime Last modification time to set.
+ * @throws IgniteCheckedException If failed.
+ */
+ public Boolean setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteCheckedException,
+ IOException;
+
+ /**
+ * Command to rename given path.
+ *
+ * @param src Source path.
+ * @param dest Destination path.
+ * @return Future for rename operation.
+ * @throws IgniteCheckedException If failed.
+ */
+ public Boolean rename(IgfsPath src, IgfsPath dest) throws IgniteCheckedException, IOException;
+
+ /**
+ * Command to delete given path.
+ *
+ * @param path Path to delete.
+ * @param recursive {@code True} if deletion is recursive.
+ * @return Future for delete operation.
+ * @throws IgniteCheckedException If failed.
+ */
+ public Boolean delete(IgfsPath path, boolean recursive) throws IgniteCheckedException, IOException;
+
+ /**
+ * Command to get affinity for given path, offset and length.
+ *
+ * @param path Path to get affinity for.
+ * @param start Start position (offset).
+ * @param len Data length.
+ * @return Future for affinity command.
+ * @throws IgniteCheckedException If failed.
+ */
+ public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) throws IgniteCheckedException,
+ IOException;
+
+ /**
+ * Gets path summary.
+ *
+ * @param path Path to get summary for.
+ * @return Future that will be completed when summary is received.
+ * @throws IgniteCheckedException If failed.
+ */
+ public IgfsPathSummary contentSummary(IgfsPath path) throws IgniteCheckedException, IOException;
+
+ /**
+ * Command to create directories.
+ *
+ * @param path Path to create.
+ * @return Future for mkdirs operation.
+ * @throws IgniteCheckedException If failed.
+ */
+ public Boolean mkdirs(IgfsPath path, Map<String, String> props) throws IgniteCheckedException, IOException;
+
+ /**
+ * Command to get list of files in directory.
+ *
+ * @param path Path to list.
+ * @return Future for listFiles operation.
+ * @throws IgniteCheckedException If failed.
+ */
+ public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteCheckedException, IOException;
+
+ /**
+ * Command to get directory listing.
+ *
+ * @param path Path to list.
+ * @return Future for listPaths operation.
+ * @throws IgniteCheckedException If failed.
+ */
+ public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteCheckedException, IOException;
+
+ /**
+ * Performs status request.
+ *
+ * @return Status response.
+ * @throws IgniteCheckedException If failed.
+ */
+ public IgfsStatus fsStatus() throws IgniteCheckedException, IOException;
+
+ /**
+ * Command to open file for reading.
+ *
+ * @param path File path to open.
+ * @return Future for open operation.
+ * @throws IgniteCheckedException If failed.
+ */
+ public HadoopIgfsStreamDelegate open(IgfsPath path) throws IgniteCheckedException, IOException;
+
+ /**
+ * Command to open file for reading.
+ *
+ * @param path File path to open.
+ * @return Future for open operation.
+ * @throws IgniteCheckedException If failed.
+ */
+ public HadoopIgfsStreamDelegate open(IgfsPath path, int seqReadsBeforePrefetch) throws IgniteCheckedException,
+ IOException;
+
+ /**
+ * Command to create file and open it for output.
+ *
+ * @param path Path to file.
+ * @param overwrite If {@code true} then old file contents will be lost.
+ * @param colocate If {@code true} and called on data node, file will be written on that node.
+ * @param replication Replication factor.
+ * @param props File properties for creation.
+ * @return Stream descriptor.
+ * @throws IgniteCheckedException If failed.
+ */
+ public HadoopIgfsStreamDelegate create(IgfsPath path, boolean overwrite, boolean colocate,
+ int replication, long blockSize, @Nullable Map<String, String> props) throws IgniteCheckedException, IOException;
+
+ /**
+ * Open file for output appending data to the end of a file.
+ *
+ * @param path Path to file.
+ * @param create If {@code true}, file will be created if does not exist.
+ * @param props File properties.
+ * @return Stream descriptor.
+ * @throws IgniteCheckedException If failed.
+ */
+ public HadoopIgfsStreamDelegate append(IgfsPath path, boolean create,
+ @Nullable Map<String, String> props) throws IgniteCheckedException, IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsCommunicationException.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsCommunicationException.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsCommunicationException.java
new file mode 100644
index 0000000..ff69478
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsCommunicationException.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.ignite.*;
+
+/**
+ * Communication exception indicating a problem between file system and IGFS instance.
+ */
+public class HadoopIgfsCommunicationException extends IgniteCheckedException {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * Creates new exception with given throwable as a nested cause and
+ * source of error message.
+ *
+ * @param cause Non-null throwable cause.
+ */
+ public HadoopIgfsCommunicationException(Exception cause) {
+ super(cause);
+ }
+
+ /**
+ * Creates a new exception with given error message and optional nested cause exception.
+ *
+ * @param msg Error message.
+ */
+ public HadoopIgfsCommunicationException(String msg) {
+ super(msg);
+ }
+
+ /**
+ * Creates a new exception with given error message and optional nested cause exception.
+ *
+ * @param msg Error message.
+ * @param cause Cause.
+ */
+ public HadoopIgfsCommunicationException(String msg, Exception cause) {
+ super(msg, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java
new file mode 100644
index 0000000..7502f57
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.net.*;
+
+import static org.apache.ignite.configuration.FileSystemConfiguration.*;
+
+/**
+ * IGFS endpoint abstraction.
+ */
+public class HadoopIgfsEndpoint {
+ /** Localhost. */
+ public static final String LOCALHOST = "127.0.0.1";
+
+ /** IGFS name. */
+ private final String igfsName;
+
+ /** Grid name. */
+ private final String gridName;
+
+ /** Host. */
+ private final String host;
+
+ /** Port. */
+ private final int port;
+
+ /**
+ * Normalize IGFS URI.
+ *
+ * @param uri URI.
+ * @return Normalized URI.
+ * @throws IOException If failed.
+ */
+ public static URI normalize(URI uri) throws IOException {
+ try {
+ if (!F.eq(IgniteFileSystem.IGFS_SCHEME, uri.getScheme()))
+ throw new IOException("Failed to normalize UIR because it has non IGFS scheme: " + uri);
+
+ HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(uri.getAuthority());
+
+ StringBuilder sb = new StringBuilder();
+
+ if (endpoint.igfs() != null)
+ sb.append(endpoint.igfs());
+
+ if (endpoint.grid() != null)
+ sb.append(":").append(endpoint.grid());
+
+ return new URI(uri.getScheme(), sb.length() != 0 ? sb.toString() : null, endpoint.host(), endpoint.port(),
+ uri.getPath(), uri.getQuery(), uri.getFragment());
+ }
+ catch (URISyntaxException | IgniteCheckedException e) {
+ throw new IOException("Failed to normalize URI: " + uri, e);
+ }
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param connStr Connection string.
+ * @throws IgniteCheckedException If failed to parse connection string.
+ */
+ public HadoopIgfsEndpoint(@Nullable String connStr) throws IgniteCheckedException {
+ if (connStr == null)
+ connStr = "";
+
+ String[] tokens = connStr.split("@", -1);
+
+ IgniteBiTuple<String, Integer> hostPort;
+
+ if (tokens.length == 1) {
+ igfsName = null;
+ gridName = null;
+
+ hostPort = hostPort(connStr, connStr);
+ }
+ else if (tokens.length == 2) {
+ String authStr = tokens[0];
+
+ if (authStr.isEmpty()) {
+ gridName = null;
+ igfsName = null;
+ }
+ else {
+ String[] authTokens = authStr.split(":", -1);
+
+ igfsName = F.isEmpty(authTokens[0]) ? null : authTokens[0];
+
+ if (authTokens.length == 1)
+ gridName = null;
+ else if (authTokens.length == 2)
+ gridName = F.isEmpty(authTokens[1]) ? null : authTokens[1];
+ else
+ throw new IgniteCheckedException("Invalid connection string format: " + connStr);
+ }
+
+ hostPort = hostPort(connStr, tokens[1]);
+ }
+ else
+ throw new IgniteCheckedException("Invalid connection string format: " + connStr);
+
+ host = hostPort.get1();
+
+ assert hostPort.get2() != null;
+
+ port = hostPort.get2();
+ }
+
+ /**
+ * Parse host and port.
+ *
+ * @param connStr Full connection string.
+ * @param hostPortStr Host/port connection string part.
+ * @return Tuple with host and port.
+ * @throws IgniteCheckedException If failed to parse connection string.
+ */
+ private IgniteBiTuple<String, Integer> hostPort(String connStr, String hostPortStr) throws IgniteCheckedException {
+ String[] tokens = hostPortStr.split(":", -1);
+
+ String host = tokens[0];
+
+ if (F.isEmpty(host))
+ host = LOCALHOST;
+
+ int port;
+
+ if (tokens.length == 1)
+ port = DFLT_IPC_PORT;
+ else if (tokens.length == 2) {
+ String portStr = tokens[1];
+
+ try {
+ port = Integer.valueOf(portStr);
+
+ if (port < 0 || port > 65535)
+ throw new IgniteCheckedException("Invalid port number: " + connStr);
+ }
+ catch (NumberFormatException e) {
+ throw new IgniteCheckedException("Invalid port number: " + connStr);
+ }
+ }
+ else
+ throw new IgniteCheckedException("Invalid connection string format: " + connStr);
+
+ return F.t(host, port);
+ }
+
+ /**
+ * @return IGFS name.
+ */
+ @Nullable public String igfs() {
+ return igfsName;
+ }
+
+ /**
+ * @return Grid name.
+ */
+ @Nullable public String grid() {
+ return gridName;
+ }
+
+ /**
+ * @return Host.
+ */
+ public String host() {
+ return host;
+ }
+
+ /**
+ * @return Host.
+ */
+ public boolean isLocal() {
+ return F.eq(LOCALHOST, host);
+ }
+
+ /**
+ * @return Port.
+ */
+ public int port() {
+ return port;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopIgfsEndpoint.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEx.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEx.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEx.java
new file mode 100644
index 0000000..2200e78
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEx.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.ignite.internal.util.lang.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * Extended IGFS server interface.
+ */
+public interface HadoopIgfsEx extends HadoopIgfs {
+ /**
+ * Adds event listener that will be invoked when connection with server is lost or remote error has occurred.
+ * If connection is closed already, callback will be invoked synchronously inside this method.
+ *
+ * @param delegate Stream delegate.
+ * @param lsnr Event listener.
+ */
+ public void addEventListener(HadoopIgfsStreamDelegate delegate, HadoopIgfsStreamEventListener lsnr);
+
+ /**
+ * Removes event listener that will be invoked when connection with server is lost or remote error has occurred.
+ *
+ * @param delegate Stream delegate.
+ */
+ public void removeEventListener(HadoopIgfsStreamDelegate delegate);
+
+ /**
+ * Asynchronously reads specified amount of bytes from opened input stream.
+ *
+ * @param delegate Stream delegate.
+ * @param pos Position to read from.
+ * @param len Data length to read.
+ * @param outBuf Optional output buffer. If buffer length is less then {@code len}, all remaining
+ * bytes will be read into new allocated buffer of length {len - outBuf.length} and this buffer will
+ * be the result of read future.
+ * @param outOff Output offset.
+ * @param outLen Output length.
+ * @return Read data.
+ */
+ public GridPlainFuture<byte[]> readData(HadoopIgfsStreamDelegate delegate, long pos, int len,
+ @Nullable final byte[] outBuf, final int outOff, final int outLen);
+
+ /**
+ * Writes data to the stream with given streamId. This method does not return any future since
+ * no response to write request is sent.
+ *
+ * @param delegate Stream delegate.
+ * @param data Data to write.
+ * @param off Offset.
+ * @param len Length.
+ * @throws IOException If failed.
+ */
+ public void writeData(HadoopIgfsStreamDelegate delegate, byte[] data, int off, int len) throws IOException;
+
+ /**
+ * Close server stream.
+ *
+ * @param delegate Stream delegate.
+ * @throws IOException If failed.
+ */
+ public void closeStream(HadoopIgfsStreamDelegate delegate) throws IOException;
+
+ /**
+ * Flush output stream.
+ *
+ * @param delegate Stream delegate.
+ * @throws IOException If failed.
+ */
+ public void flush(HadoopIgfsStreamDelegate delegate) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsFuture.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsFuture.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsFuture.java
new file mode 100644
index 0000000..59a8f49
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsFuture.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.ignite.internal.util.lang.*;
+import org.jetbrains.annotations.*;
+
+/**
+ * IGFS client future that holds response parse closure.
+ */
+public class HadoopIgfsFuture<T> extends GridPlainFutureAdapter<T> {
+ /** Output buffer. */
+ private byte[] outBuf;
+
+ /** Output offset. */
+ private int outOff;
+
+ /** Output length. */
+ private int outLen;
+
+ /** Read future flag. */
+ private boolean read;
+
+ /**
+ * @return Output buffer.
+ */
+ public byte[] outputBuffer() {
+ return outBuf;
+ }
+
+ /**
+ * @param outBuf Output buffer.
+ */
+ public void outputBuffer(@Nullable byte[] outBuf) {
+ this.outBuf = outBuf;
+ }
+
+ /**
+ * @return Offset in output buffer to write from.
+ */
+ public int outputOffset() {
+ return outOff;
+ }
+
+ /**
+ * @param outOff Offset in output buffer to write from.
+ */
+ public void outputOffset(int outOff) {
+ this.outOff = outOff;
+ }
+
+ /**
+ * @return Length to write to output buffer.
+ */
+ public int outputLength() {
+ return outLen;
+ }
+
+ /**
+ * @param outLen Length to write to output buffer.
+ */
+ public void outputLength(int outLen) {
+ this.outLen = outLen;
+ }
+
+ /**
+ * @param read {@code True} if this is a read future.
+ */
+ public void read(boolean read) {
+ this.read = read;
+ }
+
+ /**
+ * @return {@code True} if this is a read future.
+ */
+ public boolean read() {
+ return read;
+ }
+}
[11/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemAbstractSelfTest.java
deleted file mode 100644
index 7359fdf..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemAbstractSelfTest.java
+++ /dev/null
@@ -1,2366 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.*;
-import org.apache.ignite.*;
-import org.apache.ignite.cache.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.hadoop.*;
-import org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem;
-import org.apache.ignite.internal.igfs.hadoop.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.lang.*;
-import org.apache.ignite.spi.communication.*;
-import org.apache.ignite.spi.communication.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
-import org.apache.ignite.testframework.*;
-import org.jdk8.backport.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.lang.reflect.*;
-import java.net.*;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.cache.CacheMode.*;
-import static org.apache.ignite.events.EventType.*;
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * Test hadoop file system implementation.
- */
-@SuppressWarnings("all")
-public abstract class IgfsHadoopFileSystemAbstractSelfTest extends IgfsCommonAbstractTest {
- /** Primary file system authority. */
- private static final String PRIMARY_AUTHORITY = "igfs:grid0@";
-
- /** Primary file systme URI. */
- private static final String PRIMARY_URI = "igfs://" + PRIMARY_AUTHORITY + "/";
-
- /** Secondary file system authority. */
- private static final String SECONDARY_AUTHORITY = "igfs_secondary:grid_secondary@127.0.0.1:11500";
-
- /** Secondary file systme URI. */
- private static final String SECONDARY_URI = "igfs://" + SECONDARY_AUTHORITY + "/";
-
- /** Secondary file system configuration path. */
- private static final String SECONDARY_CFG_PATH = "/work/core-site-test.xml";
-
- /** Secondary endpoint configuration. */
- protected static final Map<String, String> SECONDARY_ENDPOINT_CFG = new HashMap<String, String>() {{
- put("type", "tcp");
- put("port", "11500");
- }};
-
- /** Group size. */
- public static final int GRP_SIZE = 128;
-
- /** Path to the default hadoop configuration. */
- public static final String HADOOP_FS_CFG = "examples/config/filesystem/core-site.xml";
-
- /** Thread count for multithreaded tests. */
- private static final int THREAD_CNT = 8;
-
- /** IP finder. */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** Barrier for multithreaded tests. */
- private static CyclicBarrier barrier;
-
- /** File system. */
- private static FileSystem fs;
-
- /** Default IGFS mode. */
- protected final IgfsMode mode;
-
- /** Skip embedded mode flag. */
- private final boolean skipEmbed;
-
- /** Skip local shmem flag. */
- private final boolean skipLocShmem;
-
- /** Endpoint. */
- private final String endpoint;
-
- /** Primary file system URI. */
- protected URI primaryFsUri;
-
- /** Primary file system configuration. */
- protected Configuration primaryFsCfg;
-
- /** File statuses comparator. */
- private static final Comparator<FileStatus> STATUS_COMPARATOR = new Comparator<FileStatus>() {
- @SuppressWarnings("deprecation")
- @Override public int compare(FileStatus o1, FileStatus o2) {
- if (o1 == null || o2 == null)
- return o1 == o2 ? 0 : o1 == null ? -1 : 1;
-
- return o1.isDir() == o2.isDir() ? o1.getPath().compareTo(o2.getPath()) : o1.isDir() ? -1 : 1;
- }
- };
-
- /**
- * Constructor.
- *
- * @param mode Default IGFS mode.
- * @param skipEmbed Whether to skip embedded mode.
- * @param skipLocShmem Whether to skip local shmem mode.
- * @param skipLocTcp Whether to skip local TCP mode.
- */
- protected IgfsHadoopFileSystemAbstractSelfTest(IgfsMode mode, boolean skipEmbed, boolean skipLocShmem) {
- this.mode = mode;
- this.skipEmbed = skipEmbed;
- this.skipLocShmem = skipLocShmem;
-
- endpoint = skipLocShmem ? "127.0.0.1:10500" : "shmem:10500";
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- Configuration secondaryConf = configuration(SECONDARY_AUTHORITY, true, true);
-
- secondaryConf.setInt("fs.igfs.block.size", 1024);
-
- String path = U.getIgniteHome() + SECONDARY_CFG_PATH;
-
- File file = new File(path);
-
- try (FileOutputStream fos = new FileOutputStream(file)) {
- secondaryConf.writeXml(fos);
- }
-
- startNodes();
- }
-
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return 10 * 60 * 1000;
- }
-
- /**
- * Starts the nodes for this test.
- *
- * @throws Exception If failed.
- */
- private void startNodes() throws Exception {
- if (mode != PRIMARY) {
- // Start secondary IGFS.
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("partitioned");
- igfsCfg.setMetaCacheName("replicated");
- igfsCfg.setName("igfs_secondary");
- igfsCfg.setIpcEndpointConfiguration(SECONDARY_ENDPOINT_CFG);
- igfsCfg.setBlockSize(512 * 1024);
- igfsCfg.setPrefetchBlocks(1);
-
- CacheConfiguration cacheCfg = defaultCacheConfiguration();
-
- cacheCfg.setName("partitioned");
- cacheCfg.setCacheMode(PARTITIONED);
- cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
- cacheCfg.setBackups(0);
- cacheCfg.setQueryIndexEnabled(false);
- cacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("replicated");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- IgniteConfiguration cfg = new IgniteConfiguration();
-
- cfg.setGridName("grid_secondary");
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
- cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
-
- cfg.setCommunicationSpi(communicationSpi());
-
- G.start(cfg);
- }
-
- startGrids(4);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- G.stopAll(true);
-
- String path = U.getIgniteHome() + SECONDARY_CFG_PATH;
-
- new File(path).delete();
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- primaryFsUri = new URI(PRIMARY_URI);
-
- primaryFsCfg = configuration(PRIMARY_AUTHORITY, skipEmbed, skipLocShmem);
-
- fs = FileSystem.get(primaryFsUri, primaryFsCfg);
-
- barrier = new CyclicBarrier(THREAD_CNT);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- try {
- fs.delete(new Path("/"), true);
- }
- catch (Exception ignore) {
- // No-op.
- }
-
- U.closeQuiet(fs);
- }
-
- /**
- * Get primary IPC endpoint configuration.
- *
- * @param gridName Grid name.
- * @return IPC primary endpoint configuration.
- */
- protected abstract Map<String, String> primaryIpcEndpointConfiguration(String gridName);
-
- /** {@inheritDoc} */
- @Override public String getTestGridName() {
- return "grid";
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(IP_FINDER);
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(cacheConfiguration(gridName));
- cfg.setIgfsConfiguration(igfsConfiguration(gridName));
- cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
- cfg.setCommunicationSpi(communicationSpi());
-
- return cfg;
- }
-
- /**
- * Gets cache configuration.
- *
- * @param gridName Grid name.
- * @return Cache configuration.
- */
- protected CacheConfiguration[] cacheConfiguration(String gridName) {
- CacheConfiguration cacheCfg = defaultCacheConfiguration();
-
- cacheCfg.setName("partitioned");
- cacheCfg.setCacheMode(PARTITIONED);
- cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
- cacheCfg.setBackups(0);
- cacheCfg.setQueryIndexEnabled(false);
- cacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("replicated");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- return new CacheConfiguration[] {metaCacheCfg, cacheCfg};
- }
-
- /**
- * Gets IGFS configuration.
- *
- * @param gridName Grid name.
- * @return IGFS configuration.
- */
- protected IgfsConfiguration igfsConfiguration(String gridName) throws IgniteCheckedException {
- IgfsConfiguration cfg = new IgfsConfiguration();
-
- cfg.setDataCacheName("partitioned");
- cfg.setMetaCacheName("replicated");
- cfg.setName("igfs");
- cfg.setPrefetchBlocks(1);
- cfg.setDefaultMode(mode);
-
- if (mode != PRIMARY)
- cfg.setSecondaryFileSystem(new IgfsHadoopFileSystemWrapper(SECONDARY_URI, SECONDARY_CFG_PATH));
-
- cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(gridName));
-
- cfg.setManagementPort(-1);
- cfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups.
-
- return cfg;
- }
-
- /** @return Communication SPI. */
- private CommunicationSpi communicationSpi() {
- TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
-
- commSpi.setSharedMemoryPort(-1);
-
- return commSpi;
- }
-
- /** @throws Exception If failed. */
- public void testGetUriIfFSIsNotInitialized() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return new IgfsHadoopFileSystem().getUri();
- }
- }, IllegalStateException.class, "URI is null (was IgfsHadoopFileSystem properly initialized?).");
- }
-
- /** @throws Exception If failed. */
- @SuppressWarnings("NullableProblems")
- public void testInitializeCheckParametersNameIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- new IgfsHadoopFileSystem().initialize(null, new Configuration());
-
- return null;
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: name");
- }
-
- /** @throws Exception If failed. */
- @SuppressWarnings("NullableProblems")
- public void testInitializeCheckParametersCfgIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- new IgfsHadoopFileSystem().initialize(new URI(""), null);
-
- return null;
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: cfg");
- }
-
- /** @throws Exception If failed. */
- public void testInitialize() throws Exception {
- final IgfsHadoopFileSystem fs = new IgfsHadoopFileSystem();
-
- fs.initialize(primaryFsUri, primaryFsCfg);
-
- // Check repeatable initialization.
- try {
- fs.initialize(primaryFsUri, primaryFsCfg);
-
- fail();
- }
- catch (IOException e) {
- assertTrue(e.getMessage().contains("File system is already initialized"));
- }
-
- assertEquals(primaryFsUri, fs.getUri());
-
- assertEquals(0, fs.getUsed());
-
- fs.close();
- }
-
- /**
- * Test how IPC cache map works.
- *
- * @throws Exception If failed.
- */
- public void testIpcCache() throws Exception {
- IgfsHadoopEx hadoop = GridTestUtils.getFieldValue(fs, "rmtClient", "delegateRef", "value", "hadoop");
-
- if (hadoop instanceof IgfsHadoopOutProc) {
- FileSystem fsOther = null;
-
- try {
- Field field = IgfsHadoopIpcIo.class.getDeclaredField("ipcCache");
-
- field.setAccessible(true);
-
- Map<String, IgfsHadoopIpcIo> cache = (Map<String, IgfsHadoopIpcIo>)field.get(null);
-
- Configuration cfg = configuration(PRIMARY_AUTHORITY, skipEmbed, skipLocShmem);
-
- // we disable caching in order to obtain new FileSystem instance.
- cfg.setBoolean("fs.igfs.impl.disable.cache", true);
-
- // Initial cache size.
- int initSize = cache.size();
-
- // Ensure that when IO is used by multiple file systems and one of them is closed, IO is not stopped.
- fsOther = FileSystem.get(new URI(PRIMARY_URI), cfg);
-
- assert fs != fsOther;
-
- assertEquals(initSize, cache.size());
-
- fsOther.close();
-
- assertEquals(initSize, cache.size());
-
- Field stopField = IgfsHadoopIpcIo.class.getDeclaredField("stopping");
-
- stopField.setAccessible(true);
-
- IgfsHadoopIpcIo io = null;
-
- for (Map.Entry<String, IgfsHadoopIpcIo> ioEntry : cache.entrySet()) {
- if (endpoint.contains(ioEntry.getKey())) {
- io = ioEntry.getValue();
-
- break;
- }
- }
-
- assert io != null;
-
- assert !(Boolean)stopField.get(io);
-
- // Ensure that IO is stopped when nobody else is need it.
- fs.close();
-
- assertEquals(initSize - 1, cache.size());
-
- assert (Boolean)stopField.get(io);
- }
- finally {
- U.closeQuiet(fsOther);
- }
- }
- }
-
- /** @throws Exception If failed. */
- public void testCloseIfNotInitialized() throws Exception {
- final FileSystem fs = new IgfsHadoopFileSystem();
-
- // Check close makes nothing harmful.
- fs.close();
- }
-
- /** @throws Exception If failed. */
- public void testClose() throws Exception {
- final Path path = new Path("dir");
-
- fs.close();
-
- // Check double close makes nothing harmful.
- fs.close();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Nullable @Override public Object call() throws Exception {
- fs.initialize(primaryFsUri, primaryFsCfg);
-
- return null;
- }
- }, IOException.class, "File system is stopped.");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Nullable @Override public Object call() throws Exception {
- fs.setPermission(path, FsPermission.createImmutable((short)777));
-
- return null;
- }
- }, IOException.class, "File system is stopped.");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Nullable @Override public Object call() throws Exception {
- fs.setOwner(path, "user", "group");
-
- return null;
- }
- }, IOException.class, "File system is stopped.");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.open(path, 256);
- }
- }, IOException.class, "File system is stopped.");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.create(path);
- }
- }, IOException.class, "File system is stopped.");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.append(path);
- }
- }, IOException.class, "File system is stopped.");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.rename(path, new Path("newDir"));
- }
- }, IOException.class, "File system is stopped.");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.delete(path, true);
- }
- }, IOException.class, "File system is stopped.");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.listStatus(path);
- }
- }, IOException.class, "File system is stopped.");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.mkdirs(path);
- }
- }, IOException.class, "File system is stopped.");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.getFileStatus(path);
- }
- }, IOException.class, "File system is stopped.");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.getFileBlockLocations(new FileStatus(1L, false, 1, 1L, 1L, new Path("path")), 0L, 256L);
- }
- }, IOException.class, "File system is stopped.");
- }
-
- /** @throws Exception If failed. */
- public void testCreateCheckParameters() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.create(null);
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: f");
- }
-
- /** @throws Exception If failed. */
- @SuppressWarnings("deprecation")
- public void testCreateBase() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
- Path file = new Path(dir, "someFile");
-
- assertPathDoesNotExist(fs, file);
-
- FsPermission fsPerm = new FsPermission((short)644);
-
- FSDataOutputStream os = fs.create(file, fsPerm, false, 1, (short)1, 1L, null);
-
- // Try to write something in file.
- os.write("abc".getBytes());
-
- os.close();
-
- // Check file status.
- FileStatus fileStatus = fs.getFileStatus(file);
-
- assertFalse(fileStatus.isDir());
- assertEquals(file, fileStatus.getPath());
- assertEquals(fsPerm, fileStatus.getPermission());
- }
-
- /** @throws Exception If failed. */
- @SuppressWarnings("deprecation")
- public void testCreateCheckOverwrite() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
- final Path file = new Path(dir, "someFile");
-
- FSDataOutputStream out = fs.create(file, FsPermission.getDefault(), false, 64 * 1024,
- fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
-
- out.close();
-
- // Check intermediate directory permissions.
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir).getPermission());
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir.getParent()).getPermission());
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir.getParent().getParent()).getPermission());
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.create(file, FsPermission.getDefault(), false, 1024, (short)1, 2048, null);
- }
- }, PathExistsException.class, null);
-
- // Overwrite should be successful.
- FSDataOutputStream out1 = fs.create(file, true);
-
- out1.close();
- }
-
- /** @throws Exception If failed. */
- public void testDeleteIfNoSuchPath() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
-
- assertPathDoesNotExist(fs, dir);
-
- assertFalse(fs.delete(dir, true));
- }
-
- /** @throws Exception If failed. */
- public void testDeleteSuccessfulIfPathIsOpenedToRead() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "myFile");
-
- FSDataOutputStream os = fs.create(file, false, 128);
-
- final int cnt = 5 * IgfsConfiguration.DFLT_BLOCK_SIZE; // Write 5 blocks.
-
- for (int i = 0; i < cnt; i++)
- os.writeInt(i);
-
- os.close();
-
- final FSDataInputStream is = fs.open(file, -1);
-
- for (int i = 0; i < cnt / 2; i++)
- assertEquals(i, is.readInt());
-
- assert fs.delete(file, false);
-
- assert !fs.exists(file);
-
- is.close();
- }
-
- /** @throws Exception If failed. */
- public void testDeleteIfFilePathExists() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "myFile");
-
- FSDataOutputStream os = fs.create(file);
-
- os.close();
-
- assertTrue(fs.delete(file, false));
-
- assertPathDoesNotExist(fs, file);
- }
-
- /** @throws Exception If failed. */
- public void testDeleteIfDirectoryPathExists() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
-
- FSDataOutputStream os = fs.create(dir);
-
- os.close();
-
- assertTrue(fs.delete(dir, false));
-
- assertPathDoesNotExist(fs, dir);
- }
-
- /** @throws Exception If failed. */
- public void testDeleteFailsIfNonRecursive() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
-
- fs.create(someDir3).close();
-
- Path someDir2 = new Path(fsHome, "/someDir1/someDir2");
-
- assertFalse(fs.delete(someDir2, false));
-
- assertPathExists(fs, someDir2);
- assertPathExists(fs, someDir3);
- }
-
- /** @throws Exception If failed. */
- public void testDeleteRecursively() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
-
- FSDataOutputStream os = fs.create(someDir3);
-
- os.close();
-
- Path someDir2 = new Path(fsHome, "/someDir1/someDir2");
-
- assertTrue(fs.delete(someDir2, true));
-
- assertPathDoesNotExist(fs, someDir2);
- assertPathDoesNotExist(fs, someDir3);
- }
-
- /** @throws Exception If failed. */
- public void testDeleteRecursivelyFromRoot() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
-
- FSDataOutputStream os = fs.create(someDir3);
-
- os.close();
-
- Path root = new Path(fsHome, "/");
-
- assertTrue(fs.delete(root, true));
-
- assertPathDoesNotExist(fs, someDir3);
- assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));
- assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1"));
- assertPathExists(fs, root);
- }
-
- /** @throws Exception If failed. */
- @SuppressWarnings("deprecation")
- public void testSetPermissionCheckDefaultPermission() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file, FsPermission.getDefault(), false, 64 * 1024,
- fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
-
- os.close();
-
- fs.setPermission(file, null);
-
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(file.getParent()).getPermission());
- }
-
- /** @throws Exception If failed. */
- @SuppressWarnings("deprecation")
- public void testSetPermissionCheckNonRecursiveness() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file, FsPermission.getDefault(), false, 64 * 1024,
- fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
-
- os.close();
-
- Path tmpDir = new Path(fsHome, "/tmp");
-
- FsPermission perm = new FsPermission((short)123);
-
- fs.setPermission(tmpDir, perm);
-
- assertEquals(perm, fs.getFileStatus(tmpDir).getPermission());
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
- }
-
- /** @throws Exception If failed. */
- @SuppressWarnings("OctalInteger")
- public void testSetPermission() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file);
-
- os.close();
-
- for (short i = 0; i <= 0777; i += 7) {
- FsPermission perm = new FsPermission(i);
-
- fs.setPermission(file, perm);
-
- assertEquals(perm, fs.getFileStatus(file).getPermission());
- }
- }
-
- /** @throws Exception If failed. */
- public void testSetPermissionIfOutputStreamIsNotClosed() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "myFile");
-
- FsPermission perm = new FsPermission((short)123);
-
- FSDataOutputStream os = fs.create(file);
-
- fs.setPermission(file, perm);
-
- os.close();
-
- assertEquals(perm, fs.getFileStatus(file).getPermission());
- }
-
- /** @throws Exception If failed. */
- public void testSetOwnerCheckParametersPathIsNull() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file);
-
- os.close();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.setOwner(null, "aUser", "aGroup");
-
- return null;
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: p");
- }
-
- /** @throws Exception If failed. */
- public void testSetOwnerCheckParametersUserIsNull() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file);
-
- os.close();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.setOwner(file, null, "aGroup");
-
- return null;
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: username");
- }
-
- /** @throws Exception If failed. */
- public void testSetOwnerCheckParametersGroupIsNull() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file);
-
- os.close();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- fs.setOwner(file, "aUser", null);
-
- return null;
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: grpName");
- }
-
- /** @throws Exception If failed. */
- public void testSetOwner() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file);
-
- os.close();
-
- fs.setOwner(file, "aUser", "aGroup");
-
- assertEquals("aUser", fs.getFileStatus(file).getOwner());
- assertEquals("aGroup", fs.getFileStatus(file).getGroup());
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testSetTimes() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "/heartbeat");
-
- fs.create(file).close();
-
- FileStatus status = fs.getFileStatus(file);
-
- assertTrue(status.getAccessTime() > 0);
- assertTrue(status.getModificationTime() > 0);
-
- long mtime = System.currentTimeMillis() - 5000;
- long atime = System.currentTimeMillis() - 4000;
-
- fs.setTimes(file, mtime, atime);
-
- status = fs.getFileStatus(file);
-
- assertEquals(mtime, status.getModificationTime());
- assertEquals(atime, status.getAccessTime());
-
- mtime -= 5000;
-
- fs.setTimes(file, mtime, -1);
-
- status = fs.getFileStatus(file);
-
- assertEquals(mtime, status.getModificationTime());
- assertEquals(atime, status.getAccessTime());
-
- atime -= 5000;
-
- fs.setTimes(file, -1, atime);
-
- status = fs.getFileStatus(file);
-
- assertEquals(mtime, status.getModificationTime());
- assertEquals(atime, status.getAccessTime());
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testSetOwnerIfOutputStreamIsNotClosed() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "myFile");
-
- FSDataOutputStream os = fs.create(file);
-
- fs.setOwner(file, "aUser", "aGroup");
-
- os.close();
-
- assertEquals("aUser", fs.getFileStatus(file).getOwner());
- assertEquals("aGroup", fs.getFileStatus(file).getGroup());
- }
-
- /** @throws Exception If failed. */
- public void testSetOwnerCheckNonRecursiveness() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file);
-
- os.close();
-
- Path tmpDir = new Path(fsHome, "/tmp");
-
- fs.setOwner(file, "fUser", "fGroup");
- fs.setOwner(tmpDir, "dUser", "dGroup");
-
- assertEquals("dUser", fs.getFileStatus(tmpDir).getOwner());
- assertEquals("dGroup", fs.getFileStatus(tmpDir).getGroup());
-
- assertEquals("fUser", fs.getFileStatus(file).getOwner());
- assertEquals("fGroup", fs.getFileStatus(file).getGroup());
- }
-
- /** @throws Exception If failed. */
- public void testOpenCheckParametersPathIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.open(null, 1024);
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: f");
- }
-
- /** @throws Exception If failed. */
- public void testOpenNoSuchPath() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "someFile");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.open(file, 1024);
- }
- }, FileNotFoundException.class, null);
- }
-
- /** @throws Exception If failed. */
- public void testOpenIfPathIsAlreadyOpened() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "someFile");
-
- FSDataOutputStream os = fs.create(file);
-
- os.close();
-
- FSDataInputStream is1 = fs.open(file);
- FSDataInputStream is2 = fs.open(file);
-
- is1.close();
- is2.close();
- }
-
- /** @throws Exception If failed. */
- public void testOpen() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "someFile");
-
- int cnt = 2 * 1024;
-
- FSDataOutputStream out = fs.create(file, true, 1024);
-
- for (long i = 0; i < cnt; i++)
- out.writeLong(i);
-
- out.close();
-
- FSDataInputStream in = fs.open(file, 1024);
-
- for (long i = 0; i < cnt; i++)
- assertEquals(i, in.readLong());
-
- in.close();
- }
-
- /** @throws Exception If failed. */
- public void testAppendCheckParametersPathIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.append(null);
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: f");
- }
-
- /** @throws Exception If failed. */
- public void testAppendIfPathPointsToDirectory() throws Exception {
- final Path fsHome = new Path(primaryFsUri);
- final Path dir = new Path(fsHome, "/tmp");
- Path file = new Path(dir, "my");
-
- FSDataOutputStream os = fs.create(file);
-
- os.close();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.append(new Path(fsHome, dir), 1024);
- }
- }, IOException.class, null);
- }
-
- /** @throws Exception If failed. */
- public void testAppendIfFileIsAlreadyBeingOpenedToWrite() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "someFile");
-
- FSDataOutputStream os = fs.create(file);
-
- os.close();
-
- FSDataOutputStream appendOs = fs.append(file);
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- return fs.append(file);
- }
- }, IOException.class, null);
-
- appendOs.close();
- }
-
- /** @throws Exception If failed. */
- public void testAppend() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "someFile");
-
- int cnt = 1024;
-
- FSDataOutputStream out = fs.create(file, true, 1024);
-
- for (int i = 0; i < cnt; i++)
- out.writeLong(i);
-
- out.close();
-
- out = fs.append(file);
-
- for (int i = cnt; i < cnt * 2; i++)
- out.writeLong(i);
-
- out.close();
-
- FSDataInputStream in = fs.open(file, 1024);
-
- for (int i = 0; i < cnt * 2; i++)
- assertEquals(i, in.readLong());
-
- in.close();
- }
-
- /** @throws Exception If failed. */
- public void testRenameCheckParametersSrcPathIsNull() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "someFile");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.rename(null, file);
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: src");
- }
-
- /** @throws Exception If failed. */
- public void testRenameCheckParametersDstPathIsNull() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "someFile");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- return fs.rename(file, null);
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: dst");
- }
-
- /** @throws Exception If failed. */
- public void testRenameIfSrcPathDoesNotExist() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path srcFile = new Path(fsHome, "srcFile");
- Path dstFile = new Path(fsHome, "dstFile");
-
- assertPathDoesNotExist(fs, srcFile);
-
- assertFalse(fs.rename(srcFile, dstFile));
-
- assertPathDoesNotExist(fs, dstFile);
- }
-
- /** @throws Exception If failed. */
- public void testRenameIfSrcPathIsAlreadyBeingOpenedToWrite() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path srcFile = new Path(fsHome, "srcFile");
- Path dstFile = new Path(fsHome, "dstFile");
-
- FSDataOutputStream os = fs.create(srcFile);
-
- os.close();
-
- os = fs.append(srcFile);
-
- assertTrue(fs.rename(srcFile, dstFile));
-
- assertPathExists(fs, dstFile);
-
- String testStr = "Test";
-
- try {
- os.writeBytes(testStr);
- }
- finally {
- os.close();
- }
-
- try (FSDataInputStream is = fs.open(dstFile)) {
- byte[] buf = new byte[testStr.getBytes().length];
-
- is.readFully(buf);
-
- assertEquals(testStr, new String(buf));
- }
- }
-
- /** @throws Exception If failed. */
- public void testRenameFileIfDstPathExists() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path srcFile = new Path(fsHome, "srcFile");
- Path dstFile = new Path(fsHome, "dstFile");
-
- FSDataOutputStream os = fs.create(srcFile);
-
- os.close();
-
- os = fs.create(dstFile);
-
- os.close();
-
- assertFalse(fs.rename(srcFile, dstFile));
-
- assertPathExists(fs, srcFile);
- assertPathExists(fs, dstFile);
- }
-
- /** @throws Exception If failed. */
- public void testRenameFile() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path srcFile = new Path(fsHome, "/tmp/srcFile");
- Path dstFile = new Path(fsHome, "/tmp/dstFile");
-
- FSDataOutputStream os = fs.create(srcFile);
-
- os.close();
-
- assertTrue(fs.rename(srcFile, dstFile));
-
- assertPathDoesNotExist(fs, srcFile);
- assertPathExists(fs, dstFile);
- }
-
- /** @throws Exception If failed. */
- public void testRenameIfSrcPathIsAlreadyBeingOpenedToRead() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path srcFile = new Path(fsHome, "srcFile");
- Path dstFile = new Path(fsHome, "dstFile");
-
- FSDataOutputStream os = fs.create(srcFile);
-
- int cnt = 1024;
-
- for (int i = 0; i < cnt; i++)
- os.writeInt(i);
-
- os.close();
-
- FSDataInputStream is = fs.open(srcFile);
-
- for (int i = 0; i < cnt; i++) {
- if (i == 100)
- // Rename file during the read process.
- assertTrue(fs.rename(srcFile, dstFile));
-
- assertEquals(i, is.readInt());
- }
-
- assertPathDoesNotExist(fs, srcFile);
- assertPathExists(fs, dstFile);
-
- os.close();
- is.close();
- }
-
- /** @throws Exception If failed. */
- public void testRenameDirectoryIfDstPathExists() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path srcDir = new Path(fsHome, "/tmp/");
- Path dstDir = new Path(fsHome, "/tmpNew/");
-
- FSDataOutputStream os = fs.create(new Path(srcDir, "file1"));
-
- os.close();
-
- os = fs.create(new Path(dstDir, "file2"));
-
- os.close();
-
- assertTrue("Rename succeeded [srcDir=" + srcDir + ", dstDir=" + dstDir + ']', fs.rename(srcDir, dstDir));
-
- assertPathExists(fs, dstDir);
- assertPathExists(fs, new Path(fsHome, "/tmpNew/tmp"));
- assertPathExists(fs, new Path(fsHome, "/tmpNew/tmp/file1"));
- }
-
- /** @throws Exception If failed. */
- public void testRenameDirectory() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path dir = new Path(fsHome, "/tmp/");
- Path newDir = new Path(fsHome, "/tmpNew/");
-
- FSDataOutputStream os = fs.create(new Path(dir, "myFile"));
-
- os.close();
-
- assertTrue("Rename failed [dir=" + dir + ", newDir=" + newDir + ']', fs.rename(dir, newDir));
-
- assertPathDoesNotExist(fs, dir);
- assertPathExists(fs, newDir);
- }
-
- /** @throws Exception If failed. */
- public void testListStatusIfPathIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.listStatus((Path)null);
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: f");
- }
-
- /** @throws Exception If failed. */
- public void testListStatusIfPathDoesNotExist() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.listStatus(new Path("/tmp/some/dir"));
- }
- }, FileNotFoundException.class, null);
- }
-
- /**
- * Test directory listing.
- *
- * @throws Exception If failed.
- */
- public void testListStatus() throws Exception {
- Path igfsHome = new Path(PRIMARY_URI);
-
- // Test listing of an empty directory.
- Path dir = new Path(igfsHome, "dir");
-
- assert fs.mkdirs(dir);
-
- FileStatus[] list = fs.listStatus(dir);
-
- assert list.length == 0;
-
- // Test listing of a not empty directory.
- Path subDir = new Path(dir, "subDir");
-
- assert fs.mkdirs(subDir);
-
- Path file = new Path(dir, "file");
-
- FSDataOutputStream fos = fs.create(file);
-
- fos.close();
-
- list = fs.listStatus(dir);
-
- assert list.length == 2;
-
- String listRes1 = list[0].getPath().getName();
- String listRes2 = list[1].getPath().getName();
-
- assert "subDir".equals(listRes1) && "file".equals(listRes2) || "subDir".equals(listRes2) &&
- "file".equals(listRes1);
-
- // Test listing of a file.
- list = fs.listStatus(file);
-
- assert list.length == 1;
-
- assert "file".equals(list[0].getPath().getName());
- }
-
- /** @throws Exception If failed. */
- public void testSetWorkingDirectoryIfPathIsNull() throws Exception {
- fs.setWorkingDirectory(null);
-
- Path file = new Path("file");
-
- FSDataOutputStream os = fs.create(file);
- os.close();
-
- String path = fs.getFileStatus(file).getPath().toString();
-
- assertTrue(path.endsWith("/user/" + System.getProperty("user.name", "anonymous") + "/file"));
- }
-
- /** @throws Exception If failed. */
- public void testSetWorkingDirectoryIfPathDoesNotExist() throws Exception {
- // Should not throw any exceptions.
- fs.setWorkingDirectory(new Path("/someDir"));
- }
-
- /** @throws Exception If failed. */
- public void testSetWorkingDirectory() throws Exception {
- Path dir = new Path("/tmp/nested/dir");
- Path file = new Path("file");
-
- fs.mkdirs(dir);
-
- fs.setWorkingDirectory(dir);
-
- FSDataOutputStream os = fs.create(file);
- os.close();
-
- String filePath = fs.getFileStatus(new Path(dir, file)).getPath().toString();
-
- assertTrue(filePath.contains("/tmp/nested/dir/file"));
- }
-
- /** @throws Exception If failed. */
- public void testGetWorkingDirectoryIfDefault() throws Exception {
- String path = fs.getWorkingDirectory().toString();
-
- assertTrue(path.endsWith("/user/" + System.getProperty("user.name", "anonymous")));
- }
-
- /** @throws Exception If failed. */
- public void testGetWorkingDirectory() throws Exception {
- Path dir = new Path("/tmp/some/dir");
-
- fs.mkdirs(dir);
-
- fs.setWorkingDirectory(dir);
-
- String path = fs.getWorkingDirectory().toString();
-
- assertTrue(path.endsWith("/tmp/some/dir"));
- }
-
- /** @throws Exception If failed. */
- public void testMkdirsIfPathIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.mkdirs(null);
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: f");
- }
-
- /** @throws Exception If failed. */
- public void testMkdirsIfPermissionIsNull() throws Exception {
- Path dir = new Path("/tmp");
-
- assertTrue(fs.mkdirs(dir, null));
-
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir).getPermission());
- }
-
- /** @throws Exception If failed. */
- @SuppressWarnings("OctalInteger")
- public void testMkdirs() throws Exception {
- Path fsHome = new Path(PRIMARY_URI);
- Path dir = new Path(fsHome, "/tmp/staging");
- Path nestedDir = new Path(dir, "nested");
-
- FsPermission dirPerm = FsPermission.createImmutable((short)0700);
- FsPermission nestedDirPerm = FsPermission.createImmutable((short)111);
-
- assertTrue(fs.mkdirs(dir, dirPerm));
- assertTrue(fs.mkdirs(nestedDir, nestedDirPerm));
-
- assertEquals(dirPerm, fs.getFileStatus(dir).getPermission());
- assertEquals(nestedDirPerm, fs.getFileStatus(nestedDir).getPermission());
- }
-
- /** @throws Exception If failed. */
- public void testGetFileStatusIfPathIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.getFileStatus(null);
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: f");
- }
-
- /** @throws Exception If failed. */
- public void testGetFileStatusIfPathDoesNotExist() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.getFileStatus(new Path("someDir"));
- }
- }, FileNotFoundException.class, "File not found: someDir");
- }
-
- /** @throws Exception If failed. */
- public void testGetFileBlockLocationsIfFileStatusIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- // Argument is checked by Hadoop.
- return fs.getFileBlockLocations((Path)null, 1, 2);
- }
- }, NullPointerException.class, null);
- }
-
- /** @throws Exception If failed. */
- public void testGetFileBlockLocationsIfFileStatusReferenceNotExistingPath() throws Exception {
- Path path = new Path("someFile");
-
- fs.create(path).close();
-
- final FileStatus status = fs.getFileStatus(path);
-
- fs.delete(path, true);
-
- BlockLocation[] locations = fs.getFileBlockLocations(status, 1, 2);
-
- assertEquals(0, locations.length);
- }
-
- /** @throws Exception If failed. */
- public void testGetFileBlockLocations() throws Exception {
- Path igfsHome = new Path(PRIMARY_URI);
-
- Path file = new Path(igfsHome, "someFile");
-
- try (OutputStream out = new BufferedOutputStream(fs.create(file, true, 1024 * 1024))) {
- byte[] data = new byte[128 * 1024];
-
- for (int i = 0; i < 100; i++)
- out.write(data);
-
- out.flush();
- }
-
- try (FSDataInputStream in = fs.open(file, 1024 * 1024)) {
- byte[] data = new byte[128 * 1024];
-
- int read;
-
- do {
- read = in.read(data);
- }
- while (read > 0);
- }
-
- FileStatus status = fs.getFileStatus(file);
-
- int grpLen = 128 * 512 * 1024;
-
- int grpCnt = (int)((status.getLen() + grpLen - 1) / grpLen);
-
- BlockLocation[] locations = fs.getFileBlockLocations(status, 0, status.getLen());
-
- assertEquals(grpCnt, locations.length);
- }
-
- /** @throws Exception If failed. */
- @SuppressWarnings("deprecation")
- public void testGetDefaultBlockSize() throws Exception {
- assertEquals(1L << 26, fs.getDefaultBlockSize());
- }
-
- /** @throws Exception If failed. */
- public void testZeroReplicationFactor() throws Exception {
- // This test doesn't make sense for any mode except of PRIMARY.
- if (mode == PRIMARY) {
- Path igfsHome = new Path(PRIMARY_URI);
-
- Path file = new Path(igfsHome, "someFile");
-
- try (FSDataOutputStream out = fs.create(file, (short)0)) {
- out.write(new byte[1024 * 1024]);
- }
-
- IgniteFs igfs = grid(0).fileSystem("igfs");
-
- IgfsPath filePath = new IgfsPath("/someFile");
-
- IgfsFile fileInfo = igfs.info(filePath);
-
- Collection<IgfsBlockLocation> locations = igfs.affinity(filePath, 0, fileInfo.length());
-
- assertEquals(1, locations.size());
-
- IgfsBlockLocation location = F.first(locations);
-
- assertEquals(1, location.nodeIds().size());
- }
- }
-
- /**
- * Ensure that when running in multithreaded mode only one create() operation succeed.
- *
- * @throws Exception If failed.
- */
- public void testMultithreadedCreate() throws Exception {
- Path dir = new Path(new Path(PRIMARY_URI), "/dir");
-
- assert fs.mkdirs(dir);
-
- final Path file = new Path(dir, "file");
-
- fs.create(file).close();
-
- final AtomicInteger cnt = new AtomicInteger();
-
- final Collection<Integer> errs = new GridConcurrentHashSet<>(THREAD_CNT, 1.0f, THREAD_CNT);
-
- final AtomicBoolean err = new AtomicBoolean();
-
- multithreaded(new Runnable() {
- @Override
- public void run() {
- int idx = cnt.getAndIncrement();
-
- byte[] data = new byte[256];
-
- Arrays.fill(data, (byte)idx);
-
- FSDataOutputStream os = null;
-
- try {
- os = fs.create(file, true);
- }
- catch (IOException ignore) {
- errs.add(idx);
- }
-
- U.awaitQuiet(barrier);
-
- try {
- if (os != null)
- os.write(data);
- }
- catch (IOException ignore) {
- err.set(true);
- }
- finally {
- U.closeQuiet(os);
- }
- }
- }, THREAD_CNT);
-
- assert !err.get();
-
- // Only one thread could obtain write lock on the file.
- assert errs.size() == THREAD_CNT - 1;
-
- int idx = -1;
-
- for (int i = 0; i < THREAD_CNT; i++) {
- if (!errs.remove(i)) {
- idx = i;
-
- break;
- }
- }
-
- byte[] expData = new byte[256];
-
- Arrays.fill(expData, (byte)idx);
-
- FSDataInputStream is = fs.open(file);
-
- byte[] data = new byte[256];
-
- is.read(data);
-
- is.close();
-
- assert Arrays.equals(expData, data) : "Expected=" + Arrays.toString(expData) + ", actual=" +
- Arrays.toString(data);
- }
-
- /**
- * Ensure that when running in multithreaded mode only one append() operation succeed.
- *
- * @throws Exception If failed.
- */
- public void testMultithreadedAppend() throws Exception {
- Path dir = new Path(new Path(PRIMARY_URI), "/dir");
-
- assert fs.mkdirs(dir);
-
- final Path file = new Path(dir, "file");
-
- fs.create(file).close();
-
- final AtomicInteger cnt = new AtomicInteger();
-
- final Collection<Integer> errs = new GridConcurrentHashSet<>(THREAD_CNT, 1.0f, THREAD_CNT);
-
- final AtomicBoolean err = new AtomicBoolean();
-
- multithreaded(new Runnable() {
- @Override public void run() {
- int idx = cnt.getAndIncrement();
-
- byte[] data = new byte[256];
-
- Arrays.fill(data, (byte)idx);
-
- U.awaitQuiet(barrier);
-
- FSDataOutputStream os = null;
-
- try {
- os = fs.append(file);
- }
- catch (IOException ignore) {
- errs.add(idx);
- }
-
- U.awaitQuiet(barrier);
-
- try {
- if (os != null)
- os.write(data);
- }
- catch (IOException ignore) {
- err.set(true);
- }
- finally {
- U.closeQuiet(os);
- }
- }
- }, THREAD_CNT);
-
- assert !err.get();
-
- // Only one thread could obtain write lock on the file.
- assert errs.size() == THREAD_CNT - 1;
-
- int idx = -1;
-
- for (int i = 0; i < THREAD_CNT; i++) {
- if (!errs.remove(i)) {
- idx = i;
-
- break;
- }
- }
-
- byte[] expData = new byte[256];
-
- Arrays.fill(expData, (byte)idx);
-
- FSDataInputStream is = fs.open(file);
-
- byte[] data = new byte[256];
-
- is.read(data);
-
- is.close();
-
- assert Arrays.equals(expData, data);
- }
-
- /**
- * Test concurrent reads within the file.
- *
- * @throws Exception If failed.
- */
- public void testMultithreadedOpen() throws Exception {
- final byte[] dataChunk = new byte[256];
-
- for (int i = 0; i < dataChunk.length; i++)
- dataChunk[i] = (byte)i;
-
- Path dir = new Path(new Path(PRIMARY_URI), "/dir");
-
- assert fs.mkdirs(dir);
-
- final Path file = new Path(dir, "file");
-
- FSDataOutputStream os = fs.create(file);
-
- // Write 256 * 2048 = 512Kb of data.
- for (int i = 0; i < 2048; i++)
- os.write(dataChunk);
-
- os.close();
-
- final AtomicBoolean err = new AtomicBoolean();
-
- multithreaded(new Runnable() {
- @Override
- public void run() {
- FSDataInputStream is = null;
-
- try {
- int pos = ThreadLocalRandom8.current().nextInt(2048);
-
- try {
- is = fs.open(file);
- }
- finally {
- U.awaitQuiet(barrier);
- }
-
- is.seek(256 * pos);
-
- byte[] buf = new byte[256];
-
- for (int i = pos; i < 2048; i++) {
- // First perform normal read.
- int read = is.read(buf);
-
- assert read == 256;
-
- Arrays.equals(dataChunk, buf);
- }
-
- int res = is.read(buf);
-
- assert res == -1;
- }
- catch (IOException ignore) {
- err.set(true);
- }
- finally {
- U.closeQuiet(is);
- }
- }
- }, THREAD_CNT);
-
- assert !err.get();
- }
-
- /**
- * Test concurrent creation of multiple directories.
- *
- * @throws Exception If failed.
- */
- public void testMultithreadedMkdirs() throws Exception {
- final Path dir = new Path(new Path(PRIMARY_URI), "/dir");
-
- assert fs.mkdirs(dir);
-
- final int depth = 3;
- final int entryCnt = 5;
-
- final AtomicReference<IOException> err = new AtomicReference();
-
- multithreaded(new Runnable() {
- @Override public void run() {
- Deque<IgniteBiTuple<Integer, Path>> queue = new ArrayDeque<>();
-
- queue.add(F.t(0, dir));
-
- U.awaitQuiet(barrier);
-
- while (!queue.isEmpty()) {
- IgniteBiTuple<Integer, Path> t = queue.pollFirst();
-
- int curDepth = t.getKey();
- Path curPath = t.getValue();
-
- if (curDepth <= depth) {
- int newDepth = curDepth + 1;
-
- // Create directories.
- for (int i = 0; i < entryCnt; i++) {
- Path subDir = new Path(curPath, "dir-" + newDepth + "-" + i);
-
- try {
- if (fs.mkdirs(subDir))
- queue.addLast(F.t(newDepth, subDir));
- }
- catch (IOException e) {
- err.compareAndSet(null, e);
- }
- }
- }
- }
- }
- }, THREAD_CNT);
-
- // Ensure there were no errors.
- assert err.get() == null : err.get();
-
- // Ensure correct folders structure.
- Deque<IgniteBiTuple<Integer, Path>> queue = new ArrayDeque<>();
-
- queue.add(F.t(0, dir));
-
- while (!queue.isEmpty()) {
- IgniteBiTuple<Integer, Path> t = queue.pollFirst();
-
- int curDepth = t.getKey();
- Path curPath = t.getValue();
-
- if (curDepth <= depth) {
- int newDepth = curDepth + 1;
-
- // Create directories.
- for (int i = 0; i < entryCnt; i++) {
- Path subDir = new Path(curPath, "dir-" + newDepth + "-" + i);
-
- assert fs.exists(subDir) : "Expected directory doesn't exist: " + subDir;
-
- queue.add(F.t(newDepth, subDir));
- }
- }
- }
- }
-
- /**
- * Test concurrent deletion of the same directory with advanced structure.
- *
- * @throws Exception If failed.
- */
- @SuppressWarnings("TooBroadScope")
- public void testMultithreadedDelete() throws Exception {
- final Path dir = new Path(new Path(PRIMARY_URI), "/dir");
-
- assert fs.mkdirs(dir);
-
- int depth = 3;
- int entryCnt = 5;
-
- Deque<IgniteBiTuple<Integer, Path>> queue = new ArrayDeque<>();
-
- queue.add(F.t(0, dir));
-
- while (!queue.isEmpty()) {
- IgniteBiTuple<Integer, Path> t = queue.pollFirst();
-
- int curDepth = t.getKey();
- Path curPath = t.getValue();
-
- if (curDepth < depth) {
- int newDepth = curDepth + 1;
-
- // Create directories.
- for (int i = 0; i < entryCnt; i++) {
- Path subDir = new Path(curPath, "dir-" + newDepth + "-" + i);
-
- fs.mkdirs(subDir);
-
- queue.addLast(F.t(newDepth, subDir));
- }
- }
- else {
- // Create files.
- for (int i = 0; i < entryCnt; i++) {
- Path file = new Path(curPath, "file " + i);
-
- fs.create(file).close();
- }
- }
- }
-
- final AtomicBoolean err = new AtomicBoolean();
-
- multithreaded(new Runnable() {
- @Override public void run() {
- try {
- U.awaitQuiet(barrier);
-
- fs.delete(dir, true);
- }
- catch (IOException ignore) {
- err.set(true);
- }
- }
- }, THREAD_CNT);
-
- // Ensure there were no errors.
- assert !err.get();
-
- // Ensure the directory was actually deleted.
-
- assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- try {
- return !fs.exists(dir);
- }
- catch (IOException e) {
- throw new AssertionError(e);
- }
- }
- }, 5000L);
- }
-
- /** @throws Exception If failed. */
- public void testConsistency() throws Exception {
- // Default buffers values
- checkConsistency(-1, 1, -1, -1, 1, -1);
- checkConsistency(-1, 10, -1, -1, 10, -1);
- checkConsistency(-1, 100, -1, -1, 100, -1);
- checkConsistency(-1, 1000, -1, -1, 1000, -1);
- checkConsistency(-1, 10000, -1, -1, 10000, -1);
- checkConsistency(-1, 100000, -1, -1, 100000, -1);
-
- checkConsistency(65 * 1024 + 13, 100000, -1, -1, 100000, -1);
-
- checkConsistency(-1, 100000, 2 * 4 * 1024 + 17, -1, 100000, -1);
-
- checkConsistency(-1, 100000, -1, 65 * 1024 + 13, 100000, -1);
-
- checkConsistency(-1, 100000, -1, -1, 100000, 2 * 4 * 1024 + 17);
-
- checkConsistency(65 * 1024 + 13, 100000, 2 * 4 * 1024 + 13, 65 * 1024 + 149, 100000, 2 * 4 * 1024 + 157);
- }
-
- /**
- * Verifies that client reconnects after connection to the server has been lost.
- *
- * @throws Exception If error occurs.
- */
- public void testClientReconnect() throws Exception {
- Path filePath = new Path(PRIMARY_URI, "file1");
-
- final FSDataOutputStream s = fs.create(filePath); // Open the stream before stopping IGFS.
-
- try {
- G.stopAll(true); // Stop the server.
-
- startNodes(); // Start server again.
-
- // Check that client is again operational.
- assertTrue(fs.mkdirs(new Path(PRIMARY_URI, "dir1/dir2")));
-
- // However, the streams, opened before disconnect, should not be valid.
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Nullable @Override public Object call() throws Exception {
- s.write("test".getBytes());
-
- s.flush(); // Flush data to the broken output stream.
-
- return null;
- }
- }, IOException.class, null);
-
- assertFalse(fs.exists(filePath));
- }
- finally {
- U.closeQuiet(s); // Safety.
- }
- }
-
- /**
- * Verifies that client reconnects after connection to the server has been lost (multithreaded mode).
- *
- * @throws Exception If error occurs.
- */
- public void testClientReconnectMultithreaded() throws Exception {
- final ConcurrentLinkedQueue<FileSystem> q = new ConcurrentLinkedQueue<>();
-
- Configuration cfg = new Configuration();
-
- for (Map.Entry<String, String> entry : primaryFsCfg)
- cfg.set(entry.getKey(), entry.getValue());
-
- cfg.setBoolean("fs.igfs.impl.disable.cache", true);
-
- final int nClients = 1;
-
- // Initialize clients.
- for (int i = 0; i < nClients; i++)
- q.add(FileSystem.get(primaryFsUri, cfg));
-
- G.stopAll(true); // Stop the server.
-
- startNodes(); // Start server again.
-
- GridTestUtils.runMultiThreaded(new Callable<Object>() {
- @Override public Object call() throws Exception {
- FileSystem fs = q.poll();
-
- try {
- // Check that client is again operational.
- assertTrue(fs.mkdirs(new Path("/" + Thread.currentThread().getName())));
-
- return true;
- }
- finally {
- U.closeQuiet(fs);
- }
- }
- }, nClients, "test-client");
- }
-
- /**
- * Checks consistency of create --> open --> append --> open operations with different buffer sizes.
- *
- * @param createBufSize Buffer size used for file creation.
- * @param writeCntsInCreate Count of times to write in file creation.
- * @param openAfterCreateBufSize Buffer size used for file opening after creation.
- * @param appendBufSize Buffer size used for file appending.
- * @param writeCntsInAppend Count of times to write in file appending.
- * @param openAfterAppendBufSize Buffer size used for file opening after appending.
- * @throws Exception If failed.
- */
- private void checkConsistency(int createBufSize, int writeCntsInCreate, int openAfterCreateBufSize,
- int appendBufSize, int writeCntsInAppend, int openAfterAppendBufSize) throws Exception {
- final Path igfsHome = new Path(PRIMARY_URI);
-
- Path file = new Path(igfsHome, "/someDir/someInnerDir/someFile");
-
- FSDataOutputStream os = fs.create(file, true, createBufSize);
-
- for (int i = 0; i < writeCntsInCreate; i++)
- os.writeInt(i);
-
- os.close();
-
- FSDataInputStream is = fs.open(file, openAfterCreateBufSize);
-
- for (int i = 0; i < writeCntsInCreate; i++)
- assertEquals(i, is.readInt());
-
- is.close();
-
- os = fs.append(file, appendBufSize);
-
- for (int i = writeCntsInCreate; i < writeCntsInCreate + writeCntsInAppend; i++)
- os.writeInt(i);
-
- os.close();
-
- is = fs.open(file, openAfterAppendBufSize);
-
- for (int i = 0; i < writeCntsInCreate + writeCntsInAppend; i++)
- assertEquals(i, is.readInt());
-
- is.close();
- }
-
- /**
- * Gets instance of Hadoop local file system.
- *
- * @param home File system home.
- * @return File system.
- * @throws IOException If failed.
- */
- private FileSystem local(Path home) throws IOException {
- Configuration cfg = new Configuration();
-
- cfg.addResource(U.resolveIgniteUrl(HADOOP_FS_CFG));
-
- return FileSystem.get(home.toUri(), cfg);
- }
-
- /**
- * Copy files from one FS to another.
- *
- * @param msg Info message to display after copying finishes.
- * @param srcFs Source file system.
- * @param src Source path to copy from.
- * @param destFs Destination file system.
- * @param dest Destination path to copy to.
- * @throws IOException If failed.
- */
- private void copy(String msg, FileSystem srcFs, Path src, FileSystem destFs, Path dest) throws IOException {
- assert destFs.delete(dest, true) || !destFs.exists(dest) : "Failed to remove: " + dest;
-
- destFs.mkdirs(dest);
-
- Configuration conf = new Configuration(true);
-
- long time = System.currentTimeMillis();
-
- FileUtil.copy(srcFs, src, destFs, dest, false, true, conf);
-
- time = System.currentTimeMillis() - time;
-
- info("Copying finished, " + msg + " [time=" + time + "ms, src=" + src + ", dest=" + dest + ']');
- }
-
- /**
- * Compare content of two folders.
- *
- * @param cfg Paths configuration to compare.
- * @throws IOException If failed.
- */
- @SuppressWarnings("deprecation")
- private void compareContent(Config cfg) throws IOException {
- Deque<Config> queue = new LinkedList<>();
-
- queue.add(cfg);
-
- for (Config c = queue.poll(); c != null; c = queue.poll()) {
- boolean exists;
-
- assertEquals("Check existence [src=" + c.src + ", dest=" + c.dest + ']',
- exists = c.srcFs.exists(c.src), c.destFs.exists(c.dest));
-
- assertEquals("Check types (files?) [src=" + c.src + ", dest=" + c.dest + ']',
- c.srcFs.isFile(c.src), c.destFs.isFile(c.dest));
-
- if (exists) {
- ContentSummary srcSummary = c.srcFs.getContentSummary(c.src);
- ContentSummary dstSummary = c.destFs.getContentSummary(c.dest);
-
- assertEquals("Directories number comparison failed",
- srcSummary.getDirectoryCount(), dstSummary.getDirectoryCount());
-
- assertEquals("Files number comparison failed",
- srcSummary.getFileCount(), dstSummary.getFileCount());
-
- assertEquals("Space consumed comparison failed",
- srcSummary.getSpaceConsumed(), dstSummary.getSpaceConsumed());
-
- assertEquals("Length comparison failed",
- srcSummary.getLength(), dstSummary.getLength());
-
- // Intentionally skipping quotas checks as they can vary.
- }
- else {
- assertContentSummaryFails(c.srcFs, c.src);
- assertContentSummaryFails(c.destFs, c.dest);
- }
-
- if (!exists)
- continue;
-
- FileStatus[] srcSt = c.srcFs.listStatus(c.src);
- FileStatus[] destSt = c.destFs.listStatus(c.dest);
-
- assert srcSt != null && destSt != null : "Both not null" +
- " [srcSt=" + Arrays.toString(srcSt) + ", destSt=" + Arrays.toString(destSt) + ']';
-
- assertEquals("Check listing [src=" + c.src + ", dest=" + c.dest + ']', srcSt.length, destSt.length);
-
- // Listing of the file returns the only element with this file.
- if (srcSt.length == 1 && c.src.equals(srcSt[0].getPath())) {
- assertEquals(c.dest, destSt[0].getPath());
-
- assertTrue("Expects file [src=" + c.src + ", srcSt[0]=" + srcSt[0] + ']', !srcSt[0].isDir());
- assertTrue("Expects file [dest=" + c.dest + ", destSt[0]=" + destSt[0] + ']', !destSt[0].isDir());
-
- FSDataInputStream srcIn = null;
- FSDataInputStream destIn = null;
-
- try {
- srcIn = c.srcFs.open(c.src);
- destIn = c.destFs.open(c.dest);
-
- GridTestIoUtils.assertEqualStreams(srcIn, destIn, srcSt[0].getLen());
- }
- finally {
- U.closeQuiet(srcIn);
- U.closeQuiet(destIn);
- }
-
- continue; // Skip the following directories validations.
- }
-
- // Sort both arrays.
- Arrays.sort(srcSt, STATUS_COMPARATOR);
- Arrays.sort(destSt, STATUS_COMPARATOR);
-
- for (int i = 0; i < srcSt.length; i++)
- // Dig in deep to the last leaf, instead of collecting full tree in memory.
- queue.addFirst(new Config(c.srcFs, srcSt[i].getPath(), c.destFs, destSt[i].getPath()));
-
- // Add non-existent file to check in the current folder.
- String rndFile = "Non-existent file #" + UUID.randomUUID().toString();
-
- queue.addFirst(new Config(c.srcFs, new Path(c.src, rndFile), c.destFs, new Path(c.dest, rndFile)));
- }
- }
-
- /**
- * Test expected failures for 'close' operation.
- *
- * @param fs File system to test.
- * @param msg Expected exception message.
- */
- public void assertCloseFails(final FileSystem fs, String msg) {
- GridTestUtils.assertThrows(log, new Callable() {
- @Override public Object call() throws Exception {
- fs.close();
-
- return null;
- }
- }, IOException.class, msg);
- }
-
- /**
- * Test expected failures for 'get content summary' operation.
- *
- * @param fs File system to test.
- * @param path Path to evaluate content summary for.
- */
- private void assertContentSummaryFails(final FileSystem fs, final Path path) {
- GridTestUtils.assertThrows(log, new Callable<ContentSummary>() {
- @Override public ContentSummary call() throws Exception {
- return fs.getContentSummary(path);
- }
- }, FileNotFoundException.class, null);
- }
-
- /**
- * Assert that a given path exists in a given FileSystem.
- *
- * @param fs FileSystem to check.
- * @param p Path to check.
- * @throws IOException if the path does not exist.
- */
- private void assertPathExists(FileSystem fs, Path p) throws IOException {
- FileStatus fileStatus = fs.getFileStatus(p);
-
- assertEquals(p, fileStatus.getPath());
- assertNotSame(0, fileStatus.getModificationTime());
- }
-
- /**
- * Check path does not exist in a given FileSystem.
- *
- * @param fs FileSystem to check.
- * @param path Path to check.
- */
- private void assertPathDoesNotExist(final FileSystem fs, final Path path) {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.getFileStatus(path);
- }
- }, FileNotFoundException.class, null);
- }
-
- /** Helper class to encapsulate source and destination folders. */
- @SuppressWarnings({"PublicInnerClass", "PublicField"})
- public static final class Config {
- /** Source file system. */
- public final FileSystem srcFs;
-
- /** Source path to work with. */
- public final Path src;
-
- /** Destination file system. */
- public final FileSystem destFs;
-
- /** Destination path to work with. */
- public final Path dest;
-
- /**
- * Copying task configuration.
- *
- * @param srcFs Source file system.
- * @param src Source path.
- * @param destFs Destination file system.
- * @param dest Destination path.
- */
- public Config(FileSystem srcFs, Path src, FileSystem destFs, Path dest) {
- this.srcFs = srcFs;
- this.src = src;
- this.destFs = destFs;
- this.dest = dest;
- }
- }
-
- /**
- * Convert path for exception message testing purposes.
- *
- * @param path Path.
- * @return Converted path.
- * @throws Exception If failed.
- */
- private Path convertPath(Path path) throws Exception {
- if (mode != PROXY)
- return path;
- else {
- URI secondaryUri = new URI(SECONDARY_URI);
-
- URI pathUri = path.toUri();
-
- return new Path(new URI(pathUri.getScheme() != null ? secondaryUri.getScheme() : null,
- pathUri.getAuthority() != null ? secondaryUri.getAuthority() : null, pathUri.getPath(), null, null));
- }
- }
-
- /**
- * Create configuration for test.
- *
- * @param authority Authority.
- * @param skipEmbed Whether to skip embedded mode.
- * @param skipLocShmem Whether to skip local shmem mode.
- * @return Configuration.
- */
- private static Configuration configuration(String authority, boolean skipEmbed, boolean skipLocShmem) {
- Configuration cfg = new Configuration();
-
- cfg.set("fs.defaultFS", "igfs://" + authority + "/");
- cfg.set("fs.igfs.impl", IgfsHadoopFileSystem.class.getName());
- cfg.set("fs.AbstractFileSystem.igfs.impl",
- org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem.class.getName());
-
- cfg.setBoolean("fs.igfs.impl.disable.cache", true);
-
- if (skipEmbed)
- cfg.setBoolean(String.format(IgfsHadoopUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);
-
- if (skipLocShmem)
- cfg.setBoolean(String.format(IgfsHadoopUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);
-
- return cfg;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemClientSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemClientSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemClientSelfTest.java
deleted file mode 100644
index 599fd1d..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemClientSelfTest.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.commons.logging.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.igfs.common.*;
-import org.apache.ignite.internal.igfs.hadoop.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.spi.discovery.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
-import org.apache.ignite.testframework.*;
-
-import java.io.*;
-import java.lang.reflect.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.cache.CacheDistributionMode.*;
-import static org.apache.ignite.cache.CacheMode.*;
-import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*;
-import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
-
-/**
- * Test interaction between a IGFS client and a IGFS server.
- */
-public class IgfsHadoopFileSystemClientSelfTest extends IgfsCommonAbstractTest {
- /** Logger. */
- private static final Log LOG = LogFactory.getLog(IgfsHadoopFileSystemClientSelfTest.class);
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- startGrids(1);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- G.stopAll(true);
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
-
- cfg.setDiscoverySpi(discoSpi);
-
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("partitioned");
- igfsCfg.setMetaCacheName("replicated");
- igfsCfg.setName("igfs");
- igfsCfg.setBlockSize(512 * 1024);
- igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
- put("type", "tcp");
- put("port", String.valueOf(DFLT_IPC_PORT));
- }});
-
- cfg.setCacheConfiguration(cacheConfiguration());
- cfg.setIgfsConfiguration(igfsCfg);
-
- return cfg;
- }
-
- /**
- * Gets cache configuration.
- *
- * @return Cache configuration.
- */
- protected CacheConfiguration[] cacheConfiguration() {
- CacheConfiguration cacheCfg = defaultCacheConfiguration();
-
- cacheCfg.setName("partitioned");
- cacheCfg.setCacheMode(PARTITIONED);
- cacheCfg.setDistributionMode(PARTITIONED_ONLY);
- cacheCfg.setWriteSynchronizationMode(FULL_SYNC);
- cacheCfg.setEvictionPolicy(null);
- cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
- cacheCfg.setBackups(0);
- cacheCfg.setQueryIndexEnabled(false);
- cacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("replicated");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setWriteSynchronizationMode(FULL_SYNC);
- metaCacheCfg.setEvictionPolicy(null);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- return new CacheConfiguration[] {metaCacheCfg, cacheCfg};
- }
-
- /**
- * Test output stream deferred exception (GG-4440).
- *
- * @throws Exception If failed.
- */
- @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
- public void testOutputStreamDeferredException() throws Exception {
- final byte[] data = "test".getBytes();
-
- try {
- switchHandlerErrorFlag(true);
-
- IgfsHadoop client = new IgfsHadoopOutProc("127.0.0.1", 10500, getTestGridName(0), "igfs", LOG);
-
- client.handshake(null);
-
- IgfsPath path = new IgfsPath("/test1.file");
-
- IgfsHadoopStreamDelegate delegate = client.create(path, true, false, 1, 1024, null);
-
- final IgfsHadoopOutputStream igfsOut = new IgfsHadoopOutputStream(delegate, LOG,
- IgfsLogger.disabledLogger(), 0);
-
- // This call should return fine as exception is thrown for the first time.
- igfsOut.write(data);
-
- U.sleep(500);
-
- // This call should throw an IO exception.
- GridTestUtils.assertThrows(null, new Callable<Object>() {
- @Override public Object call() throws Exception {
- igfsOut.write(data);
-
- return null;
- }
- }, IOException.class, "Failed to write data to server (test).");
- }
- finally {
- switchHandlerErrorFlag(false);
- }
- }
-
- /**
- * Set IGFS REST handler error flag to the given state.
- *
- * @param flag Flag state.
- * @throws Exception If failed.
- */
- @SuppressWarnings("ConstantConditions")
- private void switchHandlerErrorFlag(boolean flag) throws Exception {
- IgfsProcessorAdapter igfsProc = ((IgniteKernal)grid(0)).context().igfs();
-
- Map<String, IgfsContext> igfsMap = getField(igfsProc, "igfsCache");
-
- IgfsServerManager srvMgr = F.first(igfsMap.values()).server();
-
- Collection<IgfsServer> srvrs = getField(srvMgr, "srvrs");
-
- IgfsServerHandler igfsHnd = getField(F.first(srvrs), "hnd");
-
- Field field = igfsHnd.getClass().getDeclaredField("errWrite");
-
- field.setAccessible(true);
-
- field.set(null, flag);
- }
-
- /**
- * Get value of the field with the given name of the given object.
- *
- * @param obj Object.
- * @param fieldName Field name.
- * @return Value of the field.
- * @throws Exception If failed.
- */
- @SuppressWarnings("unchecked")
- private <T> T getField(Object obj, String fieldName) throws Exception {
- Field field = obj.getClass().getDeclaredField(fieldName);
-
- field.setAccessible(true);
-
- return (T)field.get(obj);
- }
-}
[19/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopIpcToNioAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopIpcToNioAdapter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopIpcToNioAdapter.java
deleted file mode 100644
index a39451d..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopIpcToNioAdapter.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.util.ipc.*;
-import org.apache.ignite.internal.util.nio.*;
-
-import java.io.*;
-import java.nio.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-
-/**
- * Allows to re-use existing {@link GridNioFilter}s on IPC (specifically shared memory IPC)
- * communications.
- *
- * Note that this class consumes an entire thread inside {@link #serve()} method
- * in order to serve one {@link org.apache.ignite.internal.util.ipc.IpcEndpoint}.
- */
-public class GridHadoopIpcToNioAdapter<T> {
- /** */
- private final IpcEndpoint endp;
-
- /** */
- private final GridNioFilterChain<T> chain;
-
- /** */
- private final GridNioSessionImpl ses;
-
- /** */
- private final AtomicReference<CountDownLatch> latchRef = new AtomicReference<>();
-
- /** */
- private final ByteBuffer writeBuf;
-
- /**
- * @param log Log.
- * @param endp Endpoint.
- * @param lsnr Listener.
- * @param filters Filters.
- */
- public GridHadoopIpcToNioAdapter(IgniteLogger log, IpcEndpoint endp, boolean accepted,
- GridNioServerListener<T> lsnr, GridNioFilter... filters) {
- this.endp = endp;
-
- chain = new GridNioFilterChain<>(log, lsnr, new HeadFilter(), filters);
- ses = new GridNioSessionImpl(chain, null, null, accepted);
-
- writeBuf = ByteBuffer.allocate(8 << 10);
-
- writeBuf.order(ByteOrder.nativeOrder());
- }
-
- /**
- * Serves given set of listeners repeatedly reading data from the endpoint.
- *
- * @throws InterruptedException If interrupted.
- */
- public void serve() throws InterruptedException {
- try {
- chain.onSessionOpened(ses);
-
- InputStream in = endp.inputStream();
-
- ByteBuffer readBuf = ByteBuffer.allocate(8 << 10);
-
- readBuf.order(ByteOrder.nativeOrder());
-
- assert readBuf.hasArray();
-
- while (!Thread.interrupted()) {
- int pos = readBuf.position();
-
- int read = in.read(readBuf.array(), pos, readBuf.remaining());
-
- if (read > 0) {
- readBuf.position(0);
- readBuf.limit(pos + read);
-
- chain.onMessageReceived(ses, readBuf);
-
- if (readBuf.hasRemaining())
- readBuf.compact();
- else
- readBuf.clear();
-
- CountDownLatch latch = latchRef.get();
-
- if (latch != null)
- latch.await();
- }
- else if (read < 0) {
- endp.close();
-
- break; // And close below.
- }
- }
-
- // Assuming remote end closed connection - pushing event from head to tail.
- chain.onSessionClosed(ses);
- }
- catch (Exception e) {
- chain.onExceptionCaught(ses, new IgniteCheckedException("Failed to read from IPC endpoint.", e));
- }
- }
-
- /**
- * Gets dummy session for this adapter.
- *
- * @return Session.
- */
- public GridNioSession session() {
- return ses;
- }
-
- /**
- * Handles write events on chain.
- *
- * @param msg Buffer to send.
- * @return Send result.
- */
- private GridNioFuture<?> send(ByteBuffer msg) {
- assert writeBuf.hasArray();
-
- try {
- while (msg.hasRemaining()) {
- writeBuf.clear();
-
- writeBuf.put(msg);
-
- endp.outputStream().write(writeBuf.array(), 0, writeBuf.position());
- }
- }
- catch (IOException | IgniteCheckedException e) {
- return new GridNioFinishedFuture<Object>(e);
- }
-
- return new GridNioFinishedFuture<>((Object)null);
- }
-
- /**
- * Filter forwarding messages from chain's head to this server.
- */
- private class HeadFilter extends GridNioFilterAdapter {
- /**
- * Assigns filter name.
- */
- protected HeadFilter() {
- super("HeadFilter");
- }
-
- /** {@inheritDoc} */
- @Override public void onSessionOpened(GridNioSession ses) throws IgniteCheckedException {
- proceedSessionOpened(ses);
- }
-
- /** {@inheritDoc} */
- @Override public void onSessionClosed(GridNioSession ses) throws IgniteCheckedException {
- proceedSessionClosed(ses);
- }
-
- /** {@inheritDoc} */
- @Override public void onExceptionCaught(GridNioSession ses, IgniteCheckedException ex) throws IgniteCheckedException {
- proceedExceptionCaught(ses, ex);
- }
-
- /** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) {
- assert ses == GridHadoopIpcToNioAdapter.this.ses : "ses=" + ses +
- ", this.ses=" + GridHadoopIpcToNioAdapter.this.ses;
-
- return send((ByteBuffer)msg);
- }
-
- /** {@inheritDoc} */
- @Override public void onMessageReceived(GridNioSession ses, Object msg) throws IgniteCheckedException {
- proceedMessageReceived(ses, msg);
- }
-
- /** {@inheritDoc} */
- @Override public GridNioFuture<?> onPauseReads(GridNioSession ses) throws IgniteCheckedException {
- // This call should be synced externally to avoid races.
- boolean b = latchRef.compareAndSet(null, new CountDownLatch(1));
-
- assert b;
-
- return new GridNioFinishedFuture<>(b);
- }
-
- /** {@inheritDoc} */
- @Override public GridNioFuture<?> onResumeReads(GridNioSession ses) throws IgniteCheckedException {
- // This call should be synced externally to avoid races.
- CountDownLatch latch = latchRef.getAndSet(null);
-
- if (latch != null)
- latch.countDown();
-
- return new GridNioFinishedFuture<Object>(latch != null);
- }
-
- /** {@inheritDoc} */
- @Override public GridNioFuture<Boolean> onSessionClose(GridNioSession ses) {
- assert ses == GridHadoopIpcToNioAdapter.this.ses;
-
- boolean closed = GridHadoopIpcToNioAdapter.this.ses.setClosed();
-
- if (closed)
- endp.close();
-
- return new GridNioFinishedFuture<>(closed);
- }
-
- /** {@inheritDoc} */
- @Override public void onSessionIdleTimeout(GridNioSession ses) throws IgniteCheckedException {
- proceedSessionIdleTimeout(ses);
- }
-
- /** {@inheritDoc} */
- @Override public void onSessionWriteTimeout(GridNioSession ses) throws IgniteCheckedException {
- proceedSessionWriteTimeout(ses);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopMarshallerFilter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopMarshallerFilter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopMarshallerFilter.java
deleted file mode 100644
index 2a25357..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopMarshallerFilter.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.util.nio.*;
-import org.apache.ignite.marshaller.*;
-
-/**
- * Serialization filter.
- */
-public class GridHadoopMarshallerFilter extends GridNioFilterAdapter {
- /** Marshaller. */
- private Marshaller marshaller;
-
- /**
- * @param marshaller Marshaller to use.
- */
- public GridHadoopMarshallerFilter(Marshaller marshaller) {
- super("GridHadoopMarshallerFilter");
-
- this.marshaller = marshaller;
- }
-
- /** {@inheritDoc} */
- @Override public void onSessionOpened(GridNioSession ses) throws IgniteCheckedException {
- proceedSessionOpened(ses);
- }
-
- /** {@inheritDoc} */
- @Override public void onSessionClosed(GridNioSession ses) throws IgniteCheckedException {
- proceedSessionClosed(ses);
- }
-
- /** {@inheritDoc} */
- @Override public void onExceptionCaught(GridNioSession ses, IgniteCheckedException ex) throws IgniteCheckedException {
- proceedExceptionCaught(ses, ex);
- }
-
- /** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
- assert msg instanceof GridHadoopMessage : "Invalid message type: " + msg;
-
- return proceedSessionWrite(ses, marshaller.marshal(msg));
- }
-
- @Override public void onMessageReceived(GridNioSession ses, Object msg) throws IgniteCheckedException {
- assert msg instanceof byte[];
-
- // Always unmarshal with system classloader.
- proceedMessageReceived(ses, marshaller.unmarshal((byte[])msg, null));
- }
-
- /** {@inheritDoc} */
- @Override public GridNioFuture<Boolean> onSessionClose(GridNioSession ses) throws IgniteCheckedException {
- return proceedSessionClose(ses);
- }
-
- /** {@inheritDoc} */
- @Override public void onSessionIdleTimeout(GridNioSession ses) throws IgniteCheckedException {
- proceedSessionIdleTimeout(ses);
- }
-
- /** {@inheritDoc} */
- @Override public void onSessionWriteTimeout(GridNioSession ses) throws IgniteCheckedException {
- proceedSessionWriteTimeout(ses);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopMessageListener.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopMessageListener.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopMessageListener.java
deleted file mode 100644
index 219f4db..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopMessageListener.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
-
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
-
-/**
- * Hadoop communication message listener.
- */
-public interface GridHadoopMessageListener {
- /**
- * @param desc Process descriptor.
- * @param msg Hadoop message.
- */
- public void onMessageReceived(GridHadoopProcessDescriptor desc, GridHadoopMessage msg);
-
- /**
- * Called when connection to remote process was lost.
- *
- * @param desc Process descriptor.
- */
- public void onConnectionLost(GridHadoopProcessDescriptor desc);
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopTcpNioCommunicationClient.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopTcpNioCommunicationClient.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopTcpNioCommunicationClient.java
deleted file mode 100644
index 2be16cb..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopTcpNioCommunicationClient.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
-import org.apache.ignite.internal.util.nio.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-
-/**
- * Grid client for NIO server.
- */
-public class GridHadoopTcpNioCommunicationClient extends GridHadoopAbstractCommunicationClient {
- /** Socket. */
- private final GridNioSession ses;
-
- /**
- * Constructor for test purposes only.
- */
- public GridHadoopTcpNioCommunicationClient() {
- ses = null;
- }
-
- /**
- * @param ses Session.
- */
- public GridHadoopTcpNioCommunicationClient(GridNioSession ses) {
- assert ses != null;
-
- this.ses = ses;
- }
-
- /** {@inheritDoc} */
- @Override public boolean close() {
- boolean res = super.close();
-
- if (res)
- ses.close();
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public void forceClose() {
- super.forceClose();
-
- ses.close();
- }
-
- /** {@inheritDoc} */
- @Override public void sendMessage(GridHadoopProcessDescriptor desc, GridHadoopMessage msg)
- throws IgniteCheckedException {
- if (closed())
- throw new IgniteCheckedException("Client was closed: " + this);
-
- GridNioFuture<?> fut = ses.send(msg);
-
- if (fut.isDone()) {
- try {
- fut.get();
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to send message [client=" + this + ']', e);
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public long getIdleTime() {
- long now = U.currentTimeMillis();
-
- // Session can be used for receiving and sending.
- return Math.min(Math.min(now - ses.lastReceiveTime(), now - ses.lastSendScheduleTime()),
- now - ses.lastSendTime());
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopTcpNioCommunicationClient.class, this, super.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopAbstractCommunicationClient.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopAbstractCommunicationClient.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopAbstractCommunicationClient.java
new file mode 100644
index 0000000..3a5d84a
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopAbstractCommunicationClient.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
+
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.util.concurrent.atomic.*;
+
+/**
+ * Implements basic lifecycle for communication clients.
+ */
+public abstract class HadoopAbstractCommunicationClient implements HadoopCommunicationClient {
+ /** Time when this client was last used. */
+ private volatile long lastUsed = U.currentTimeMillis();
+
+ /** Reservations. */
+ private final AtomicInteger reserves = new AtomicInteger();
+
+ /** {@inheritDoc} */
+ @Override public boolean close() {
+ return reserves.compareAndSet(0, -1);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void forceClose() {
+ reserves.set(-1);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean closed() {
+ return reserves.get() == -1;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean reserve() {
+ while (true) {
+ int r = reserves.get();
+
+ if (r == -1)
+ return false;
+
+ if (reserves.compareAndSet(r, r + 1))
+ return true;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void release() {
+ while (true) {
+ int r = reserves.get();
+
+ if (r == -1)
+ return;
+
+ if (reserves.compareAndSet(r, r - 1))
+ return;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean reserved() {
+ return reserves.get() > 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getIdleTime() {
+ return U.currentTimeMillis() - lastUsed;
+ }
+
+ /**
+ * Updates used time.
+ */
+ protected void markUsed() {
+ lastUsed = U.currentTimeMillis();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopAbstractCommunicationClient.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopCommunicationClient.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopCommunicationClient.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopCommunicationClient.java
new file mode 100644
index 0000000..ce42e9a
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopCommunicationClient.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
+
+/**
+ *
+ */
+public interface HadoopCommunicationClient {
+ /**
+ * @return {@code True} if client has been closed by this call,
+ * {@code false} if failed to close client (due to concurrent reservation or concurrent close).
+ */
+ public boolean close();
+
+ /**
+ * Forces client close.
+ */
+ public void forceClose();
+
+ /**
+ * @return {@code True} if client is closed;
+ */
+ public boolean closed();
+
+ /**
+ * @return {@code True} if client was reserved, {@code false} otherwise.
+ */
+ public boolean reserve();
+
+ /**
+ * Releases this client by decreasing reservations.
+ */
+ public void release();
+
+ /**
+ * @return {@code True} if client was reserved.
+ */
+ public boolean reserved();
+
+ /**
+ * Gets idle time of this client.
+ *
+ * @return Idle time of this client.
+ */
+ public long getIdleTime();
+
+ /**
+ * @param desc Process descriptor.
+ * @param msg Message to send.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void sendMessage(HadoopProcessDescriptor desc, HadoopMessage msg) throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
new file mode 100644
index 0000000..4df8188
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
@@ -0,0 +1,1431 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.ipc.*;
+import org.apache.ignite.internal.util.ipc.shmem.*;
+import org.apache.ignite.internal.util.nio.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.internal.util.worker.*;
+import org.apache.ignite.marshaller.*;
+import org.apache.ignite.thread.*;
+import org.jdk8.backport.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.net.*;
+import java.nio.*;
+import java.nio.channels.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+/**
+ * Hadoop external communication class.
+ */
+public class HadoopExternalCommunication {
+ /** IPC error message. */
+ public static final String OUT_OF_RESOURCES_TCP_MSG = "Failed to allocate shared memory segment " +
+ "(switching to TCP, may be slower)."; // TODO IGNITE-70 Add link to documentation
+
+ /** Default port which node sets listener to (value is <tt>47100</tt>). */
+ public static final int DFLT_PORT = 27100;
+
+ /** Default connection timeout (value is <tt>1000</tt>ms). */
+ public static final long DFLT_CONN_TIMEOUT = 1000;
+
+ /** Default Maximum connection timeout (value is <tt>600,000</tt>ms). */
+ public static final long DFLT_MAX_CONN_TIMEOUT = 10 * 60 * 1000;
+
+ /** Default reconnect attempts count (value is <tt>10</tt>). */
+ public static final int DFLT_RECONNECT_CNT = 10;
+
+ /** Default message queue limit per connection (for incoming and outgoing . */
+ public static final int DFLT_MSG_QUEUE_LIMIT = GridNioServer.DFLT_SEND_QUEUE_LIMIT;
+
+ /**
+ * Default count of selectors for TCP server equals to
+ * {@code "Math.min(4, Runtime.getRuntime().availableProcessors())"}.
+ */
+ public static final int DFLT_SELECTORS_CNT = 1;
+
+ /** Node ID meta for session. */
+ private static final int PROCESS_META = GridNioSessionMetaKey.nextUniqueKey();
+
+ /** Handshake timeout meta for session. */
+ private static final int HANDSHAKE_FINISH_META = GridNioSessionMetaKey.nextUniqueKey();
+
+ /** Message tracker meta for session. */
+ private static final int TRACKER_META = GridNioSessionMetaKey.nextUniqueKey();
+
+ /**
+ * Default local port range (value is <tt>100</tt>).
+ * See {@link #setLocalPortRange(int)} for details.
+ */
+ public static final int DFLT_PORT_RANGE = 100;
+
+ /** Default value for {@code TCP_NODELAY} socket option (value is <tt>true</tt>). */
+ public static final boolean DFLT_TCP_NODELAY = true;
+
+ /** Server listener. */
+ private final GridNioServerListener<HadoopMessage> srvLsnr =
+ new GridNioServerListenerAdapter<HadoopMessage>() {
+ @Override public void onConnected(GridNioSession ses) {
+ HadoopProcessDescriptor desc = ses.meta(PROCESS_META);
+
+ assert desc != null : "Received connected notification without finished handshake: " + ses;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onDisconnected(GridNioSession ses, @Nullable Exception e) {
+ if (log.isDebugEnabled())
+ log.debug("Closed connection for session: " + ses);
+
+ if (e != null)
+ U.error(log, "Session disconnected due to exception: " + ses, e);
+
+ HadoopProcessDescriptor desc = ses.meta(PROCESS_META);
+
+ if (desc != null) {
+ HadoopCommunicationClient rmv = clients.remove(desc.processId());
+
+ if (rmv != null)
+ rmv.forceClose();
+ }
+
+ HadoopMessageListener lsnr0 = lsnr;
+
+ if (lsnr0 != null)
+ // Notify listener about connection close.
+ lsnr0.onConnectionLost(desc);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onMessage(GridNioSession ses, HadoopMessage msg) {
+ notifyListener(ses.<HadoopProcessDescriptor>meta(PROCESS_META), msg);
+
+ if (msgQueueLimit > 0) {
+ GridNioMessageTracker tracker = ses.meta(TRACKER_META);
+
+ assert tracker != null : "Missing tracker for limited message queue: " + ses;
+
+ tracker.run();
+ }
+ }
+ };
+
+ /** Logger. */
+ private IgniteLogger log;
+
+ /** Local process descriptor. */
+ private HadoopProcessDescriptor locProcDesc;
+
+ /** Marshaller. */
+ private Marshaller marsh;
+
+ /** Message notification executor service. */
+ private ExecutorService execSvc;
+
+ /** Grid name. */
+ private String gridName;
+
+ /** Complex variable that represents this node IP address. */
+ private volatile InetAddress locHost;
+
+ /** Local port which node uses. */
+ private int locPort = DFLT_PORT;
+
+ /** Local port range. */
+ private int locPortRange = DFLT_PORT_RANGE;
+
+ /** Local port which node uses to accept shared memory connections. */
+ private int shmemPort = -1;
+
+ /** Allocate direct buffer or heap buffer. */
+ private boolean directBuf = true;
+
+ /** Connect timeout. */
+ private long connTimeout = DFLT_CONN_TIMEOUT;
+
+ /** Maximum connect timeout. */
+ private long maxConnTimeout = DFLT_MAX_CONN_TIMEOUT;
+
+ /** Reconnect attempts count. */
+ @SuppressWarnings({"FieldAccessedSynchronizedAndUnsynchronized"})
+ private int reconCnt = DFLT_RECONNECT_CNT;
+
+ /** Socket send buffer. */
+ private int sockSndBuf;
+
+ /** Socket receive buffer. */
+ private int sockRcvBuf;
+
+ /** Message queue limit. */
+ private int msgQueueLimit = DFLT_MSG_QUEUE_LIMIT;
+
+ /** NIO server. */
+ private GridNioServer<HadoopMessage> nioSrvr;
+
+ /** Shared memory server. */
+ private IpcSharedMemoryServerEndpoint shmemSrv;
+
+ /** {@code TCP_NODELAY} option value for created sockets. */
+ private boolean tcpNoDelay = DFLT_TCP_NODELAY;
+
+ /** Shared memory accept worker. */
+ private ShmemAcceptWorker shmemAcceptWorker;
+
+ /** Shared memory workers. */
+ private final Collection<ShmemWorker> shmemWorkers = new ConcurrentLinkedDeque8<>();
+
+ /** Clients. */
+ private final ConcurrentMap<UUID, HadoopCommunicationClient> clients = GridConcurrentFactory.newMap();
+
+ /** Message listener. */
+ private volatile HadoopMessageListener lsnr;
+
+ /** Bound port. */
+ private int boundTcpPort = -1;
+
+ /** Bound port for shared memory server. */
+ private int boundTcpShmemPort = -1;
+
+ /** Count of selectors to use in TCP server. */
+ private int selectorsCnt = DFLT_SELECTORS_CNT;
+
+ /** Local node ID message. */
+ private ProcessHandshakeMessage locIdMsg;
+
+ /** Locks. */
+ private final GridKeyLock locks = new GridKeyLock();
+
+ /**
+ * @param parentNodeId Parent node ID.
+ * @param procId Process ID.
+ * @param marsh Marshaller to use.
+ * @param log Logger.
+ * @param execSvc Executor service for message notification.
+ * @param gridName Grid name.
+ */
+ public HadoopExternalCommunication(
+ UUID parentNodeId,
+ UUID procId,
+ Marshaller marsh,
+ IgniteLogger log,
+ ExecutorService execSvc,
+ String gridName
+ ) {
+ locProcDesc = new HadoopProcessDescriptor(parentNodeId, procId);
+
+ this.marsh = marsh;
+ this.log = log.getLogger(HadoopExternalCommunication.class);
+ this.execSvc = execSvc;
+ this.gridName = gridName;
+ }
+
+ /**
+ * Sets local port for socket binding.
+ * <p>
+ * If not provided, default value is {@link #DFLT_PORT}.
+ *
+ * @param locPort Port number.
+ */
+ public void setLocalPort(int locPort) {
+ this.locPort = locPort;
+ }
+
+ /**
+ * Gets local port for socket binding.
+ *
+ * @return Local port.
+ */
+ public int getLocalPort() {
+ return locPort;
+ }
+
+ /**
+ * Sets local port range for local host ports (value must greater than or equal to <tt>0</tt>).
+ * If provided local port (see {@link #setLocalPort(int)}} is occupied,
+ * implementation will try to increment the port number for as long as it is less than
+ * initial value plus this range.
+ * <p>
+ * If port range value is <tt>0</tt>, then implementation will try bind only to the port provided by
+ * {@link #setLocalPort(int)} method and fail if binding to this port did not succeed.
+ * <p>
+ * Local port range is very useful during development when more than one grid nodes need to run
+ * on the same physical machine.
+ * <p>
+ * If not provided, default value is {@link #DFLT_PORT_RANGE}.
+ *
+ * @param locPortRange New local port range.
+ */
+ public void setLocalPortRange(int locPortRange) {
+ this.locPortRange = locPortRange;
+ }
+
+ /**
+ * @return Local port range.
+ */
+ public int getLocalPortRange() {
+ return locPortRange;
+ }
+
+ /**
+ * Sets local port to accept shared memory connections.
+ * <p>
+ * If set to {@code -1} shared memory communication will be disabled.
+ * <p>
+ * If not provided, shared memory is disabled.
+ *
+ * @param shmemPort Port number.
+ */
+ public void setSharedMemoryPort(int shmemPort) {
+ this.shmemPort = shmemPort;
+ }
+
+ /**
+ * Gets shared memory port to accept incoming connections.
+ *
+ * @return Shared memory port.
+ */
+ public int getSharedMemoryPort() {
+ return shmemPort;
+ }
+
+ /**
+ * Sets connect timeout used when establishing connection
+ * with remote nodes.
+ * <p>
+ * {@code 0} is interpreted as infinite timeout.
+ * <p>
+ * If not provided, default value is {@link #DFLT_CONN_TIMEOUT}.
+ *
+ * @param connTimeout Connect timeout.
+ */
+ public void setConnectTimeout(long connTimeout) {
+ this.connTimeout = connTimeout;
+ }
+
+ /**
+ * @return Connection timeout.
+ */
+ public long getConnectTimeout() {
+ return connTimeout;
+ }
+
+ /**
+ * Sets maximum connect timeout. If handshake is not established within connect timeout,
+ * then SPI tries to repeat handshake procedure with increased connect timeout.
+ * Connect timeout can grow till maximum timeout value,
+ * if maximum timeout value is reached then the handshake is considered as failed.
+ * <p>
+ * {@code 0} is interpreted as infinite timeout.
+ * <p>
+ * If not provided, default value is {@link #DFLT_MAX_CONN_TIMEOUT}.
+ *
+ * @param maxConnTimeout Maximum connect timeout.
+ */
+ public void setMaxConnectTimeout(long maxConnTimeout) {
+ this.maxConnTimeout = maxConnTimeout;
+ }
+
+ /**
+ * Gets maximum connection timeout.
+ *
+ * @return Maximum connection timeout.
+ */
+ public long getMaxConnectTimeout() {
+ return maxConnTimeout;
+ }
+
+ /**
+ * Sets maximum number of reconnect attempts used when establishing connection
+ * with remote nodes.
+ * <p>
+ * If not provided, default value is {@link #DFLT_RECONNECT_CNT}.
+ *
+ * @param reconCnt Maximum number of reconnection attempts.
+ */
+ public void setReconnectCount(int reconCnt) {
+ this.reconCnt = reconCnt;
+ }
+
+ /**
+ * @return Reconnect count.
+ */
+ public int getReconnectCount() {
+ return reconCnt;
+ }
+
+ /**
+ * Sets flag to allocate direct or heap buffer in SPI.
+ * If value is {@code true}, then SPI will use {@link ByteBuffer#allocateDirect(int)} call.
+ * Otherwise, SPI will use {@link ByteBuffer#allocate(int)} call.
+ * <p>
+ * If not provided, default value is {@code true}.
+ *
+ * @param directBuf Flag indicates to allocate direct or heap buffer in SPI.
+ */
+ public void setDirectBuffer(boolean directBuf) {
+ this.directBuf = directBuf;
+ }
+
+ /**
+ * @return Direct buffer flag.
+ */
+ public boolean isDirectBuffer() {
+ return directBuf;
+ }
+
+ /**
+ * Sets the count of selectors te be used in TCP server.
+ * <p/>
+ * If not provided, default value is {@link #DFLT_SELECTORS_CNT}.
+ *
+ * @param selectorsCnt Selectors count.
+ */
+ public void setSelectorsCount(int selectorsCnt) {
+ this.selectorsCnt = selectorsCnt;
+ }
+
+ /**
+ * @return Number of selectors to use.
+ */
+ public int getSelectorsCount() {
+ return selectorsCnt;
+ }
+
+ /**
+ * Sets value for {@code TCP_NODELAY} socket option. Each
+ * socket will be opened using provided value.
+ * <p>
+ * Setting this option to {@code true} disables Nagle's algorithm
+ * for socket decreasing latency and delivery time for small messages.
+ * <p>
+ * For systems that work under heavy network load it is advisable to
+ * set this value to {@code false}.
+ * <p>
+ * If not provided, default value is {@link #DFLT_TCP_NODELAY}.
+ *
+ * @param tcpNoDelay {@code True} to disable TCP delay.
+ */
+ public void setTcpNoDelay(boolean tcpNoDelay) {
+ this.tcpNoDelay = tcpNoDelay;
+ }
+
+ /**
+ * @return {@code TCP_NO_DELAY} flag.
+ */
+ public boolean isTcpNoDelay() {
+ return tcpNoDelay;
+ }
+
+ /**
+ * Sets receive buffer size for sockets created or accepted by this SPI.
+ * <p>
+ * If not provided, default is {@code 0} which leaves buffer unchanged after
+ * socket creation (OS defaults).
+ *
+ * @param sockRcvBuf Socket receive buffer size.
+ */
+ public void setSocketReceiveBuffer(int sockRcvBuf) {
+ this.sockRcvBuf = sockRcvBuf;
+ }
+
+ /**
+ * @return Socket receive buffer size.
+ */
+ public int getSocketReceiveBuffer() {
+ return sockRcvBuf;
+ }
+
+ /**
+ * Sets send buffer size for sockets created or accepted by this SPI.
+ * <p>
+ * If not provided, default is {@code 0} which leaves the buffer unchanged
+ * after socket creation (OS defaults).
+ *
+ * @param sockSndBuf Socket send buffer size.
+ */
+ public void setSocketSendBuffer(int sockSndBuf) {
+ this.sockSndBuf = sockSndBuf;
+ }
+
+ /**
+ * @return Socket send buffer size.
+ */
+ public int getSocketSendBuffer() {
+ return sockSndBuf;
+ }
+
+ /**
+ * Sets message queue limit for incoming and outgoing messages.
+ * <p>
+ * When set to positive number send queue is limited to the configured value.
+ * {@code 0} disables the size limitations.
+ * <p>
+ * If not provided, default is {@link #DFLT_MSG_QUEUE_LIMIT}.
+ *
+ * @param msgQueueLimit Send queue size limit.
+ */
+ public void setMessageQueueLimit(int msgQueueLimit) {
+ this.msgQueueLimit = msgQueueLimit;
+ }
+
+ /**
+ * @return Message queue size limit.
+ */
+ public int getMessageQueueLimit() {
+ return msgQueueLimit;
+ }
+
+ /**
+ * Sets Hadoop communication message listener.
+ *
+ * @param lsnr Message listener.
+ */
+ public void setListener(HadoopMessageListener lsnr) {
+ this.lsnr = lsnr;
+ }
+
+ /**
+ * @return Outbound message queue size.
+ */
+ public int getOutboundMessagesQueueSize() {
+ return nioSrvr.outboundMessagesQueueSize();
+ }
+
+ /**
+ * Starts communication.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public void start() throws IgniteCheckedException {
+ try {
+ locHost = U.getLocalHost();
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to initialize local address.", e);
+ }
+
+ try {
+ shmemSrv = resetShmemServer();
+ }
+ catch (IgniteCheckedException e) {
+ U.warn(log, "Failed to start shared memory communication server.", e);
+ }
+
+ try {
+ // This method potentially resets local port to the value
+ // local node was bound to.
+ nioSrvr = resetNioServer();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteCheckedException("Failed to initialize TCP server: " + locHost, e);
+ }
+
+ locProcDesc.address(locHost.getHostAddress());
+ locProcDesc.sharedMemoryPort(boundTcpShmemPort);
+ locProcDesc.tcpPort(boundTcpPort);
+
+ locIdMsg = new ProcessHandshakeMessage(locProcDesc);
+
+ if (shmemSrv != null) {
+ shmemAcceptWorker = new ShmemAcceptWorker(shmemSrv);
+
+ new IgniteThread(shmemAcceptWorker).start();
+ }
+
+ nioSrvr.start();
+ }
+
+ /**
+ * Gets local process descriptor.
+ *
+ * @return Local process descriptor.
+ */
+ public HadoopProcessDescriptor localProcessDescriptor() {
+ return locProcDesc;
+ }
+
+ /**
+ * Gets filters used by communication.
+ *
+ * @return Filters array.
+ */
+ private GridNioFilter[] filters() {
+ return new GridNioFilter[] {
+ new GridNioAsyncNotifyFilter(gridName, execSvc, log),
+ new HandshakeAndBackpressureFilter(),
+ new HadoopMarshallerFilter(marsh),
+ new GridNioCodecFilter(new GridBufferedParser(directBuf, ByteOrder.nativeOrder()), log, false)
+ };
+ }
+
+ /**
+ * Recreates tpcSrvr socket instance.
+ *
+ * @return Server instance.
+ * @throws IgniteCheckedException Thrown if it's not possible to create server.
+ */
+ private GridNioServer<HadoopMessage> resetNioServer() throws IgniteCheckedException {
+ if (boundTcpPort >= 0)
+ throw new IgniteCheckedException("Tcp NIO server was already created on port " + boundTcpPort);
+
+ IgniteCheckedException lastEx = null;
+
+ // If configured TCP port is busy, find first available in range.
+ for (int port = locPort; port < locPort + locPortRange; port++) {
+ try {
+ GridNioServer<HadoopMessage> srvr =
+ GridNioServer.<HadoopMessage>builder()
+ .address(locHost)
+ .port(port)
+ .listener(srvLsnr)
+ .logger(log.getLogger(GridNioServer.class))
+ .selectorCount(selectorsCnt)
+ .gridName(gridName)
+ .tcpNoDelay(tcpNoDelay)
+ .directBuffer(directBuf)
+ .byteOrder(ByteOrder.nativeOrder())
+ .socketSendBufferSize(sockSndBuf)
+ .socketReceiveBufferSize(sockRcvBuf)
+ .sendQueueLimit(msgQueueLimit)
+ .directMode(false)
+ .filters(filters())
+ .build();
+
+ boundTcpPort = port;
+
+ // Ack Port the TCP server was bound to.
+ if (log.isInfoEnabled())
+ log.info("Successfully bound to TCP port [port=" + boundTcpPort +
+ ", locHost=" + locHost + ']');
+
+ return srvr;
+ }
+ catch (IgniteCheckedException e) {
+ lastEx = e;
+
+ if (log.isDebugEnabled())
+ log.debug("Failed to bind to local port (will try next port within range) [port=" + port +
+ ", locHost=" + locHost + ']');
+ }
+ }
+
+ // If free port wasn't found.
+ throw new IgniteCheckedException("Failed to bind to any port within range [startPort=" + locPort +
+ ", portRange=" + locPortRange + ", locHost=" + locHost + ']', lastEx);
+ }
+
+ /**
+ * Creates new shared memory communication server.
+ * @return Server.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable private IpcSharedMemoryServerEndpoint resetShmemServer() throws IgniteCheckedException {
+ if (boundTcpShmemPort >= 0)
+ throw new IgniteCheckedException("Shared memory server was already created on port " + boundTcpShmemPort);
+
+ if (shmemPort == -1 || U.isWindows())
+ return null;
+
+ IgniteCheckedException lastEx = null;
+
+ // If configured TCP port is busy, find first available in range.
+ for (int port = shmemPort; port < shmemPort + locPortRange; port++) {
+ try {
+ IpcSharedMemoryServerEndpoint srv = new IpcSharedMemoryServerEndpoint(
+ log.getLogger(IpcSharedMemoryServerEndpoint.class),
+ locProcDesc.processId(), gridName);
+
+ srv.setPort(port);
+
+ srv.omitOutOfResourcesWarning(true);
+
+ srv.start();
+
+ boundTcpShmemPort = port;
+
+ // Ack Port the TCP server was bound to.
+ if (log.isInfoEnabled())
+ log.info("Successfully bound shared memory communication to TCP port [port=" + boundTcpShmemPort +
+ ", locHost=" + locHost + ']');
+
+ return srv;
+ }
+ catch (IgniteCheckedException e) {
+ lastEx = e;
+
+ if (log.isDebugEnabled())
+ log.debug("Failed to bind to local port (will try next port within range) [port=" + port +
+ ", locHost=" + locHost + ']');
+ }
+ }
+
+ // If free port wasn't found.
+ throw new IgniteCheckedException("Failed to bind shared memory communication to any port within range [startPort=" +
+ locPort + ", portRange=" + locPortRange + ", locHost=" + locHost + ']', lastEx);
+ }
+
+ /**
+ * Stops the server.
+ *
+ * @throws IgniteCheckedException
+ */
+ public void stop() throws IgniteCheckedException {
+ // Stop TCP server.
+ if (nioSrvr != null)
+ nioSrvr.stop();
+
+ U.cancel(shmemAcceptWorker);
+ U.join(shmemAcceptWorker, log);
+
+ U.cancel(shmemWorkers);
+ U.join(shmemWorkers, log);
+
+ shmemWorkers.clear();
+
+ // Force closing on stop (safety).
+ for (HadoopCommunicationClient client : clients.values())
+ client.forceClose();
+
+ // Clear resources.
+ nioSrvr = null;
+
+ boundTcpPort = -1;
+ }
+
+ /**
+ * Sends message to Hadoop process.
+ *
+ * @param desc
+ * @param msg
+ * @throws IgniteCheckedException
+ */
+ public void sendMessage(HadoopProcessDescriptor desc, HadoopMessage msg) throws
+ IgniteCheckedException {
+ assert desc != null;
+ assert msg != null;
+
+ if (log.isTraceEnabled())
+ log.trace("Sending message to Hadoop process [desc=" + desc + ", msg=" + msg + ']');
+
+ HadoopCommunicationClient client = null;
+
+ boolean closeOnRelease = true;
+
+ try {
+ client = reserveClient(desc);
+
+ client.sendMessage(desc, msg);
+
+ closeOnRelease = false;
+ }
+ finally {
+ if (client != null) {
+ if (closeOnRelease) {
+ client.forceClose();
+
+ clients.remove(desc.processId(), client);
+ }
+ else
+ client.release();
+ }
+ }
+ }
+
+ /**
+ * Returns existing or just created client to node.
+ *
+ * @param desc Node to which client should be open.
+ * @return The existing or just created client.
+ * @throws IgniteCheckedException Thrown if any exception occurs.
+ */
+ private HadoopCommunicationClient reserveClient(HadoopProcessDescriptor desc) throws IgniteCheckedException {
+ assert desc != null;
+
+ UUID procId = desc.processId();
+
+ while (true) {
+ HadoopCommunicationClient client = clients.get(procId);
+
+ if (client == null) {
+ if (log.isDebugEnabled())
+ log.debug("Did not find client for remote process [locProcDesc=" + locProcDesc + ", desc=" +
+ desc + ']');
+
+ // Do not allow concurrent connects.
+ Object sync = locks.lock(procId);
+
+ try {
+ client = clients.get(procId);
+
+ if (client == null) {
+ HadoopCommunicationClient old = clients.put(procId, client = createNioClient(desc));
+
+ assert old == null;
+ }
+ }
+ finally {
+ locks.unlock(procId, sync);
+ }
+
+ assert client != null;
+ }
+
+ if (client.reserve())
+ return client;
+ else
+ // Client has just been closed by idle worker. Help it and try again.
+ clients.remove(procId, client);
+ }
+ }
+
+ /**
+ * @param desc Process descriptor.
+ * @return Client.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable protected HadoopCommunicationClient createNioClient(HadoopProcessDescriptor desc)
+ throws IgniteCheckedException {
+ assert desc != null;
+
+ int shmemPort = desc.sharedMemoryPort();
+
+ // If remote node has shared memory server enabled and has the same set of MACs
+ // then we are likely to run on the same host and shared memory communication could be tried.
+ if (shmemPort != -1 && locProcDesc.parentNodeId().equals(desc.parentNodeId())) {
+ try {
+ return createShmemClient(desc, shmemPort);
+ }
+ catch (IgniteCheckedException e) {
+ if (e.hasCause(IpcOutOfSystemResourcesException.class))
+ // Has cause or is itself the IpcOutOfSystemResourcesException.
+ LT.warn(log, null, OUT_OF_RESOURCES_TCP_MSG);
+ else if (log.isDebugEnabled())
+ log.debug("Failed to establish shared memory connection with local hadoop process: " +
+ desc);
+ }
+ }
+
+ return createTcpClient(desc);
+ }
+
+ /**
+ * @param desc Process descriptor.
+ * @param port Port.
+ * @return Client.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable protected HadoopCommunicationClient createShmemClient(HadoopProcessDescriptor desc, int port)
+ throws IgniteCheckedException {
+ int attempt = 1;
+
+ int connectAttempts = 1;
+
+ long connTimeout0 = connTimeout;
+
+ while (true) {
+ IpcEndpoint clientEndpoint;
+
+ try {
+ clientEndpoint = new IpcSharedMemoryClientEndpoint(port, (int)connTimeout, log);
+ }
+ catch (IgniteCheckedException e) {
+ // Reconnect for the second time, if connection is not established.
+ if (connectAttempts < 2 && X.hasCause(e, ConnectException.class)) {
+ connectAttempts++;
+
+ continue;
+ }
+
+ throw e;
+ }
+
+ HadoopCommunicationClient client = null;
+
+ try {
+ ShmemWorker worker = new ShmemWorker(clientEndpoint, false);
+
+ shmemWorkers.add(worker);
+
+ GridNioSession ses = worker.session();
+
+ HandshakeFinish fin = new HandshakeFinish();
+
+ // We are in lock, it is safe to get session and attach
+ ses.addMeta(HANDSHAKE_FINISH_META, fin);
+
+ client = new HadoopTcpNioCommunicationClient(ses);
+
+ new IgniteThread(worker).start();
+
+ fin.await(connTimeout0);
+ }
+ catch (HadoopHandshakeTimeoutException e) {
+ if (log.isDebugEnabled())
+ log.debug("Handshake timed out (will retry with increased timeout) [timeout=" + connTimeout0 +
+ ", err=" + e.getMessage() + ", client=" + client + ']');
+
+ if (client != null)
+ client.forceClose();
+
+ if (attempt == reconCnt || connTimeout0 > maxConnTimeout) {
+ if (log.isDebugEnabled())
+ log.debug("Handshake timedout (will stop attempts to perform the handshake) " +
+ "[timeout=" + connTimeout0 + ", maxConnTimeout=" + maxConnTimeout +
+ ", attempt=" + attempt + ", reconCnt=" + reconCnt +
+ ", err=" + e.getMessage() + ", client=" + client + ']');
+
+ throw e;
+ }
+ else {
+ attempt++;
+
+ connTimeout0 *= 2;
+
+ continue;
+ }
+ }
+ catch (RuntimeException | Error e) {
+ if (log.isDebugEnabled())
+ log.debug(
+ "Caught exception (will close client) [err=" + e.getMessage() + ", client=" + client + ']');
+
+ if (client != null)
+ client.forceClose();
+
+ throw e;
+ }
+
+ return client;
+ }
+ }
+
+ /**
+ * Establish TCP connection to remote hadoop process and returns client.
+ *
+ * @param desc Process descriptor.
+ * @return Client.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected HadoopCommunicationClient createTcpClient(HadoopProcessDescriptor desc) throws IgniteCheckedException {
+ String addr = desc.address();
+
+ int port = desc.tcpPort();
+
+ if (log.isDebugEnabled())
+ log.debug("Trying to connect to remote process [locProcDesc=" + locProcDesc + ", desc=" + desc + ']');
+
+ boolean conn = false;
+ HadoopTcpNioCommunicationClient client = null;
+ IgniteCheckedException errs = null;
+
+ int connectAttempts = 1;
+
+ long connTimeout0 = connTimeout;
+
+ int attempt = 1;
+
+ while (!conn) { // Reconnection on handshake timeout.
+ try {
+ SocketChannel ch = SocketChannel.open();
+
+ ch.configureBlocking(true);
+
+ ch.socket().setTcpNoDelay(tcpNoDelay);
+ ch.socket().setKeepAlive(true);
+
+ if (sockRcvBuf > 0)
+ ch.socket().setReceiveBufferSize(sockRcvBuf);
+
+ if (sockSndBuf > 0)
+ ch.socket().setSendBufferSize(sockSndBuf);
+
+ ch.socket().connect(new InetSocketAddress(addr, port), (int)connTimeout);
+
+ HandshakeFinish fin = new HandshakeFinish();
+
+ GridNioSession ses = nioSrvr.createSession(ch, F.asMap(HANDSHAKE_FINISH_META, fin)).get();
+
+ client = new HadoopTcpNioCommunicationClient(ses);
+
+ if (log.isDebugEnabled())
+ log.debug("Waiting for handshake finish for client: " + client);
+
+ fin.await(connTimeout0);
+
+ conn = true;
+ }
+ catch (HadoopHandshakeTimeoutException e) {
+ if (client != null) {
+ client.forceClose();
+
+ client = null;
+ }
+
+ if (log.isDebugEnabled())
+ log.debug(
+ "Handshake timedout (will retry with increased timeout) [timeout=" + connTimeout0 +
+ ", desc=" + desc + ", port=" + port + ", err=" + e + ']');
+
+ if (attempt == reconCnt || connTimeout0 > maxConnTimeout) {
+ if (log.isDebugEnabled())
+ log.debug("Handshake timed out (will stop attempts to perform the handshake) " +
+ "[timeout=" + connTimeout0 + ", maxConnTimeout=" + maxConnTimeout +
+ ", attempt=" + attempt + ", reconCnt=" + reconCnt +
+ ", err=" + e.getMessage() + ", addr=" + addr + ']');
+
+ if (errs == null)
+ errs = new IgniteCheckedException("Failed to connect to remote Hadoop process " +
+ "(is process still running?) [desc=" + desc + ", addrs=" + addr + ']');
+
+ errs.addSuppressed(e);
+
+ break;
+ }
+ else {
+ attempt++;
+
+ connTimeout0 *= 2;
+
+ // Continue loop.
+ }
+ }
+ catch (Exception e) {
+ if (client != null) {
+ client.forceClose();
+
+ client = null;
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Client creation failed [addr=" + addr + ", port=" + port +
+ ", err=" + e + ']');
+
+ if (X.hasCause(e, SocketTimeoutException.class))
+ LT.warn(log, null, "Connect timed out (consider increasing 'connTimeout' " +
+ "configuration property) [addr=" + addr + ", port=" + port + ']');
+
+ if (errs == null)
+ errs = new IgniteCheckedException("Failed to connect to remote Hadoop process (is process still running?) " +
+ "[desc=" + desc + ", addrs=" + addr + ']');
+
+ errs.addSuppressed(e);
+
+ // Reconnect for the second time, if connection is not established.
+ if (connectAttempts < 2 &&
+ (e instanceof ConnectException || X.hasCause(e, ConnectException.class))) {
+ connectAttempts++;
+
+ continue;
+ }
+
+ break;
+ }
+ }
+
+ if (client == null) {
+ assert errs != null;
+
+ if (X.hasCause(errs, ConnectException.class))
+ LT.warn(log, null, "Failed to connect to a remote Hadoop process (is process still running?). " +
+ "Make sure operating system firewall is disabled on local and remote host) " +
+ "[addrs=" + addr + ", port=" + port + ']');
+
+ throw errs;
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Created client: " + client);
+
+ return client;
+ }
+
+ /**
+ * @param desc Sender process descriptor.
+ * @param msg Communication message.
+ */
+ protected void notifyListener(HadoopProcessDescriptor desc, HadoopMessage msg) {
+ HadoopMessageListener lsnr = this.lsnr;
+
+ if (lsnr != null)
+ // Notify listener of a new message.
+ lsnr.onMessageReceived(desc, msg);
+ else if (log.isDebugEnabled())
+ log.debug("Received communication message without any registered listeners (will ignore) " +
+ "[senderProcDesc=" + desc + ", msg=" + msg + ']');
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopExternalCommunication.class, this);
+ }
+
+ /**
+ * This worker takes responsibility to shut the server down when stopping,
+ * No other thread shall stop passed server.
+ */
+ private class ShmemAcceptWorker extends GridWorker {
+ /** */
+ private final IpcSharedMemoryServerEndpoint srv;
+
+ /**
+ * @param srv Server.
+ */
+ ShmemAcceptWorker(IpcSharedMemoryServerEndpoint srv) {
+ super(gridName, "shmem-communication-acceptor", log);
+
+ this.srv = srv;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void body() throws InterruptedException {
+ try {
+ while (!Thread.interrupted()) {
+ ShmemWorker e = new ShmemWorker(srv.accept(), true);
+
+ shmemWorkers.add(e);
+
+ new IgniteThread(e).start();
+ }
+ }
+ catch (IgniteCheckedException e) {
+ if (!isCancelled())
+ U.error(log, "Shmem server failed.", e);
+ }
+ finally {
+ srv.close();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancel() {
+ super.cancel();
+
+ srv.close();
+ }
+ }
+
+ /**
+ *
+ */
+ private class ShmemWorker extends GridWorker {
+ /** */
+ private final IpcEndpoint endpoint;
+
+ /** Adapter. */
+ private HadoopIpcToNioAdapter<HadoopMessage> adapter;
+
+ /**
+ * @param endpoint Endpoint.
+ */
+ private ShmemWorker(IpcEndpoint endpoint, boolean accepted) {
+ super(gridName, "shmem-worker", log);
+
+ this.endpoint = endpoint;
+
+ adapter = new HadoopIpcToNioAdapter<>(
+ HadoopExternalCommunication.this.log,
+ endpoint,
+ accepted,
+ srvLsnr,
+ filters());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void body() throws InterruptedException {
+ try {
+ adapter.serve();
+ }
+ finally {
+ shmemWorkers.remove(this);
+
+ endpoint.close();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancel() {
+ super.cancel();
+
+ endpoint.close();
+ }
+
+ /** @{@inheritDoc} */
+ @Override protected void cleanup() {
+ super.cleanup();
+
+ endpoint.close();
+ }
+
+ /** @{@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ShmemWorker.class, this);
+ }
+
+ /**
+ * @return NIO session for this worker.
+ */
+ public GridNioSession session() {
+ return adapter.session();
+ }
+ }
+
+ /**
+ *
+ */
+ private static class HandshakeFinish {
+ /** Await latch. */
+ private CountDownLatch latch = new CountDownLatch(1);
+
+ /**
+ * Finishes handshake.
+ */
+ public void finish() {
+ latch.countDown();
+ }
+
+ /**
+ * @param time Time to wait.
+ * @throws HadoopHandshakeTimeoutException If failed to wait.
+ */
+ public void await(long time) throws HadoopHandshakeTimeoutException {
+ try {
+ if (!latch.await(time, TimeUnit.MILLISECONDS))
+ throw new HadoopHandshakeTimeoutException("Failed to wait for handshake to finish [timeout=" +
+ time + ']');
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new HadoopHandshakeTimeoutException("Failed to wait for handshake to finish (thread was " +
+ "interrupted) [timeout=" + time + ']', e);
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private class HandshakeAndBackpressureFilter extends GridNioFilterAdapter {
+ /**
+ * Assigns filter name to a filter.
+ */
+ protected HandshakeAndBackpressureFilter() {
+ super("HadoopHandshakeFilter");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onSessionOpened(final GridNioSession ses) throws IgniteCheckedException {
+ if (ses.accepted()) {
+ if (log.isDebugEnabled())
+ log.debug("Accepted connection, initiating handshake: " + ses);
+
+ // Server initiates handshake.
+ ses.send(locIdMsg).listenAsync(new CI1<GridNioFuture<?>>() {
+ @Override public void apply(GridNioFuture<?> fut) {
+ try {
+ // Make sure there were no errors.
+ fut.get();
+ }
+ catch (IgniteCheckedException | IOException e) {
+ log.warning("Failed to send handshake message, will close session: " + ses, e);
+
+ ses.close();
+ }
+ }
+ });
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onSessionClosed(GridNioSession ses) throws IgniteCheckedException {
+ proceedSessionClosed(ses);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onExceptionCaught(GridNioSession ses, IgniteCheckedException ex) throws IgniteCheckedException {
+ proceedExceptionCaught(ses, ex);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+ if (ses.meta(PROCESS_META) == null && !(msg instanceof ProcessHandshakeMessage))
+ log.warning("Writing message before handshake has finished [ses=" + ses + ", msg=" + msg + ']');
+
+ return proceedSessionWrite(ses, msg);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onMessageReceived(GridNioSession ses, Object msg) throws IgniteCheckedException {
+ HadoopProcessDescriptor desc = ses.meta(PROCESS_META);
+
+ UUID rmtProcId = desc == null ? null : desc.processId();
+
+ if (rmtProcId == null) {
+ if (!(msg instanceof ProcessHandshakeMessage)) {
+ log.warning("Invalid handshake message received, will close connection [ses=" + ses +
+ ", msg=" + msg + ']');
+
+ ses.close();
+
+ return;
+ }
+
+ ProcessHandshakeMessage nId = (ProcessHandshakeMessage)msg;
+
+ if (log.isDebugEnabled())
+ log.debug("Received handshake message [ses=" + ses + ", msg=" + msg + ']');
+
+ ses.addMeta(PROCESS_META, nId.processDescriptor());
+
+ if (!ses.accepted())
+ // Send handshake reply.
+ ses.send(locIdMsg);
+ else {
+ //
+ rmtProcId = nId.processDescriptor().processId();
+
+ if (log.isDebugEnabled())
+ log.debug("Finished handshake with remote client: " + ses);
+
+ Object sync = locks.tryLock(rmtProcId);
+
+ if (sync != null) {
+ try {
+ if (clients.get(rmtProcId) == null) {
+ if (log.isDebugEnabled())
+ log.debug("Will reuse session for descriptor: " + rmtProcId);
+
+ // Handshake finished flag is true.
+ clients.put(rmtProcId, new HadoopTcpNioCommunicationClient(ses));
+ }
+ else {
+ if (log.isDebugEnabled())
+ log.debug("Will not reuse client as another already exists [locProcDesc=" +
+ locProcDesc + ", desc=" + desc + ']');
+ }
+ }
+ finally {
+ locks.unlock(rmtProcId, sync);
+ }
+ }
+ else {
+ if (log.isDebugEnabled())
+ log.debug("Concurrent connection is being established, will not reuse client session [" +
+ "locProcDesc=" + locProcDesc + ", desc=" + desc + ']');
+ }
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Handshake is finished for session [ses=" + ses + ", locProcDesc=" + locProcDesc + ']');
+
+ HandshakeFinish to = ses.meta(HANDSHAKE_FINISH_META);
+
+ if (to != null)
+ to.finish();
+
+ // Notify session opened (both parties).
+ proceedSessionOpened(ses);
+ }
+ else {
+ if (msgQueueLimit > 0) {
+ GridNioMessageTracker tracker = ses.meta(TRACKER_META);
+
+ if (tracker == null) {
+ GridNioMessageTracker old = ses.addMeta(TRACKER_META, tracker =
+ new GridNioMessageTracker(ses, msgQueueLimit));
+
+ assert old == null;
+ }
+
+ tracker.onMessageReceived();
+ }
+
+ proceedMessageReceived(ses, msg);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridNioFuture<Boolean> onSessionClose(GridNioSession ses) throws IgniteCheckedException {
+ return proceedSessionClose(ses);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onSessionIdleTimeout(GridNioSession ses) throws IgniteCheckedException {
+ proceedSessionIdleTimeout(ses);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onSessionWriteTimeout(GridNioSession ses) throws IgniteCheckedException {
+ proceedSessionWriteTimeout(ses);
+ }
+ }
+
+ /**
+ * Process ID message.
+ */
+ @SuppressWarnings("PublicInnerClass")
+ public static class ProcessHandshakeMessage implements HadoopMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Node ID. */
+ private HadoopProcessDescriptor procDesc;
+
+ /** */
+ public ProcessHandshakeMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param procDesc Process descriptor.
+ */
+ private ProcessHandshakeMessage(HadoopProcessDescriptor procDesc) {
+ this.procDesc = procDesc;
+ }
+
+ /**
+ * @return Process ID.
+ */
+ public HadoopProcessDescriptor processDescriptor() {
+ return procDesc;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeObject(procDesc);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ procDesc = (HadoopProcessDescriptor)in.readObject();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ProcessHandshakeMessage.class, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopHandshakeTimeoutException.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopHandshakeTimeoutException.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopHandshakeTimeoutException.java
new file mode 100644
index 0000000..36cefcb
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopHandshakeTimeoutException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
+
+import org.apache.ignite.*;
+import org.jetbrains.annotations.*;
+
+/** Internal exception class for proper timeout handling. */
+class HadoopHandshakeTimeoutException extends IgniteCheckedException {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * @param msg Message.
+ */
+ HadoopHandshakeTimeoutException(String msg) {
+ super(msg);
+ }
+
+ /**
+ * @param msg Message.
+ * @param cause Cause.
+ */
+ HadoopHandshakeTimeoutException(String msg, @Nullable Throwable cause) {
+ super(msg, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java
new file mode 100644
index 0000000..8dbc96b
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.util.ipc.*;
+import org.apache.ignite.internal.util.nio.*;
+
+import java.io.*;
+import java.nio.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+
+/**
+ * Allows to re-use existing {@link GridNioFilter}s on IPC (specifically shared memory IPC)
+ * communications.
+ *
+ * Note that this class consumes an entire thread inside {@link #serve()} method
+ * in order to serve one {@link org.apache.ignite.internal.util.ipc.IpcEndpoint}.
+ */
+public class HadoopIpcToNioAdapter<T> {
+ /** */
+ private final IpcEndpoint endp;
+
+ /** */
+ private final GridNioFilterChain<T> chain;
+
+ /** */
+ private final GridNioSessionImpl ses;
+
+ /** */
+ private final AtomicReference<CountDownLatch> latchRef = new AtomicReference<>();
+
+ /** */
+ private final ByteBuffer writeBuf;
+
+ /**
+ * @param log Log.
+ * @param endp Endpoint.
+ * @param lsnr Listener.
+ * @param filters Filters.
+ */
+ public HadoopIpcToNioAdapter(IgniteLogger log, IpcEndpoint endp, boolean accepted,
+ GridNioServerListener<T> lsnr, GridNioFilter... filters) {
+ this.endp = endp;
+
+ chain = new GridNioFilterChain<>(log, lsnr, new HeadFilter(), filters);
+ ses = new GridNioSessionImpl(chain, null, null, accepted);
+
+ writeBuf = ByteBuffer.allocate(8 << 10);
+
+ writeBuf.order(ByteOrder.nativeOrder());
+ }
+
+ /**
+ * Serves given set of listeners repeatedly reading data from the endpoint.
+ *
+ * @throws InterruptedException If interrupted.
+ */
+ public void serve() throws InterruptedException {
+ try {
+ chain.onSessionOpened(ses);
+
+ InputStream in = endp.inputStream();
+
+ ByteBuffer readBuf = ByteBuffer.allocate(8 << 10);
+
+ readBuf.order(ByteOrder.nativeOrder());
+
+ assert readBuf.hasArray();
+
+ while (!Thread.interrupted()) {
+ int pos = readBuf.position();
+
+ int read = in.read(readBuf.array(), pos, readBuf.remaining());
+
+ if (read > 0) {
+ readBuf.position(0);
+ readBuf.limit(pos + read);
+
+ chain.onMessageReceived(ses, readBuf);
+
+ if (readBuf.hasRemaining())
+ readBuf.compact();
+ else
+ readBuf.clear();
+
+ CountDownLatch latch = latchRef.get();
+
+ if (latch != null)
+ latch.await();
+ }
+ else if (read < 0) {
+ endp.close();
+
+ break; // And close below.
+ }
+ }
+
+ // Assuming remote end closed connection - pushing event from head to tail.
+ chain.onSessionClosed(ses);
+ }
+ catch (Exception e) {
+ chain.onExceptionCaught(ses, new IgniteCheckedException("Failed to read from IPC endpoint.", e));
+ }
+ }
+
+ /**
+ * Gets dummy session for this adapter.
+ *
+ * @return Session.
+ */
+ public GridNioSession session() {
+ return ses;
+ }
+
+ /**
+ * Handles write events on chain.
+ *
+ * @param msg Buffer to send.
+ * @return Send result.
+ */
+ private GridNioFuture<?> send(ByteBuffer msg) {
+ assert writeBuf.hasArray();
+
+ try {
+ while (msg.hasRemaining()) {
+ writeBuf.clear();
+
+ writeBuf.put(msg);
+
+ endp.outputStream().write(writeBuf.array(), 0, writeBuf.position());
+ }
+ }
+ catch (IOException | IgniteCheckedException e) {
+ return new GridNioFinishedFuture<Object>(e);
+ }
+
+ return new GridNioFinishedFuture<>((Object)null);
+ }
+
+ /**
+ * Filter forwarding messages from chain's head to this server.
+ */
+ private class HeadFilter extends GridNioFilterAdapter {
+ /**
+ * Assigns filter name.
+ */
+ protected HeadFilter() {
+ super("HeadFilter");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onSessionOpened(GridNioSession ses) throws IgniteCheckedException {
+ proceedSessionOpened(ses);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onSessionClosed(GridNioSession ses) throws IgniteCheckedException {
+ proceedSessionClosed(ses);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onExceptionCaught(GridNioSession ses, IgniteCheckedException ex) throws IgniteCheckedException {
+ proceedExceptionCaught(ses, ex);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) {
+ assert ses == HadoopIpcToNioAdapter.this.ses : "ses=" + ses +
+ ", this.ses=" + HadoopIpcToNioAdapter.this.ses;
+
+ return send((ByteBuffer)msg);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onMessageReceived(GridNioSession ses, Object msg) throws IgniteCheckedException {
+ proceedMessageReceived(ses, msg);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridNioFuture<?> onPauseReads(GridNioSession ses) throws IgniteCheckedException {
+ // This call should be synced externally to avoid races.
+ boolean b = latchRef.compareAndSet(null, new CountDownLatch(1));
+
+ assert b;
+
+ return new GridNioFinishedFuture<>(b);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridNioFuture<?> onResumeReads(GridNioSession ses) throws IgniteCheckedException {
+ // This call should be synced externally to avoid races.
+ CountDownLatch latch = latchRef.getAndSet(null);
+
+ if (latch != null)
+ latch.countDown();
+
+ return new GridNioFinishedFuture<Object>(latch != null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridNioFuture<Boolean> onSessionClose(GridNioSession ses) {
+ assert ses == HadoopIpcToNioAdapter.this.ses;
+
+ boolean closed = HadoopIpcToNioAdapter.this.ses.setClosed();
+
+ if (closed)
+ endp.close();
+
+ return new GridNioFinishedFuture<>(closed);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onSessionIdleTimeout(GridNioSession ses) throws IgniteCheckedException {
+ proceedSessionIdleTimeout(ses);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onSessionWriteTimeout(GridNioSession ses) throws IgniteCheckedException {
+ proceedSessionWriteTimeout(ses);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java
new file mode 100644
index 0000000..4cba117
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.util.nio.*;
+import org.apache.ignite.marshaller.*;
+
+/**
+ * Serialization filter.
+ */
+public class HadoopMarshallerFilter extends GridNioFilterAdapter {
+ /** Marshaller. */
+ private Marshaller marshaller;
+
+ /**
+ * @param marshaller Marshaller to use.
+ */
+ public HadoopMarshallerFilter(Marshaller marshaller) {
+ super("HadoopMarshallerFilter");
+
+ this.marshaller = marshaller;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onSessionOpened(GridNioSession ses) throws IgniteCheckedException {
+ proceedSessionOpened(ses);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onSessionClosed(GridNioSession ses) throws IgniteCheckedException {
+ proceedSessionClosed(ses);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onExceptionCaught(GridNioSession ses, IgniteCheckedException ex) throws IgniteCheckedException {
+ proceedExceptionCaught(ses, ex);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+ assert msg instanceof HadoopMessage : "Invalid message type: " + msg;
+
+ return proceedSessionWrite(ses, marshaller.marshal(msg));
+ }
+
+ @Override public void onMessageReceived(GridNioSession ses, Object msg) throws IgniteCheckedException {
+ assert msg instanceof byte[];
+
+ // Always unmarshal with system classloader.
+ proceedMessageReceived(ses, marshaller.unmarshal((byte[])msg, null));
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridNioFuture<Boolean> onSessionClose(GridNioSession ses) throws IgniteCheckedException {
+ return proceedSessionClose(ses);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onSessionIdleTimeout(GridNioSession ses) throws IgniteCheckedException {
+ proceedSessionIdleTimeout(ses);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onSessionWriteTimeout(GridNioSession ses) throws IgniteCheckedException {
+ proceedSessionWriteTimeout(ses);
+ }
+}
[05/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTasksAllVersionsTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTasksAllVersionsTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTasksAllVersionsTest.java
deleted file mode 100644
index af3f872..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTasksAllVersionsTest.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import com.google.common.base.*;
-import org.apache.hadoop.io.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.processors.hadoop.examples.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-
-/**
- * Tests of Map, Combine and Reduce task executions of any version of hadoop API.
- */
-abstract class GridHadoopTasksAllVersionsTest extends GridHadoopAbstractWordCountTest {
- /** Empty hosts array. */
- private static final String[] HOSTS = new String[0];
-
- /**
- * Creates some grid hadoop job. Override this method to create tests for any job implementation.
- *
- * @param inFile Input file name for the job.
- * @param outFile Output file name for the job.
- * @return Hadoop job.
- * @throws IOException If fails.
- */
- public abstract GridHadoopV2Job getHadoopJob(String inFile, String outFile) throws Exception;
-
- /**
- * @return prefix of reducer output file name. It's "part-" for v1 and "part-r-" for v2 API
- */
- public abstract String getOutputFileNamePrefix();
-
- /**
- * Tests map task execution.
- *
- * @throws Exception If fails.
- */
- @SuppressWarnings("ConstantConditions")
- public void testMapTask() throws Exception {
- IgfsPath inDir = new IgfsPath(PATH_INPUT);
-
- igfs.mkdirs(inDir);
-
- IgfsPath inFile = new IgfsPath(inDir, GridHadoopWordCount2.class.getSimpleName() + "-input");
-
- URI inFileUri = URI.create(igfsScheme() + inFile.toString());
-
- try (PrintWriter pw = new PrintWriter(igfs.create(inFile, true))) {
- pw.println("hello0 world0");
- pw.println("world1 hello1");
- }
-
- GridHadoopFileBlock fileBlock1 = new GridHadoopFileBlock(HOSTS, inFileUri, 0, igfs.info(inFile).length() - 1);
-
- try (PrintWriter pw = new PrintWriter(igfs.append(inFile, false))) {
- pw.println("hello2 world2");
- pw.println("world3 hello3");
- }
- GridHadoopFileBlock fileBlock2 = new GridHadoopFileBlock(HOSTS, inFileUri, fileBlock1.length(),
- igfs.info(inFile).length() - fileBlock1.length());
-
- GridHadoopV2Job gridJob = getHadoopJob(igfsScheme() + inFile.toString(), igfsScheme() + PATH_OUTPUT);
-
- GridHadoopTaskInfo taskInfo = new GridHadoopTaskInfo(GridHadoopTaskType.MAP, gridJob.id(), 0, 0, fileBlock1);
-
- GridHadoopTestTaskContext ctx = new GridHadoopTestTaskContext(taskInfo, gridJob);
-
- ctx.mockOutput().clear();
-
- ctx.run();
-
- assertEquals("hello0,1; world0,1; world1,1; hello1,1", Joiner.on("; ").join(ctx.mockOutput()));
-
- ctx.mockOutput().clear();
-
- ctx.taskInfo(new GridHadoopTaskInfo(GridHadoopTaskType.MAP, gridJob.id(), 0, 0, fileBlock2));
-
- ctx.run();
-
- assertEquals("hello2,1; world2,1; world3,1; hello3,1", Joiner.on("; ").join(ctx.mockOutput()));
- }
-
- /**
- * Generates input data for reduce-like operation into mock context input and runs the operation.
- *
- * @param gridJob Job is to create reduce task from.
- * @param taskType Type of task - combine or reduce.
- * @param taskNum Number of task in job.
- * @param words Pairs of words and its counts.
- * @return Context with mock output.
- * @throws IgniteCheckedException If fails.
- */
- private GridHadoopTestTaskContext runTaskWithInput(GridHadoopV2Job gridJob, GridHadoopTaskType taskType,
- int taskNum, String... words) throws IgniteCheckedException {
- GridHadoopTaskInfo taskInfo = new GridHadoopTaskInfo(taskType, gridJob.id(), taskNum, 0, null);
-
- GridHadoopTestTaskContext ctx = new GridHadoopTestTaskContext(taskInfo, gridJob);
-
- for (int i = 0; i < words.length; i+=2) {
- List<IntWritable> valList = new ArrayList<>();
-
- for (int j = 0; j < Integer.parseInt(words[i + 1]); j++)
- valList.add(new IntWritable(1));
-
- ctx.mockInput().put(new Text(words[i]), valList);
- }
-
- ctx.run();
-
- return ctx;
- }
-
- /**
- * Tests reduce task execution.
- *
- * @throws Exception If fails.
- */
- public void testReduceTask() throws Exception {
- GridHadoopV2Job gridJob = getHadoopJob(igfsScheme() + PATH_INPUT, igfsScheme() + PATH_OUTPUT);
-
- runTaskWithInput(gridJob, GridHadoopTaskType.REDUCE, 0, "word1", "5", "word2", "10");
- runTaskWithInput(gridJob, GridHadoopTaskType.REDUCE, 1, "word3", "7", "word4", "15");
-
- assertEquals(
- "word1\t5\n" +
- "word2\t10\n",
- readAndSortFile(PATH_OUTPUT + "/_temporary/0/task_00000000-0000-0000-0000-000000000000_0000_r_000000/" +
- getOutputFileNamePrefix() + "00000")
- );
-
- assertEquals(
- "word3\t7\n" +
- "word4\t15\n",
- readAndSortFile(PATH_OUTPUT + "/_temporary/0/task_00000000-0000-0000-0000-000000000000_0000_r_000001/" +
- getOutputFileNamePrefix() + "00001")
- );
- }
-
- /**
- * Tests combine task execution.
- *
- * @throws Exception If fails.
- */
- public void testCombinerTask() throws Exception {
- GridHadoopV2Job gridJob = getHadoopJob("/", "/");
-
- GridHadoopTestTaskContext ctx =
- runTaskWithInput(gridJob, GridHadoopTaskType.COMBINE, 0, "word1", "5", "word2", "10");
-
- assertEquals("word1,5; word2,10", Joiner.on("; ").join(ctx.mockOutput()));
-
- ctx = runTaskWithInput(gridJob, GridHadoopTaskType.COMBINE, 1, "word3", "7", "word4", "15");
-
- assertEquals("word3,7; word4,15", Joiner.on("; ").join(ctx.mockOutput()));
- }
-
- /**
- * Runs chain of map-combine task on file block.
- *
- * @param fileBlock block of input file to be processed.
- * @param gridJob Hadoop job implementation.
- * @return Context of combine task with mock output.
- * @throws IgniteCheckedException If fails.
- */
- private GridHadoopTestTaskContext runMapCombineTask(GridHadoopFileBlock fileBlock, GridHadoopV2Job gridJob)
- throws IgniteCheckedException {
- GridHadoopTaskInfo taskInfo = new GridHadoopTaskInfo(GridHadoopTaskType.MAP, gridJob.id(), 0, 0, fileBlock);
-
- GridHadoopTestTaskContext mapCtx = new GridHadoopTestTaskContext(taskInfo, gridJob);
-
- mapCtx.run();
-
- //Prepare input for combine
- taskInfo = new GridHadoopTaskInfo(GridHadoopTaskType.COMBINE, gridJob.id(), 0, 0, null);
-
- GridHadoopTestTaskContext combineCtx = new GridHadoopTestTaskContext(taskInfo, gridJob);
-
- combineCtx.makeTreeOfWritables(mapCtx.mockOutput());
-
- combineCtx.run();
-
- return combineCtx;
- }
-
- /**
- * Tests all job in complex.
- * Runs 2 chains of map-combine tasks and sends result into one reduce task.
- *
- * @throws Exception If fails.
- */
- @SuppressWarnings("ConstantConditions")
- public void testAllTasks() throws Exception {
- IgfsPath inDir = new IgfsPath(PATH_INPUT);
-
- igfs.mkdirs(inDir);
-
- IgfsPath inFile = new IgfsPath(inDir, GridHadoopWordCount2.class.getSimpleName() + "-input");
-
- URI inFileUri = URI.create(igfsScheme() + inFile.toString());
-
- generateTestFile(inFile.toString(), "red", 100, "blue", 200, "green", 150, "yellow", 70);
-
- //Split file into two blocks
- long fileLen = igfs.info(inFile).length();
-
- Long l = fileLen / 2;
-
- GridHadoopFileBlock fileBlock1 = new GridHadoopFileBlock(HOSTS, inFileUri, 0, l);
- GridHadoopFileBlock fileBlock2 = new GridHadoopFileBlock(HOSTS, inFileUri, l, fileLen - l);
-
- GridHadoopV2Job gridJob = getHadoopJob(inFileUri.toString(), igfsScheme() + PATH_OUTPUT);
-
- GridHadoopTestTaskContext combine1Ctx = runMapCombineTask(fileBlock1, gridJob);
-
- GridHadoopTestTaskContext combine2Ctx = runMapCombineTask(fileBlock2, gridJob);
-
- //Prepare input for combine
- GridHadoopTaskInfo taskInfo = new GridHadoopTaskInfo(GridHadoopTaskType.REDUCE, gridJob.id(), 0, 0, null);
-
- GridHadoopTestTaskContext reduceCtx = new GridHadoopTestTaskContext(taskInfo, gridJob);
-
- reduceCtx.makeTreeOfWritables(combine1Ctx.mockOutput());
- reduceCtx.makeTreeOfWritables(combine2Ctx.mockOutput());
-
- reduceCtx.run();
-
- reduceCtx.taskInfo(new GridHadoopTaskInfo(GridHadoopTaskType.COMMIT, gridJob.id(), 0, 0, null));
-
- reduceCtx.run();
-
- assertEquals(
- "blue\t200\n" +
- "green\t150\n" +
- "red\t100\n" +
- "yellow\t70\n",
- readAndSortFile(PATH_OUTPUT + "/" + getOutputFileNamePrefix() + "00000")
- );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTasksV1Test.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTasksV1Test.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTasksV1Test.java
deleted file mode 100644
index 15ac125..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTasksV1Test.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.mapred.*;
-import org.apache.ignite.internal.processors.hadoop.examples.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-
-import java.io.*;
-import java.util.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Tests of Map, Combine and Reduce task executions via running of job of hadoop API v1.
- */
-public class GridHadoopTasksV1Test extends GridHadoopTasksAllVersionsTest {
- /**
- * Creates WordCount hadoop job for API v1.
- *
- * @param inFile Input file name for the job.
- * @param outFile Output file name for the job.
- * @return Hadoop job.
- * @throws IOException If fails.
- */
- @Override public GridHadoopV2Job getHadoopJob(String inFile, String outFile) throws Exception {
- JobConf jobConf = GridHadoopWordCount1.getJob(inFile, outFile);
-
- setupFileSystems(jobConf);
-
- GridHadoopDefaultJobInfo jobInfo = createJobInfo(jobConf);
-
- GridHadoopJobId jobId = new GridHadoopJobId(new UUID(0, 0), 0);
-
- return new GridHadoopV2Job(jobId, jobInfo, log);
- }
-
- /** {@inheritDoc} */
- @Override public String getOutputFileNamePrefix() {
- return "part-";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTasksV2Test.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTasksV2Test.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTasksV2Test.java
deleted file mode 100644
index e48eb01..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTasksV2Test.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.hadoop.mapreduce.lib.output.*;
-import org.apache.ignite.internal.processors.hadoop.examples.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-
-import java.util.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Tests of Map, Combine and Reduce task executions via running of job of hadoop API v2.
- */
-public class GridHadoopTasksV2Test extends GridHadoopTasksAllVersionsTest {
- /**
- * Creates WordCount hadoop job for API v2.
- *
- * @param inFile Input file name for the job.
- * @param outFile Output file name for the job.
- * @return Hadoop job.
- * @throws Exception if fails.
- */
- @Override public GridHadoopV2Job getHadoopJob(String inFile, String outFile) throws Exception {
- Job job = Job.getInstance();
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- GridHadoopWordCount2.setTasksClasses(job, true, true, true);
-
- Configuration conf = job.getConfiguration();
-
- setupFileSystems(conf);
-
- FileInputFormat.setInputPaths(job, new Path(inFile));
- FileOutputFormat.setOutputPath(job, new Path(outFile));
-
- job.setJarByClass(GridHadoopWordCount2.class);
-
- Job hadoopJob = GridHadoopWordCount2.getJob(inFile, outFile);
-
- GridHadoopDefaultJobInfo jobInfo = createJobInfo(hadoopJob.getConfiguration());
-
- GridHadoopJobId jobId = new GridHadoopJobId(new UUID(0, 0), 0);
-
- return new GridHadoopV2Job(jobId, jobInfo, log);
- }
-
- /** {@inheritDoc} */
- @Override public String getOutputFileNamePrefix() {
- return "part-r-";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTestRoundRobinMrPlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTestRoundRobinMrPlanner.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTestRoundRobinMrPlanner.java
deleted file mode 100644
index 5baa8cd..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTestRoundRobinMrPlanner.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.cluster.*;
-import org.apache.ignite.internal.processors.hadoop.planner.*;
-import org.jetbrains.annotations.*;
-
-import java.util.*;
-
-/**
- * Round-robin mr planner.
- */
-public class GridHadoopTestRoundRobinMrPlanner implements GridHadoopMapReducePlanner {
- /** {@inheritDoc} */
- @Override public GridHadoopMapReducePlan preparePlan(GridHadoopJob job, Collection<ClusterNode> top,
- @Nullable GridHadoopMapReducePlan oldPlan) throws IgniteCheckedException {
- if (top.isEmpty())
- throw new IllegalArgumentException("Topology is empty");
-
- // Has at least one element.
- Iterator<ClusterNode> it = top.iterator();
-
- Map<UUID, Collection<GridHadoopInputSplit>> mappers = new HashMap<>();
-
- for (GridHadoopInputSplit block : job.input()) {
- ClusterNode node = it.next();
-
- Collection<GridHadoopInputSplit> nodeBlocks = mappers.get(node.id());
-
- if (nodeBlocks == null) {
- nodeBlocks = new ArrayList<>();
-
- mappers.put(node.id(), nodeBlocks);
- }
-
- nodeBlocks.add(block);
-
- if (!it.hasNext())
- it = top.iterator();
- }
-
- int[] rdc = new int[job.info().reducers()];
-
- for (int i = 0; i < rdc.length; i++)
- rdc[i] = i;
-
- return new GridHadoopDefaultMapReducePlan(mappers, Collections.singletonMap(it.next().id(), rdc));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTestTaskContext.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTestTaskContext.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTestTaskContext.java
deleted file mode 100644
index 4e0aa9b..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTestTaskContext.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapred.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Context for test purpose.
- */
-class GridHadoopTestTaskContext extends GridHadoopV2TaskContext {
- /**
- * Simple key-vale pair.
- * @param <K> Key class.
- * @param <V> Value class.
- */
- public static class Pair<K,V> {
- /** Key */
- private K key;
-
- /** Value */
- private V val;
-
- /**
- * @param key key.
- * @param val value.
- */
- Pair(K key, V val) {
- this.key = key;
- this.val = val;
- }
-
- /**
- * Getter of key.
- * @return key.
- */
- K key() {
- return key;
- }
-
- /**
- * Getter of value.
- * @return value.
- */
- V value() {
- return val;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return key + "," + val;
- }
- }
-
- /** Mock output container- result data of task execution if it is not overridden. */
- private List<Pair<String, Integer>> mockOutput = new ArrayList<>();
-
- /** Mock input container- input data if it is not overridden. */
- private Map<Object,List> mockInput = new TreeMap<>();
-
- /** Context output implementation to write data into mockOutput. */
- private GridHadoopTaskOutput output = new GridHadoopTaskOutput() {
- /** {@inheritDoc} */
- @Override public void write(Object key, Object val) {
- //Check of casting and extract/copy values
- String strKey = new String(((Text)key).getBytes());
- int intVal = ((IntWritable)val).get();
-
- mockOutput().add(new Pair<>(strKey, intVal));
- }
-
- /** {@inheritDoc} */
- @Override public void close() {
- throw new UnsupportedOperationException();
- }
- };
-
- /** Context input implementation to read data from mockInput. */
- private GridHadoopTaskInput input = new GridHadoopTaskInput() {
- /** Iterator of keys and associated lists of values. */
- Iterator<Map.Entry<Object, List>> iter;
-
- /** Current key and associated value list. */
- Map.Entry<Object, List> currEntry;
-
- /** {@inheritDoc} */
- @Override public boolean next() {
- if (iter == null)
- iter = mockInput().entrySet().iterator();
-
- if (iter.hasNext())
- currEntry = iter.next();
- else
- currEntry = null;
-
- return currEntry != null;
- }
-
- /** {@inheritDoc} */
- @Override public Object key() {
- return currEntry.getKey();
- }
-
- /** {@inheritDoc} */
- @Override public Iterator<?> values() {
- return currEntry.getValue().iterator() ;
- }
-
- /** {@inheritDoc} */
- @Override public void close() {
- throw new UnsupportedOperationException();
- }
- };
-
- /**
- * Getter of mock output container - result of task if it is not overridden.
- *
- * @return mock output.
- */
- public List<Pair<String, Integer>> mockOutput() {
- return mockOutput;
- }
-
- /**
- * Getter of mock input container- input data if it is not overridden.
- *
- * @return mock output.
- */
- public Map<Object, List> mockInput() {
- return mockInput;
- }
-
- /**
- * Generate one-key-multiple-values tree from array of key-value pairs, and wrap its into Writable objects.
- * The result is placed into mock input.
- *
- * @param flatData list of key-value pair.
- */
- public void makeTreeOfWritables(Iterable<Pair<String, Integer>> flatData) {
- Text key = new Text();
-
- for (GridHadoopTestTaskContext.Pair<String, Integer> pair : flatData) {
- key.set(pair.key);
- ArrayList<IntWritable> valList;
-
- if (!mockInput.containsKey(key)) {
- valList = new ArrayList<>();
- mockInput.put(key, valList);
- key = new Text();
- }
- else
- valList = (ArrayList<IntWritable>) mockInput.get(key);
- valList.add(new IntWritable(pair.value()));
- }
- }
-
- /**
- * @param taskInfo Task info.
- * @param gridJob Grid Hadoop job.
- */
- public GridHadoopTestTaskContext(GridHadoopTaskInfo taskInfo, GridHadoopJob gridJob) throws IgniteCheckedException {
- super(taskInfo, gridJob, gridJob.id(), null, jobConfDataInput(gridJob));
- }
-
- /**
- * Creates DataInput to read JobConf.
- *
- * @param job Job.
- * @return DataInput with JobConf.
- * @throws IgniteCheckedException If failed.
- */
- private static DataInput jobConfDataInput(GridHadoopJob job) throws IgniteCheckedException {
- JobConf jobConf = new JobConf();
-
- for (Map.Entry<String, String> e : ((GridHadoopDefaultJobInfo)job.info()).properties().entrySet())
- jobConf.set(e.getKey(), e.getValue());
-
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
-
- try {
- jobConf.write(new DataOutputStream(buf));
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
-
- return new DataInputStream(new ByteArrayInputStream(buf.toByteArray()));
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopTaskOutput output() {
- return output;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopTaskInput input() {
- return input;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTestUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTestUtils.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTestUtils.java
deleted file mode 100644
index cdbb809..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTestUtils.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.internal.util.typedef.*;
-
-import java.io.*;
-import java.util.*;
-
-import static org.junit.Assert.*;
-
-/**
- * Utility class for tests.
- */
-public class GridHadoopTestUtils {
- /**
- * Checks that job statistics file contains valid strings only.
- *
- * @param reader Buffered reader to get lines of job statistics.
- * @return Amount of events.
- * @throws IOException If failed.
- */
- public static long simpleCheckJobStatFile(BufferedReader reader) throws IOException {
- Collection<String> phases = new HashSet<>();
-
- phases.add("submit");
- phases.add("prepare");
- phases.add("start");
- phases.add("finish");
- phases.add("requestId");
- phases.add("responseId");
-
- Collection<String> evtTypes = new HashSet<>();
-
- evtTypes.add("JOB");
- evtTypes.add("SETUP");
- evtTypes.add("MAP");
- evtTypes.add("SHUFFLE");
- evtTypes.add("REDUCE");
- evtTypes.add("COMBINE");
- evtTypes.add("COMMIT");
-
- long evtCnt = 0;
- String line;
-
- Map<Long, String> reduceNodes = new HashMap<>();
-
- while((line = reader.readLine()) != null) {
- String[] splitLine = line.split(":");
-
- //Try parse timestamp
- Long.parseLong(splitLine[1]);
-
- String[] evt = splitLine[0].split(" ");
-
- assertTrue("Unknown event '" + evt[0] + "'", evtTypes.contains(evt[0]));
-
- String phase;
-
- if ("JOB".equals(evt[0]))
- phase = evt[1];
- else {
- assertEquals(4, evt.length);
- assertTrue("The node id is not defined", !F.isEmpty(evt[3]));
-
- long taskNum = Long.parseLong(evt[1]);
-
- if (("REDUCE".equals(evt[0]) || "SHUFFLE".equals(evt[0]))) {
- String nodeId = reduceNodes.get(taskNum);
-
- if (nodeId == null)
- reduceNodes.put(taskNum, evt[3]);
- else
- assertEquals("Different nodes for SHUFFLE and REDUCE tasks", nodeId, evt[3]);
- }
-
- phase = evt[2];
- }
-
- assertTrue("Unknown phase '" + phase + "' in " + Arrays.toString(evt), phases.contains(phase));
-
- evtCnt++;
- }
-
- return evtCnt;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopV2JobSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopV2JobSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopV2JobSelfTest.java
deleted file mode 100644
index b201614..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopV2JobSelfTest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.io.serializer.*;
-import org.apache.hadoop.mapred.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-
-import java.io.*;
-import java.util.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Self test of {@link GridHadoopV2Job}.
- */
-public class GridHadoopV2JobSelfTest extends GridHadoopAbstractSelfTest {
- /** */
- private static final String TEST_SERIALIZED_VALUE = "Test serialized value";
-
- /**
- * Custom serialization class that accepts {@link Writable}.
- */
- private static class CustomSerialization extends WritableSerialization {
- /** {@inheritDoc} */
- @Override public Deserializer<Writable> getDeserializer(Class<Writable> c) {
- return new Deserializer<Writable>() {
- @Override public void open(InputStream in) { }
-
- @Override public Writable deserialize(Writable writable) {
- return new Text(TEST_SERIALIZED_VALUE);
- }
-
- @Override public void close() { }
- };
- }
- }
-
- /**
- * Tests that {@link GridHadoopJob} provides wrapped serializer if it's set in configuration.
- *
- * @throws IgniteCheckedException If fails.
- */
- public void testCustomSerializationApplying() throws IgniteCheckedException {
- JobConf cfg = new JobConf();
-
- cfg.setMapOutputKeyClass(IntWritable.class);
- cfg.setMapOutputValueClass(Text.class);
- cfg.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, CustomSerialization.class.getName());
-
- GridHadoopJob job = new GridHadoopV2Job(new GridHadoopJobId(UUID.randomUUID(), 1), createJobInfo(cfg), log);
-
- GridHadoopTaskContext taskCtx = job.getTaskContext(new GridHadoopTaskInfo(GridHadoopTaskType.MAP, null, 0, 0,
- null));
-
- GridHadoopSerialization ser = taskCtx.keySerialization();
-
- assertEquals(GridHadoopSerializationWrapper.class.getName(), ser.getClass().getName());
-
- DataInput in = new DataInputStream(new ByteArrayInputStream(new byte[0]));
-
- assertEquals(TEST_SERIALIZED_VALUE, ser.read(in, null).toString());
-
- ser = taskCtx.valueSerialization();
-
- assertEquals(GridHadoopSerializationWrapper.class.getName(), ser.getClass().getName());
-
- assertEquals(TEST_SERIALIZED_VALUE, ser.read(in, null).toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopValidationSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopValidationSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopValidationSelfTest.java
deleted file mode 100644
index 051d073..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopValidationSelfTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.configuration.*;
-
-/**
- * Configuration validation tests.
- */
-public class GridHadoopValidationSelfTest extends GridHadoopAbstractSelfTest {
- /** Peer class loading enabled flag. */
- public boolean peerClassLoading;
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids(true);
-
- peerClassLoading = false;
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setPeerClassLoadingEnabled(peerClassLoading);
-
- return cfg;
- }
-
- /**
- * Ensure that Grid starts when all configuration parameters are valid.
- *
- * @throws Exception If failed.
- */
- public void testValid() throws Exception {
- startGrids(1);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractSelfTest.java
new file mode 100644
index 0000000..7fda532
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractSelfTest.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.conf.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem;
+import org.apache.ignite.internal.processors.hadoop.fs.*;
+import org.apache.ignite.spi.communication.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+import org.apache.ignite.testframework.junits.common.*;
+
+import java.io.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*;
+
+/**
+ * Abstract class for Hadoop tests.
+ */
+public abstract class HadoopAbstractSelfTest extends GridCommonAbstractTest {
+ /** */
+ private static TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** REST port. */
+ protected static final int REST_PORT = 11212;
+
+ /** IGFS name. */
+ protected static final String igfsName = null;
+
+ /** IGFS name. */
+ protected static final String igfsMetaCacheName = "meta";
+
+ /** IGFS name. */
+ protected static final String igfsDataCacheName = "data";
+
+ /** IGFS block size. */
+ protected static final int igfsBlockSize = 1024;
+
+ /** IGFS block group size. */
+ protected static final int igfsBlockGroupSize = 8;
+
+ /** Initial REST port. */
+ private int restPort = REST_PORT;
+
+ /** Initial classpath. */
+ private static String initCp;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ // Add surefire classpath to regular classpath.
+ initCp = System.getProperty("java.class.path");
+
+ String surefireCp = System.getProperty("surefire.test.class.path");
+
+ if (surefireCp != null)
+ System.setProperty("java.class.path", initCp + File.pathSeparatorChar + surefireCp);
+
+ super.beforeTestsStarted();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ // Restore classpath.
+ System.setProperty("java.class.path", initCp);
+
+ initCp = null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setHadoopConfiguration(hadoopConfiguration(gridName));
+
+ TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
+
+ commSpi.setSharedMemoryPort(-1);
+
+ cfg.setCommunicationSpi(commSpi);
+
+ TcpDiscoverySpi discoSpi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
+
+ discoSpi.setIpFinder(IP_FINDER);
+
+ if (igfsEnabled()) {
+ cfg.setCacheConfiguration(metaCacheConfiguration(), dataCacheConfiguration());
+
+ cfg.setFileSystemConfiguration(igfsConfiguration());
+ }
+
+ if (restEnabled()) {
+ ConnectorConfiguration clnCfg = new ConnectorConfiguration();
+
+ clnCfg.setPort(restPort++);
+
+ cfg.setConnectorConfiguration(clnCfg);
+ }
+
+ cfg.setLocalHost("127.0.0.1");
+ cfg.setPeerClassLoadingEnabled(false);
+
+ return cfg;
+ }
+
+ /**
+ * @param gridName Grid name.
+ * @return Hadoop configuration.
+ */
+ public HadoopConfiguration hadoopConfiguration(String gridName) {
+ HadoopConfiguration cfg = new HadoopConfiguration();
+
+ cfg.setMaxParallelTasks(3);
+
+ return cfg;
+ }
+
+ /**
+ * @return IGFS configuration.
+ */
+ public FileSystemConfiguration igfsConfiguration() {
+ FileSystemConfiguration cfg = new FileSystemConfiguration();
+
+ cfg.setName(igfsName);
+ cfg.setBlockSize(igfsBlockSize);
+ cfg.setDataCacheName(igfsDataCacheName);
+ cfg.setMetaCacheName(igfsMetaCacheName);
+ cfg.setFragmentizerEnabled(false);
+
+ return cfg;
+ }
+
+ /**
+ * @return IGFS meta cache configuration.
+ */
+ public CacheConfiguration metaCacheConfiguration() {
+ CacheConfiguration cfg = new CacheConfiguration();
+
+ cfg.setName(igfsMetaCacheName);
+ cfg.setCacheMode(REPLICATED);
+ cfg.setAtomicityMode(TRANSACTIONAL);
+ cfg.setWriteSynchronizationMode(FULL_SYNC);
+
+ return cfg;
+ }
+
+ /**
+ * @return IGFS data cache configuration.
+ */
+ private CacheConfiguration dataCacheConfiguration() {
+ CacheConfiguration cfg = new CacheConfiguration();
+
+ cfg.setName(igfsDataCacheName);
+ cfg.setCacheMode(PARTITIONED);
+ cfg.setAtomicityMode(TRANSACTIONAL);
+ cfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(igfsBlockGroupSize));
+ cfg.setWriteSynchronizationMode(FULL_SYNC);
+
+ return cfg;
+ }
+
+ /**
+ * @return {@code True} if IGFS is enabled on Hadoop nodes.
+ */
+ protected boolean igfsEnabled() {
+ return false;
+ }
+
+ /**
+ * @return {@code True} if REST is enabled on Hadoop nodes.
+ */
+ protected boolean restEnabled() {
+ return false;
+ }
+
+ /**
+ * @return Number of nodes to start.
+ */
+ protected int gridCount() {
+ return 3;
+ }
+
+ /**
+ * @param cfg Config.
+ */
+ protected void setupFileSystems(Configuration cfg) {
+ cfg.set("fs.defaultFS", igfsScheme());
+ cfg.set("fs.igfs.impl", org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.class.getName());
+ cfg.set("fs.AbstractFileSystem.igfs.impl", IgniteHadoopFileSystem.
+ class.getName());
+
+ HadoopFileSystemsUtils.setupFileSystems(cfg);
+ }
+
+ /**
+ * @return IGFS scheme for test.
+ */
+ protected String igfsScheme() {
+ return "igfs://:" + getTestGridName(0) + "@/";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractWordCountTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractWordCountTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractWordCountTest.java
new file mode 100644
index 0000000..1390982
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractWordCountTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import com.google.common.base.*;
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Abstract class for tests based on WordCount test job.
+ */
+public abstract class HadoopAbstractWordCountTest extends HadoopAbstractSelfTest {
+ /** Input path. */
+ protected static final String PATH_INPUT = "/input";
+
+ /** Output path. */
+ protected static final String PATH_OUTPUT = "/output";
+
+ /** IGFS instance. */
+ protected IgfsEx igfs;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ Configuration cfg = new Configuration();
+
+ setupFileSystems(cfg);
+
+ // Init cache by correct LocalFileSystem implementation
+ FileSystem.getLocal(cfg);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ igfs = (IgfsEx)startGrids(gridCount()).fileSystem(igfsName);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean igfsEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int gridCount() {
+ return 1;
+ }
+
+ /**
+ * Generates test file.
+ *
+ * @param path File name.
+ * @param wordCounts Words and counts.
+ * @throws Exception If failed.
+ */
+ protected void generateTestFile(String path, Object... wordCounts) throws Exception {
+ List<String> wordsArr = new ArrayList<>();
+
+ //Generating
+ for (int i = 0; i < wordCounts.length; i += 2) {
+ String word = (String) wordCounts[i];
+ int cnt = (Integer) wordCounts[i + 1];
+
+ while (cnt-- > 0)
+ wordsArr.add(word);
+ }
+
+ //Shuffling
+ for (int i = 0; i < wordsArr.size(); i++) {
+ int j = (int)(Math.random() * wordsArr.size());
+
+ Collections.swap(wordsArr, i, j);
+ }
+
+ //Input file preparing
+ PrintWriter testInputFileWriter = new PrintWriter(igfs.create(new IgfsPath(path), true));
+
+ int j = 0;
+
+ while (j < wordsArr.size()) {
+ int i = 5 + (int)(Math.random() * 5);
+
+ List<String> subList = wordsArr.subList(j, Math.min(j + i, wordsArr.size()));
+ j += i;
+
+ testInputFileWriter.println(Joiner.on(' ').join(subList));
+ }
+
+ testInputFileWriter.close();
+ }
+
+ /**
+ * Reads whole text file into String.
+ *
+ * @param fileName Name of the file to read.
+ * @return Content of the file as String value.
+ * @throws Exception If could not read the file.
+ */
+ protected String readAndSortFile(String fileName) throws Exception {
+ BufferedReader reader = new BufferedReader(new InputStreamReader(igfs.open(new IgfsPath(fileName))));
+
+ List<String> list = new ArrayList<>();
+
+ String line;
+
+ while ((line = reader.readLine()) != null)
+ list.add(line);
+
+ Collections.sort(list);
+
+ return Joiner.on('\n').join(list) + "\n";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
new file mode 100644
index 0000000..a3289cb
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import junit.framework.*;
+import org.apache.hadoop.mapreduce.*;
+
+/**
+ *
+ */
+public class HadoopClassLoaderTest extends TestCase {
+ /** */
+ HadoopClassLoader ldr = new HadoopClassLoader(null);
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testClassLoading() throws Exception {
+ assertNotSame(Test1.class, ldr.loadClass(Test1.class.getName()));
+ assertNotSame(Test2.class, ldr.loadClass(Test2.class.getName()));
+ assertSame(Test3.class, ldr.loadClass(Test3.class.getName()));
+ }
+
+// public void testDependencySearch() {
+// assertTrue(ldr.hasExternalDependencies(Test1.class.getName(), new HashSet<String>()));
+// assertTrue(ldr.hasExternalDependencies(Test2.class.getName(), new HashSet<String>()));
+// }
+
+ /**
+ *
+ */
+ private static class Test1 {
+ /** */
+ Test2 t2;
+
+ /** */
+ Job[][] jobs = new Job[4][4];
+ }
+
+ /**
+ *
+ */
+ private static abstract class Test2 {
+ /** */
+ abstract Test1 t1();
+ }
+
+ /**
+ *
+ */
+ private static class Test3 {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopCommandLineTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopCommandLineTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopCommandLineTest.java
new file mode 100644
index 0000000..33fa358
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopCommandLineTest.java
@@ -0,0 +1,440 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import com.google.common.base.*;
+import org.apache.ignite.*;
+import org.apache.ignite.hadoop.fs.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.testframework.junits.common.*;
+import org.jdk8.backport.*;
+
+import java.io.*;
+import java.nio.file.*;
+import java.util.*;
+
+/**
+ * Test of integration with Hadoop client via command line interface.
+ */
+public class HadoopCommandLineTest extends GridCommonAbstractTest {
+ /** IGFS instance. */
+ private IgfsEx igfs;
+
+ /** */
+ private static final String igfsName = "igfs";
+
+ /** */
+ private static File testWorkDir;
+
+ /** */
+ private static String hadoopHome;
+
+ /** */
+ private static String hiveHome;
+
+ /** */
+ private static File examplesJar;
+
+ /**
+ *
+ * @param path File name.
+ * @param wordCounts Words and counts.
+ * @throws Exception If failed.
+ */
+ private void generateTestFile(File path, Object... wordCounts) throws Exception {
+ List<String> wordsArr = new ArrayList<>();
+
+ //Generating
+ for (int i = 0; i < wordCounts.length; i += 2) {
+ String word = (String) wordCounts[i];
+ int cnt = (Integer) wordCounts[i + 1];
+
+ while (cnt-- > 0)
+ wordsArr.add(word);
+ }
+
+ //Shuffling
+ for (int i = 0; i < wordsArr.size(); i++) {
+ int j = (int)(Math.random() * wordsArr.size());
+
+ Collections.swap(wordsArr, i, j);
+ }
+
+ //Writing file
+ try (PrintWriter writer = new PrintWriter(path)) {
+ int j = 0;
+
+ while (j < wordsArr.size()) {
+ int i = 5 + (int)(Math.random() * 5);
+
+ List<String> subList = wordsArr.subList(j, Math.min(j + i, wordsArr.size()));
+ j += i;
+
+ writer.println(Joiner.on(' ').join(subList));
+ }
+
+ writer.flush();
+ }
+ }
+
+ /**
+ * Generates two data files to join its with Hive.
+ *
+ * @throws FileNotFoundException If failed.
+ */
+ private void generateHiveTestFiles() throws FileNotFoundException {
+ try (PrintWriter writerA = new PrintWriter(new File(testWorkDir, "data-a"));
+ PrintWriter writerB = new PrintWriter(new File(testWorkDir, "data-b"))) {
+ char sep = '\t';
+
+ int idB = 0;
+ int idA = 0;
+ int v = 1000;
+
+ for (int i = 0; i < 1000; i++) {
+ writerA.print(idA++);
+ writerA.print(sep);
+ writerA.println(idB);
+
+ writerB.print(idB++);
+ writerB.print(sep);
+ writerB.println(v += 2);
+
+ writerB.print(idB++);
+ writerB.print(sep);
+ writerB.println(v += 2);
+ }
+
+ writerA.flush();
+ writerB.flush();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ hiveHome = IgniteSystemProperties.getString("HIVE_HOME");
+
+ assertFalse("HIVE_HOME hasn't been set.", F.isEmpty(hiveHome));
+
+ hadoopHome = IgniteSystemProperties.getString("HADOOP_HOME");
+
+ assertFalse("HADOOP_HOME hasn't been set.", F.isEmpty(hadoopHome));
+
+ String mapredHome = hadoopHome + "/share/hadoop/mapreduce";
+
+ File[] fileList = new File(mapredHome).listFiles(new FileFilter() {
+ @Override public boolean accept(File pathname) {
+ return pathname.getName().startsWith("hadoop-mapreduce-examples-") &&
+ pathname.getName().endsWith(".jar");
+ }
+ });
+
+ assertEquals("Invalid hadoop distribution.", 1, fileList.length);
+
+ examplesJar = fileList[0];
+
+ testWorkDir = Files.createTempDirectory("hadoop-cli-test").toFile();
+
+ U.copy(U.resolveIgnitePath("docs/core-site.ignite.xml"), new File(testWorkDir, "core-site.xml"), false);
+
+ File srcFile = U.resolveIgnitePath("docs/mapred-site.ignite.xml");
+ File dstFile = new File(testWorkDir, "mapred-site.xml");
+
+ try (BufferedReader in = new BufferedReader(new FileReader(srcFile));
+ PrintWriter out = new PrintWriter(dstFile)) {
+ String line;
+
+ while ((line = in.readLine()) != null) {
+ if (line.startsWith("</configuration>"))
+ out.println(
+ " <property>\n" +
+ " <name>" + HadoopUtils.JOB_COUNTER_WRITER_PROPERTY + "</name>\n" +
+ " <value>" + IgniteHadoopFileSystemCounterWriter.class.getName() + "</value>\n" +
+ " </property>\n");
+
+ out.println(line);
+ }
+
+ out.flush();
+ }
+
+ generateTestFile(new File(testWorkDir, "test-data"), "red", 100, "green", 200, "blue", 150, "yellow", 50);
+
+ generateHiveTestFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ U.delete(testWorkDir);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ igfs = (IgfsEx) Ignition.start("config/hadoop/default-config.xml").fileSystem(igfsName);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids(true);
+ }
+
+ /**
+ * Creates the process build with appropriate environment to run Hadoop CLI.
+ *
+ * @return Process builder.
+ */
+ private ProcessBuilder createProcessBuilder() {
+ String sep = ":";
+
+ String ggClsPath = HadoopJob.class.getProtectionDomain().getCodeSource().getLocation().getPath() + sep +
+ HadoopJobTracker.class.getProtectionDomain().getCodeSource().getLocation().getPath() + sep +
+ ConcurrentHashMap8.class.getProtectionDomain().getCodeSource().getLocation().getPath();
+
+ ProcessBuilder res = new ProcessBuilder();
+
+ res.environment().put("HADOOP_HOME", hadoopHome);
+ res.environment().put("HADOOP_CLASSPATH", ggClsPath);
+ res.environment().put("HADOOP_CONF_DIR", testWorkDir.getAbsolutePath());
+
+ res.redirectErrorStream(true);
+
+ return res;
+ }
+
+ /**
+ * Waits for process exit and prints the its output.
+ *
+ * @param proc Process.
+ * @return Exit code.
+ * @throws Exception If failed.
+ */
+ private int watchProcess(Process proc) throws Exception {
+ BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+
+ String line;
+
+ while ((line = reader.readLine()) != null)
+ log().info(line);
+
+ return proc.waitFor();
+ }
+
+ /**
+ * Executes Hadoop command line tool.
+ *
+ * @param args Arguments for Hadoop command line tool.
+ * @return Process exit code.
+ * @throws Exception If failed.
+ */
+ private int executeHadoopCmd(String... args) throws Exception {
+ ProcessBuilder procBuilder = createProcessBuilder();
+
+ List<String> cmd = new ArrayList<>();
+
+ cmd.add(hadoopHome + "/bin/hadoop");
+ cmd.addAll(Arrays.asList(args));
+
+ procBuilder.command(cmd);
+
+ log().info("Execute: " + procBuilder.command());
+
+ return watchProcess(procBuilder.start());
+ }
+
+ /**
+ * Executes Hive query.
+ *
+ * @param qry Query.
+ * @return Process exit code.
+ * @throws Exception If failed.
+ */
+ private int executeHiveQuery(String qry) throws Exception {
+ ProcessBuilder procBuilder = createProcessBuilder();
+
+ List<String> cmd = new ArrayList<>();
+
+ procBuilder.command(cmd);
+
+ cmd.add(hiveHome + "/bin/hive");
+
+ cmd.add("--hiveconf");
+ cmd.add("hive.rpc.query.plan=true");
+
+ cmd.add("--hiveconf");
+ cmd.add("javax.jdo.option.ConnectionURL=jdbc:derby:" + testWorkDir.getAbsolutePath() + "/metastore_db;" +
+ "databaseName=metastore_db;create=true");
+
+ cmd.add("-e");
+ cmd.add(qry);
+
+ procBuilder.command(cmd);
+
+ log().info("Execute: " + procBuilder.command());
+
+ return watchProcess(procBuilder.start());
+ }
+
+ /**
+ * Tests Hadoop command line integration.
+ */
+ public void testHadoopCommandLine() throws Exception {
+ assertEquals(0, executeHadoopCmd("fs", "-ls", "/"));
+
+ assertEquals(0, executeHadoopCmd("fs", "-mkdir", "/input"));
+
+ assertEquals(0, executeHadoopCmd("fs", "-put", new File(testWorkDir, "test-data").getAbsolutePath(), "/input"));
+
+ assertTrue(igfs.exists(new IgfsPath("/input/test-data")));
+
+ assertEquals(0, executeHadoopCmd("jar", examplesJar.getAbsolutePath(), "wordcount", "/input", "/output"));
+
+ IgfsPath path = new IgfsPath("/user/" + System.getProperty("user.name") + "/");
+
+ assertTrue(igfs.exists(path));
+
+ IgfsPath jobStatPath = null;
+
+ for (IgfsPath jobPath : igfs.listPaths(path)) {
+ assertNull(jobStatPath);
+
+ jobStatPath = jobPath;
+ }
+
+ File locStatFile = new File(testWorkDir, "performance");
+
+ assertEquals(0, executeHadoopCmd("fs", "-get", jobStatPath.toString() + "/performance", locStatFile.toString()));
+
+ long evtCnt = HadoopTestUtils.simpleCheckJobStatFile(new BufferedReader(new FileReader(locStatFile)));
+
+ assertTrue(evtCnt >= 22); //It's the minimum amount of events for job with combiner.
+
+ assertTrue(igfs.exists(new IgfsPath("/output")));
+
+ BufferedReader in = new BufferedReader(new InputStreamReader(igfs.open(new IgfsPath("/output/part-r-00000"))));
+
+ List<String> res = new ArrayList<>();
+
+ String line;
+
+ while ((line = in.readLine()) != null)
+ res.add(line);
+
+ Collections.sort(res);
+
+ assertEquals("[blue\t150, green\t200, red\t100, yellow\t50]", res.toString());
+ }
+
+ /**
+ * Runs query check result.
+ *
+ * @param expRes Expected result.
+ * @param qry Query.
+ * @throws Exception If failed.
+ */
+ private void checkQuery(String expRes, String qry) throws Exception {
+ assertEquals(0, executeHiveQuery("drop table if exists result"));
+
+ assertEquals(0, executeHiveQuery(
+ "create table result " +
+ "row format delimited fields terminated by ' ' " +
+ "stored as textfile " +
+ "location '/result' as " + qry
+ ));
+
+ IgfsInputStreamAdapter in = igfs.open(new IgfsPath("/result/000000_0"));
+
+ byte[] buf = new byte[(int) in.length()];
+
+ in.read(buf);
+
+ assertEquals(expRes, new String(buf));
+ }
+
+ /**
+ * Tests Hive integration.
+ */
+ public void testHiveCommandLine() throws Exception {
+ assertEquals(0, executeHiveQuery(
+ "create table table_a (" +
+ "id_a int," +
+ "id_b int" +
+ ") " +
+ "row format delimited fields terminated by '\\t'" +
+ "stored as textfile " +
+ "location '/table-a'"
+ ));
+
+ assertEquals(0, executeHadoopCmd("fs", "-put", new File(testWorkDir, "data-a").getAbsolutePath(), "/table-a"));
+
+ assertEquals(0, executeHiveQuery(
+ "create table table_b (" +
+ "id_b int," +
+ "rndv int" +
+ ") " +
+ "row format delimited fields terminated by '\\t'" +
+ "stored as textfile " +
+ "location '/table-b'"
+ ));
+
+ assertEquals(0, executeHadoopCmd("fs", "-put", new File(testWorkDir, "data-b").getAbsolutePath(), "/table-b"));
+
+ checkQuery(
+ "0 0\n" +
+ "1 2\n" +
+ "2 4\n" +
+ "3 6\n" +
+ "4 8\n" +
+ "5 10\n" +
+ "6 12\n" +
+ "7 14\n" +
+ "8 16\n" +
+ "9 18\n",
+ "select * from table_a order by id_a limit 10"
+ );
+
+ checkQuery("2000\n", "select count(id_b) from table_b");
+
+ checkQuery(
+ "250 500 2002\n" +
+ "251 502 2006\n" +
+ "252 504 2010\n" +
+ "253 506 2014\n" +
+ "254 508 2018\n" +
+ "255 510 2022\n" +
+ "256 512 2026\n" +
+ "257 514 2030\n" +
+ "258 516 2034\n" +
+ "259 518 2038\n",
+ "select a.id_a, a.id_b, b.rndv" +
+ " from table_a a" +
+ " inner join table_b b on a.id_b = b.id_b" +
+ " where b.rndv > 2000" +
+ " order by a.id_a limit 10"
+ );
+
+ checkQuery("1000\n", "select count(b.id_b) from table_a a inner join table_b b on a.id_b = b.id_b");
+ }
+}
[26/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolKillJobTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolKillJobTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolKillJobTask.java
deleted file mode 100644
index 384bc23..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolKillJobTask.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.proto;
-
-import org.apache.ignite.*;
-import org.apache.ignite.compute.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-
-import java.util.*;
-
-/**
- * Kill job task.
- */
-public class GridHadoopProtocolKillJobTask extends GridHadoopProtocolTaskAdapter<Boolean> {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** {@inheritDoc} */
- @Override public Boolean run(ComputeJobContext jobCtx, GridHadoop hadoop,
- GridHadoopProtocolTaskArguments args) throws IgniteCheckedException {
- UUID nodeId = UUID.fromString(args.<String>get(0));
- Integer id = args.get(1);
-
- assert nodeId != null;
- assert id != null;
-
- GridHadoopJobId jobId = new GridHadoopJobId(nodeId, id);
-
- return hadoop.kill(jobId);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolNextTaskIdTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolNextTaskIdTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolNextTaskIdTask.java
deleted file mode 100644
index f76f3b6..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolNextTaskIdTask.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.proto;
-
-import org.apache.ignite.compute.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-
-/**
- * Task to get the next job ID.
- */
-public class GridHadoopProtocolNextTaskIdTask extends GridHadoopProtocolTaskAdapter<GridHadoopJobId> {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** {@inheritDoc} */
- @Override public GridHadoopJobId run(ComputeJobContext jobCtx, GridHadoop hadoop,
- GridHadoopProtocolTaskArguments args) {
- return hadoop.nextJobId();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolSubmitJobTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolSubmitJobTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolSubmitJobTask.java
deleted file mode 100644
index c734acd..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolSubmitJobTask.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.proto;
-
-import org.apache.ignite.*;
-import org.apache.ignite.compute.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-
-import java.util.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopJobPhase.*;
-
-/**
- * Submit job task.
- */
-public class GridHadoopProtocolSubmitJobTask extends GridHadoopProtocolTaskAdapter<GridHadoopJobStatus> {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** {@inheritDoc} */
- @Override public GridHadoopJobStatus run(ComputeJobContext jobCtx, GridHadoop hadoop,
- GridHadoopProtocolTaskArguments args) throws IgniteCheckedException {
- UUID nodeId = UUID.fromString(args.<String>get(0));
- Integer id = args.get(1);
- GridHadoopDefaultJobInfo info = args.get(2);
-
- assert nodeId != null;
- assert id != null;
- assert info != null;
-
- GridHadoopJobId jobId = new GridHadoopJobId(nodeId, id);
-
- hadoop.submit(jobId, info);
-
- GridHadoopJobStatus res = hadoop.status(jobId);
-
- if (res == null) // Submission failed.
- res = new GridHadoopJobStatus(jobId, info.jobName(), info.user(), 0, 0, 0, 0, PHASE_CANCELLING, true, 1);
-
- return res;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskAdapter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskAdapter.java
deleted file mode 100644
index 086545c..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskAdapter.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.proto;
-
-import org.apache.ignite.*;
-import org.apache.ignite.cluster.*;
-import org.apache.ignite.compute.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.resources.*;
-import org.jetbrains.annotations.*;
-
-import java.util.*;
-
-/**
- * Hadoop protocol task adapter.
- */
-public abstract class GridHadoopProtocolTaskAdapter<R> implements ComputeTask<GridHadoopProtocolTaskArguments, R> {
- /** {@inheritDoc} */
- @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid,
- @Nullable GridHadoopProtocolTaskArguments arg) {
- return Collections.singletonMap(new Job(arg), subgrid.get(0));
- }
-
- /** {@inheritDoc} */
- @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
- return ComputeJobResultPolicy.REDUCE;
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public R reduce(List<ComputeJobResult> results) {
- if (!F.isEmpty(results)) {
- ComputeJobResult res = results.get(0);
-
- return res.getData();
- }
- else
- return null;
- }
-
- /**
- * Job wrapper.
- */
- private class Job implements ComputeJob {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** */
- @SuppressWarnings("UnusedDeclaration")
- @JobContextResource
- private ComputeJobContext jobCtx;
-
- /** Argument. */
- private final GridHadoopProtocolTaskArguments args;
-
- /**
- * Constructor.
- *
- * @param args Job argument.
- */
- private Job(GridHadoopProtocolTaskArguments args) {
- this.args = args;
- }
-
- /** {@inheritDoc} */
- @Override public void cancel() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public Object execute() {
- try {
- return run(jobCtx, ((IgniteEx)ignite).hadoop(), args);
- }
- catch (IgniteCheckedException e) {
- throw U.convertException(e);
- }
- }
- }
-
- /**
- * Run the task.
- *
- * @param jobCtx Job context.
- * @param hadoop Hadoop facade.
- * @param args Arguments.
- * @return Job result.
- * @throws IgniteCheckedException If failed.
- */
- public abstract R run(ComputeJobContext jobCtx, GridHadoop hadoop, GridHadoopProtocolTaskArguments args)
- throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskArguments.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskArguments.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskArguments.java
deleted file mode 100644
index ae91a52..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskArguments.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.proto;
-
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * Task arguments.
- */
-public class GridHadoopProtocolTaskArguments implements Externalizable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Arguments. */
- private Object[] args;
-
- /**
- * {@link Externalizable} support.
- */
- public GridHadoopProtocolTaskArguments() {
- // No-op.
- }
-
- /**
- * Constructor.
- *
- * @param args Arguments.
- */
- public GridHadoopProtocolTaskArguments(Object... args) {
- this.args = args;
- }
-
- /**
- * @param idx Argument index.
- * @return Argument.
- */
- @SuppressWarnings("unchecked")
- @Nullable public <T> T get(int idx) {
- return (args != null && args.length > idx) ? (T)args[idx] : null;
- }
-
- /**
- * @return Size.
- */
- public int size() {
- return args != null ? args.length : 0;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- U.writeArray(out, args);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- args = U.readArray(in);
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopProtocolTaskArguments.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopClientProtocol.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopClientProtocol.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopClientProtocol.java
new file mode 100644
index 0000000..b454760
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopClientProtocol.java
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.proto;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.ipc.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.protocol.*;
+import org.apache.hadoop.mapreduce.security.token.delegation.*;
+import org.apache.hadoop.mapreduce.v2.*;
+import org.apache.hadoop.mapreduce.v2.jobhistory.*;
+import org.apache.hadoop.security.*;
+import org.apache.hadoop.security.authorize.*;
+import org.apache.hadoop.security.token.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.client.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Hadoop client protocol.
+ */
+public class HadoopClientProtocol implements ClientProtocol {
+ /** Ignite framework name property. */
+ public static final String FRAMEWORK_NAME = "ignite";
+
+ /** Protocol version. */
+ private static final long PROTO_VER = 1L;
+
+ /** Default Ignite system directory. */
+ private static final String SYS_DIR = ".ignite/system";
+
+ /** Configuration. */
+ private final Configuration conf;
+
+ /** Ignite client. */
+ private volatile GridClient cli;
+
+ /** Last received version. */
+ private long lastVer = -1;
+
+ /** Last received status. */
+ private HadoopJobStatus lastStatus;
+
+ /**
+ * Constructor.
+ *
+ * @param conf Configuration.
+ * @param cli Ignite client.
+ */
+ public HadoopClientProtocol(Configuration conf, GridClient cli) {
+ assert cli != null;
+
+ this.conf = conf;
+ this.cli = cli;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JobID getNewJobID() throws IOException, InterruptedException {
+ try {
+ conf.setLong(REQ_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());
+
+ HadoopJobId jobID = cli.compute().execute(HadoopProtocolNextTaskIdTask.class.getName(), null);
+
+ conf.setLong(RESPONSE_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());
+
+ return new JobID(jobID.globalId().toString(), jobID.localId());
+ }
+ catch (GridClientException e) {
+ throw new IOException("Failed to get new job ID.", e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) throws IOException,
+ InterruptedException {
+ try {
+ conf.setLong(JOB_SUBMISSION_START_TS_PROPERTY, U.currentTimeMillis());
+
+ HadoopJobStatus status = cli.compute().execute(HadoopProtocolSubmitJobTask.class.getName(),
+ new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId(), createJobInfo(conf)));
+
+ if (status == null)
+ throw new IOException("Failed to submit job (null status obtained): " + jobId);
+
+ return processStatus(status);
+ }
+ catch (GridClientException | IgniteCheckedException e) {
+ throw new IOException("Failed to submit job.", e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public ClusterMetrics getClusterMetrics() throws IOException, InterruptedException {
+ return new ClusterMetrics(0, 0, 0, 0, 0, 0, 1000, 1000, 1, 100, 0, 0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Cluster.JobTrackerStatus getJobTrackerStatus() throws IOException, InterruptedException {
+ return Cluster.JobTrackerStatus.RUNNING;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTaskTrackerExpiryInterval() throws IOException, InterruptedException {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public AccessControlList getQueueAdmins(String queueName) throws IOException {
+ return new AccessControlList("*");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void killJob(JobID jobId) throws IOException, InterruptedException {
+ try {
+ cli.compute().execute(HadoopProtocolKillJobTask.class.getName(),
+ new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId()));
+ }
+ catch (GridClientException e) {
+ throw new IOException("Failed to kill job: " + jobId, e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setJobPriority(JobID jobid, String priority) throws IOException, InterruptedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean killTask(TaskAttemptID taskId, boolean shouldFail) throws IOException,
+ InterruptedException {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JobStatus getJobStatus(JobID jobId) throws IOException, InterruptedException {
+ try {
+ Long delay = conf.getLong(HadoopJobProperty.JOB_STATUS_POLL_DELAY.propertyName(), -1);
+
+ HadoopProtocolTaskArguments args = delay >= 0 ?
+ new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId(), delay) :
+ new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId());
+
+ HadoopJobStatus status = cli.compute().execute(HadoopProtocolJobStatusTask.class.getName(), args);
+
+ if (status == null)
+ throw new IOException("Job tracker doesn't have any information about the job: " + jobId);
+
+ return processStatus(status);
+ }
+ catch (GridClientException e) {
+ throw new IOException("Failed to get job status: " + jobId, e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Counters getJobCounters(JobID jobId) throws IOException, InterruptedException {
+ try {
+ final HadoopCounters counters = cli.compute().execute(HadoopProtocolJobCountersTask.class.getName(),
+ new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId()));
+
+ if (counters == null)
+ throw new IOException("Job tracker doesn't have any information about the job: " + jobId);
+
+ return new HadoopMapReduceCounters(counters);
+ }
+ catch (GridClientException e) {
+ throw new IOException("Failed to get job counters: " + jobId, e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public TaskReport[] getTaskReports(JobID jobid, TaskType type) throws IOException, InterruptedException {
+ return new TaskReport[0];
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getFilesystemName() throws IOException, InterruptedException {
+ return FileSystem.get(conf).getUri().toString();
+ }
+
+ /** {@inheritDoc} */
+ @Override public JobStatus[] getAllJobs() throws IOException, InterruptedException {
+ return new JobStatus[0];
+ }
+
+ /** {@inheritDoc} */
+ @Override public TaskCompletionEvent[] getTaskCompletionEvents(JobID jobid, int fromEventId, int maxEvents)
+ throws IOException, InterruptedException {
+ return new TaskCompletionEvent[0];
+ }
+
+ /** {@inheritDoc} */
+ @Override public String[] getTaskDiagnostics(TaskAttemptID taskId) throws IOException, InterruptedException {
+ return new String[0];
+ }
+
+ /** {@inheritDoc} */
+ @Override public TaskTrackerInfo[] getActiveTrackers() throws IOException, InterruptedException {
+ return new TaskTrackerInfo[0];
+ }
+
+ /** {@inheritDoc} */
+ @Override public TaskTrackerInfo[] getBlacklistedTrackers() throws IOException, InterruptedException {
+ return new TaskTrackerInfo[0];
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getSystemDir() throws IOException, InterruptedException {
+ Path sysDir = new Path(SYS_DIR);
+
+ return sysDir.toString();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getStagingAreaDir() throws IOException, InterruptedException {
+ String usr = UserGroupInformation.getCurrentUser().getShortUserName();
+
+ return HadoopUtils.stagingAreaDir(conf, usr).toString();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getJobHistoryDir() throws IOException, InterruptedException {
+ return JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
+ }
+
+ /** {@inheritDoc} */
+ @Override public QueueInfo[] getQueues() throws IOException, InterruptedException {
+ return new QueueInfo[0];
+ }
+
+ /** {@inheritDoc} */
+ @Override public QueueInfo getQueue(String queueName) throws IOException, InterruptedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException, InterruptedException {
+ return new QueueAclsInfo[0];
+ }
+
+ /** {@inheritDoc} */
+ @Override public QueueInfo[] getRootQueues() throws IOException, InterruptedException {
+ return new QueueInfo[0];
+ }
+
+ /** {@inheritDoc} */
+ @Override public QueueInfo[] getChildQueues(String queueName) throws IOException, InterruptedException {
+ return new QueueInfo[0];
+ }
+
+ /** {@inheritDoc} */
+ @Override public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer) throws IOException,
+ InterruptedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long renewDelegationToken(Token<DelegationTokenIdentifier> token) throws IOException,
+ InterruptedException {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancelDelegationToken(Token<DelegationTokenIdentifier> token) throws IOException,
+ InterruptedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public LogParams getLogFileParams(JobID jobID, TaskAttemptID taskAttemptID) throws IOException,
+ InterruptedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
+ return PROTO_VER;
+ }
+
+ /** {@inheritDoc} */
+ @Override public ProtocolSignature getProtocolSignature(String protocol, long clientVersion, int clientMethodsHash)
+ throws IOException {
+ return ProtocolSignature.getProtocolSignature(this, protocol, clientVersion, clientMethodsHash);
+ }
+
+ /**
+ * Process received status update.
+ *
+ * @param status Ignite status.
+ * @return Hadoop status.
+ */
+ private JobStatus processStatus(HadoopJobStatus status) {
+ // IMPORTANT! This method will only work in single-threaded environment. It is valid at the moment because
+ // IgniteHadoopClientProtocolProvider creates new instance of this class for every new job and Job class
+ // serializes invocations of submitJob() and getJobStatus() methods. However, if any of these conditions will
+ // change in future and either protocol will serve statuses for several jobs or status update will not be
+ // serialized anymore, then we have to fallback to concurrent approach (e.g. using ConcurrentHashMap).
+ // (vozerov)
+ if (lastVer < status.version()) {
+ lastVer = status.version();
+
+ lastStatus = status;
+ }
+ else
+ assert lastStatus != null;
+
+ return HadoopUtils.status(lastStatus, conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolJobCountersTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolJobCountersTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolJobCountersTask.java
new file mode 100644
index 0000000..ebdda9f
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolJobCountersTask.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.proto;
+
+import org.apache.ignite.*;
+import org.apache.ignite.compute.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
+
+import java.util.*;
+
+/**
+ * Task to get job counters.
+ */
+public class HadoopProtocolJobCountersTask extends HadoopProtocolTaskAdapter<HadoopCounters> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public HadoopCounters run(ComputeJobContext jobCtx, Hadoop hadoop,
+ HadoopProtocolTaskArguments args) throws IgniteCheckedException {
+
+ UUID nodeId = UUID.fromString(args.<String>get(0));
+ Integer id = args.get(1);
+
+ assert nodeId != null;
+ assert id != null;
+
+ return hadoop.counters(new HadoopJobId(nodeId, id));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolJobStatusTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolJobStatusTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolJobStatusTask.java
new file mode 100644
index 0000000..1734562
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolJobStatusTask.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.proto;
+
+import org.apache.ignite.*;
+import org.apache.ignite.compute.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.lang.*;
+
+import java.util.*;
+
+/**
+ * Job status task.
+ */
+public class HadoopProtocolJobStatusTask extends HadoopProtocolTaskAdapter<HadoopJobStatus> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Default poll delay */
+ private static final long DFLT_POLL_DELAY = 100L;
+
+ /** Attribute for held status. */
+ private static final String ATTR_HELD = "held";
+
+ /** {@inheritDoc} */
+ @Override public HadoopJobStatus run(final ComputeJobContext jobCtx, Hadoop hadoop,
+ HadoopProtocolTaskArguments args) throws IgniteCheckedException {
+ UUID nodeId = UUID.fromString(args.<String>get(0));
+ Integer id = args.get(1);
+ Long pollDelay = args.get(2);
+
+ assert nodeId != null;
+ assert id != null;
+
+ HadoopJobId jobId = new HadoopJobId(nodeId, id);
+
+ if (pollDelay == null)
+ pollDelay = DFLT_POLL_DELAY;
+
+ if (pollDelay > 0) {
+ IgniteInternalFuture<?> fut = hadoop.finishFuture(jobId);
+
+ if (fut != null) {
+ if (fut.isDone() || F.eq(jobCtx.getAttribute(ATTR_HELD), true))
+ return hadoop.status(jobId);
+ else {
+ fut.listenAsync(new IgniteInClosure<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> fut0) {
+ jobCtx.callcc();
+ }
+ });
+
+ jobCtx.setAttribute(ATTR_HELD, true);
+
+ return jobCtx.holdcc(pollDelay);
+ }
+ }
+ else
+ return null;
+ }
+ else
+ return hadoop.status(jobId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolKillJobTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolKillJobTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolKillJobTask.java
new file mode 100644
index 0000000..d173612
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolKillJobTask.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.proto;
+
+import org.apache.ignite.*;
+import org.apache.ignite.compute.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+
+import java.util.*;
+
+/**
+ * Kill job task.
+ */
+public class HadoopProtocolKillJobTask extends HadoopProtocolTaskAdapter<Boolean> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public Boolean run(ComputeJobContext jobCtx, Hadoop hadoop,
+ HadoopProtocolTaskArguments args) throws IgniteCheckedException {
+ UUID nodeId = UUID.fromString(args.<String>get(0));
+ Integer id = args.get(1);
+
+ assert nodeId != null;
+ assert id != null;
+
+ HadoopJobId jobId = new HadoopJobId(nodeId, id);
+
+ return hadoop.kill(jobId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolNextTaskIdTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolNextTaskIdTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolNextTaskIdTask.java
new file mode 100644
index 0000000..2782530
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolNextTaskIdTask.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.proto;
+
+import org.apache.ignite.compute.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+
+/**
+ * Task to get the next job ID.
+ */
+public class HadoopProtocolNextTaskIdTask extends HadoopProtocolTaskAdapter<HadoopJobId> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public HadoopJobId run(ComputeJobContext jobCtx, Hadoop hadoop,
+ HadoopProtocolTaskArguments args) {
+ return hadoop.nextJobId();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolSubmitJobTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolSubmitJobTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolSubmitJobTask.java
new file mode 100644
index 0000000..f65d9bb
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolSubmitJobTask.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.proto;
+
+import org.apache.ignite.*;
+import org.apache.ignite.compute.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobPhase.*;
+
+/**
+ * Submit job task.
+ */
+public class HadoopProtocolSubmitJobTask extends HadoopProtocolTaskAdapter<HadoopJobStatus> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public HadoopJobStatus run(ComputeJobContext jobCtx, Hadoop hadoop,
+ HadoopProtocolTaskArguments args) throws IgniteCheckedException {
+ UUID nodeId = UUID.fromString(args.<String>get(0));
+ Integer id = args.get(1);
+ HadoopDefaultJobInfo info = args.get(2);
+
+ assert nodeId != null;
+ assert id != null;
+ assert info != null;
+
+ HadoopJobId jobId = new HadoopJobId(nodeId, id);
+
+ hadoop.submit(jobId, info);
+
+ HadoopJobStatus res = hadoop.status(jobId);
+
+ if (res == null) // Submission failed.
+ res = new HadoopJobStatus(jobId, info.jobName(), info.user(), 0, 0, 0, 0, PHASE_CANCELLING, true, 1);
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolTaskAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolTaskAdapter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolTaskAdapter.java
new file mode 100644
index 0000000..f763ccc
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolTaskAdapter.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.proto;
+
+import org.apache.ignite.*;
+import org.apache.ignite.cluster.*;
+import org.apache.ignite.compute.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.resources.*;
+import org.jetbrains.annotations.*;
+
+import java.util.*;
+
+/**
+ * Hadoop protocol task adapter.
+ */
+public abstract class HadoopProtocolTaskAdapter<R> implements ComputeTask<HadoopProtocolTaskArguments, R> {
+ /** {@inheritDoc} */
+ @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid,
+ @Nullable HadoopProtocolTaskArguments arg) {
+ return Collections.singletonMap(new Job(arg), subgrid.get(0));
+ }
+
+ /** {@inheritDoc} */
+ @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
+ return ComputeJobResultPolicy.REDUCE;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public R reduce(List<ComputeJobResult> results) {
+ if (!F.isEmpty(results)) {
+ ComputeJobResult res = results.get(0);
+
+ return res.getData();
+ }
+ else
+ return null;
+ }
+
+ /**
+ * Job wrapper.
+ */
+ private class Job implements ComputeJob {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** */
+ @SuppressWarnings("UnusedDeclaration")
+ @JobContextResource
+ private ComputeJobContext jobCtx;
+
+ /** Argument. */
+ private final HadoopProtocolTaskArguments args;
+
+ /**
+ * Constructor.
+ *
+ * @param args Job argument.
+ */
+ private Job(HadoopProtocolTaskArguments args) {
+ this.args = args;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancel() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Object execute() {
+ try {
+ return run(jobCtx, ((IgniteEx)ignite).hadoop(), args);
+ }
+ catch (IgniteCheckedException e) {
+ throw U.convertException(e);
+ }
+ }
+ }
+
+ /**
+ * Run the task.
+ *
+ * @param jobCtx Job context.
+ * @param hadoop Hadoop facade.
+ * @param args Arguments.
+ * @return Job result.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract R run(ComputeJobContext jobCtx, Hadoop hadoop, HadoopProtocolTaskArguments args)
+ throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolTaskArguments.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolTaskArguments.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolTaskArguments.java
new file mode 100644
index 0000000..5c470ba
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/HadoopProtocolTaskArguments.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.proto;
+
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * Task arguments.
+ */
+public class HadoopProtocolTaskArguments implements Externalizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Arguments. */
+ private Object[] args;
+
+ /**
+ * {@link Externalizable} support.
+ */
+ public HadoopProtocolTaskArguments() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param args Arguments.
+ */
+ public HadoopProtocolTaskArguments(Object... args) {
+ this.args = args;
+ }
+
+ /**
+ * @param idx Argument index.
+ * @return Argument.
+ */
+ @SuppressWarnings("unchecked")
+ @Nullable public <T> T get(int idx) {
+ return (args != null && args.length > idx) ? (T)args[idx] : null;
+ }
+
+ /**
+ * @return Size.
+ */
+ public int size() {
+ return args != null ? args.length : 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ U.writeArray(out, args);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ args = U.readArray(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopProtocolTaskArguments.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffle.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffle.java
deleted file mode 100644
index 396124e..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffle.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle;
-
-import org.apache.ignite.*;
-import org.apache.ignite.cluster.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.util.future.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.lang.*;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-/**
- * Shuffle.
- */
-public class GridHadoopShuffle extends GridHadoopComponent {
- /** */
- private final ConcurrentMap<GridHadoopJobId, GridHadoopShuffleJob<UUID>> jobs = new ConcurrentHashMap<>();
-
- /** */
- protected final GridUnsafeMemory mem = new GridUnsafeMemory(0);
-
- /** {@inheritDoc} */
- @Override public void start(GridHadoopContext ctx) throws IgniteCheckedException {
- super.start(ctx);
-
- ctx.kernalContext().io().addUserMessageListener(GridTopic.TOPIC_HADOOP,
- new IgniteBiPredicate<UUID, Object>() {
- @Override public boolean apply(UUID nodeId, Object msg) {
- return onMessageReceived(nodeId, (GridHadoopMessage)msg);
- }
- });
- }
-
- /**
- * Stops shuffle.
- *
- * @param cancel If should cancel all ongoing activities.
- */
- @Override public void stop(boolean cancel) {
- for (GridHadoopShuffleJob job : jobs.values()) {
- try {
- job.close();
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to close job.", e);
- }
- }
-
- jobs.clear();
- }
-
- /**
- * Creates new shuffle job.
- *
- * @param jobId Job ID.
- * @return Created shuffle job.
- * @throws IgniteCheckedException If job creation failed.
- */
- private GridHadoopShuffleJob<UUID> newJob(GridHadoopJobId jobId) throws IgniteCheckedException {
- GridHadoopMapReducePlan plan = ctx.jobTracker().plan(jobId);
-
- GridHadoopShuffleJob<UUID> job = new GridHadoopShuffleJob<>(ctx.localNodeId(), log,
- ctx.jobTracker().job(jobId, null), mem, plan.reducers(), plan.reducers(ctx.localNodeId()));
-
- UUID[] rdcAddrs = new UUID[plan.reducers()];
-
- for (int i = 0; i < rdcAddrs.length; i++) {
- UUID nodeId = plan.nodeForReducer(i);
-
- assert nodeId != null : "Plan is missing node for reducer [plan=" + plan + ", rdc=" + i + ']';
-
- rdcAddrs[i] = nodeId;
- }
-
- boolean init = job.initializeReduceAddresses(rdcAddrs);
-
- assert init;
-
- return job;
- }
-
- /**
- * @param nodeId Node ID to send message to.
- * @param msg Message to send.
- * @throws IgniteCheckedException If send failed.
- */
- private void send0(UUID nodeId, Object msg) throws IgniteCheckedException {
- ClusterNode node = ctx.kernalContext().discovery().node(nodeId);
-
- ctx.kernalContext().io().sendUserMessage(F.asList(node), msg, GridTopic.TOPIC_HADOOP, false, 0);
- }
-
- /**
- * @param jobId Task info.
- * @return Shuffle job.
- */
- private GridHadoopShuffleJob<UUID> job(GridHadoopJobId jobId) throws IgniteCheckedException {
- GridHadoopShuffleJob<UUID> res = jobs.get(jobId);
-
- if (res == null) {
- res = newJob(jobId);
-
- GridHadoopShuffleJob<UUID> old = jobs.putIfAbsent(jobId, res);
-
- if (old != null) {
- res.close();
-
- res = old;
- }
- else if (res.reducersInitialized())
- startSending(res);
- }
-
- return res;
- }
-
- /**
- * Starts message sending thread.
- *
- * @param shuffleJob Job to start sending for.
- */
- private void startSending(GridHadoopShuffleJob<UUID> shuffleJob) {
- shuffleJob.startSending(ctx.kernalContext().gridName(),
- new IgniteInClosure2X<UUID, GridHadoopShuffleMessage>() {
- @Override public void applyx(UUID dest, GridHadoopShuffleMessage msg) throws IgniteCheckedException {
- send0(dest, msg);
- }
- }
- );
- }
-
- /**
- * Message received callback.
- *
- * @param src Sender node ID.
- * @param msg Received message.
- * @return {@code True}.
- */
- public boolean onMessageReceived(UUID src, GridHadoopMessage msg) {
- if (msg instanceof GridHadoopShuffleMessage) {
- GridHadoopShuffleMessage m = (GridHadoopShuffleMessage)msg;
-
- try {
- job(m.jobId()).onShuffleMessage(m);
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Message handling failed.", e);
- }
-
- try {
- // Reply with ack.
- send0(src, new GridHadoopShuffleAck(m.id(), m.jobId()));
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to reply back to shuffle message sender [snd=" + src + ", msg=" + msg + ']', e);
- }
- }
- else if (msg instanceof GridHadoopShuffleAck) {
- GridHadoopShuffleAck m = (GridHadoopShuffleAck)msg;
-
- try {
- job(m.jobId()).onShuffleAck(m);
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Message handling failed.", e);
- }
- }
- else
- throw new IllegalStateException("Unknown message type received to Hadoop shuffle [src=" + src +
- ", msg=" + msg + ']');
-
- return true;
- }
-
- /**
- * @param taskCtx Task info.
- * @return Output.
- */
- public GridHadoopTaskOutput output(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- return job(taskCtx.taskInfo().jobId()).output(taskCtx);
- }
-
- /**
- * @param taskCtx Task info.
- * @return Input.
- */
- public GridHadoopTaskInput input(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- return job(taskCtx.taskInfo().jobId()).input(taskCtx);
- }
-
- /**
- * @param jobId Job id.
- */
- public void jobFinished(GridHadoopJobId jobId) {
- GridHadoopShuffleJob job = jobs.remove(jobId);
-
- if (job != null) {
- try {
- job.close();
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to close job: " + jobId, e);
- }
- }
- }
-
- /**
- * Flushes all the outputs for the given job to remote nodes.
- *
- * @param jobId Job ID.
- * @return Future.
- */
- public IgniteInternalFuture<?> flush(GridHadoopJobId jobId) {
- GridHadoopShuffleJob job = jobs.get(jobId);
-
- if (job == null)
- return new GridFinishedFutureEx<>();
-
- try {
- return job.flush();
- }
- catch (IgniteCheckedException e) {
- return new GridFinishedFutureEx<>(e);
- }
- }
-
- /**
- * @return Memory.
- */
- public GridUnsafeMemory memory() {
- return mem;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleAck.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleAck.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleAck.java
deleted file mode 100644
index a8a52a9..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleAck.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.util.tostring.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-
-/**
- * Acknowledgement message.
- */
-public class GridHadoopShuffleAck implements GridHadoopMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- @GridToStringInclude
- private long msgId;
-
- /** */
- @GridToStringInclude
- private GridHadoopJobId jobId;
-
- /**
- *
- */
- public GridHadoopShuffleAck() {
- // No-op.
- }
-
- /**
- * @param msgId Message ID.
- */
- public GridHadoopShuffleAck(long msgId, GridHadoopJobId jobId) {
- assert jobId != null;
-
- this.msgId = msgId;
- this.jobId = jobId;
- }
-
- /**
- * @return Message ID.
- */
- public long id() {
- return msgId;
- }
-
- /**
- * @return Job ID.
- */
- public GridHadoopJobId jobId() {
- return jobId;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- jobId.writeExternal(out);
- out.writeLong(msgId);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- jobId = new GridHadoopJobId();
-
- jobId.readExternal(in);
- msgId = in.readLong();
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopShuffleAck.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleJob.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleJob.java
deleted file mode 100644
index 545c1b8..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleJob.java
+++ /dev/null
@@ -1,593 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.counter.*;
-import org.apache.ignite.internal.processors.hadoop.shuffle.collections.*;
-import org.apache.ignite.internal.util.future.*;
-import org.apache.ignite.internal.util.io.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.internal.util.worker.*;
-import org.apache.ignite.lang.*;
-import org.apache.ignite.thread.*;
-
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopJobProperty.*;
-import static org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory.*;
-
-/**
- * Shuffle job.
- */
-public class GridHadoopShuffleJob<T> implements AutoCloseable {
- /** */
- private static final int MSG_BUF_SIZE = 128 * 1024;
-
- /** */
- private final GridHadoopJob job;
-
- /** */
- private final GridUnsafeMemory mem;
-
- /** */
- private final boolean needPartitioner;
-
- /** Collection of task contexts for each reduce task. */
- private final Map<Integer, GridHadoopTaskContext> reducersCtx = new HashMap<>();
-
- /** Reducers addresses. */
- private T[] reduceAddrs;
-
- /** Local reducers address. */
- private final T locReduceAddr;
-
- /** */
- private final GridHadoopShuffleMessage[] msgs;
-
- /** */
- private final AtomicReferenceArray<GridHadoopMultimap> maps;
-
- /** */
- private volatile IgniteInClosure2X<T, GridHadoopShuffleMessage> io;
-
- /** */
- protected ConcurrentMap<Long, IgniteBiTuple<GridHadoopShuffleMessage, GridFutureAdapterEx<?>>> sentMsgs =
- new ConcurrentHashMap<>();
-
- /** */
- private volatile GridWorker snd;
-
- /** Latch for remote addresses waiting. */
- private final CountDownLatch ioInitLatch = new CountDownLatch(1);
-
- /** Finished flag. Set on flush or close. */
- private volatile boolean flushed;
-
- /** */
- private final IgniteLogger log;
-
- /**
- * @param locReduceAddr Local reducer address.
- * @param log Logger.
- * @param job Job.
- * @param mem Memory.
- * @param totalReducerCnt Amount of reducers in the Job.
- * @param locReducers Reducers will work on current node.
- * @throws IgniteCheckedException If error.
- */
- public GridHadoopShuffleJob(T locReduceAddr, IgniteLogger log, GridHadoopJob job, GridUnsafeMemory mem,
- int totalReducerCnt, int[] locReducers) throws IgniteCheckedException {
- this.locReduceAddr = locReduceAddr;
- this.job = job;
- this.mem = mem;
- this.log = log.getLogger(GridHadoopShuffleJob.class);
-
- if (!F.isEmpty(locReducers)) {
- for (int rdc : locReducers) {
- GridHadoopTaskInfo taskInfo = new GridHadoopTaskInfo(GridHadoopTaskType.REDUCE, job.id(), rdc, 0, null);
-
- reducersCtx.put(rdc, job.getTaskContext(taskInfo));
- }
- }
-
- needPartitioner = totalReducerCnt > 1;
-
- maps = new AtomicReferenceArray<>(totalReducerCnt);
- msgs = new GridHadoopShuffleMessage[totalReducerCnt];
- }
-
- /**
- * @param reduceAddrs Addresses of reducers.
- * @return {@code True} if addresses were initialized by this call.
- */
- public boolean initializeReduceAddresses(T[] reduceAddrs) {
- if (this.reduceAddrs == null) {
- this.reduceAddrs = reduceAddrs;
-
- return true;
- }
-
- return false;
- }
-
- /**
- * @return {@code True} if reducers addresses were initialized.
- */
- public boolean reducersInitialized() {
- return reduceAddrs != null;
- }
-
- /**
- * @param gridName Grid name.
- * @param io IO Closure for sending messages.
- */
- @SuppressWarnings("BusyWait")
- public void startSending(String gridName, IgniteInClosure2X<T, GridHadoopShuffleMessage> io) {
- assert snd == null;
- assert io != null;
-
- this.io = io;
-
- if (!flushed) {
- snd = new GridWorker(gridName, "hadoop-shuffle-" + job.id(), log) {
- @Override protected void body() throws InterruptedException {
- try {
- while (!isCancelled()) {
- Thread.sleep(5);
-
- collectUpdatesAndSend(false);
- }
- }
- catch (IgniteCheckedException e) {
- throw new IllegalStateException(e);
- }
- }
- };
-
- new IgniteThread(snd).start();
- }
-
- ioInitLatch.countDown();
- }
-
- /**
- * @param maps Maps.
- * @param idx Index.
- * @return Map.
- */
- private GridHadoopMultimap getOrCreateMap(AtomicReferenceArray<GridHadoopMultimap> maps, int idx) {
- GridHadoopMultimap map = maps.get(idx);
-
- if (map == null) { // Create new map.
- map = get(job.info(), SHUFFLE_REDUCER_NO_SORTING, false) ?
- new GridHadoopConcurrentHashMultimap(job.info(), mem, get(job.info(), PARTITION_HASHMAP_SIZE, 8 * 1024)):
- new GridHadoopSkipList(job.info(), mem);
-
- if (!maps.compareAndSet(idx, null, map)) {
- map.close();
-
- return maps.get(idx);
- }
- }
-
- return map;
- }
-
- /**
- * @param msg Message.
- * @throws IgniteCheckedException Exception.
- */
- public void onShuffleMessage(GridHadoopShuffleMessage msg) throws IgniteCheckedException {
- assert msg.buffer() != null;
- assert msg.offset() > 0;
-
- GridHadoopTaskContext taskCtx = reducersCtx.get(msg.reducer());
-
- GridHadoopPerformanceCounter perfCntr = GridHadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
-
- perfCntr.onShuffleMessage(msg.reducer(), U.currentTimeMillis());
-
- GridHadoopMultimap map = getOrCreateMap(maps, msg.reducer());
-
- // Add data from message to the map.
- try (GridHadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
- final GridUnsafeDataInput dataInput = new GridUnsafeDataInput();
- final UnsafeValue val = new UnsafeValue(msg.buffer());
-
- msg.visit(new GridHadoopShuffleMessage.Visitor() {
- /** */
- private GridHadoopMultimap.Key key;
-
- @Override public void onKey(byte[] buf, int off, int len) throws IgniteCheckedException {
- dataInput.bytes(buf, off, off + len);
-
- key = adder.addKey(dataInput, key);
- }
-
- @Override public void onValue(byte[] buf, int off, int len) {
- val.off = off;
- val.size = len;
-
- key.add(val);
- }
- });
- }
- }
-
- /**
- * @param ack Shuffle ack.
- */
- @SuppressWarnings("ConstantConditions")
- public void onShuffleAck(GridHadoopShuffleAck ack) {
- IgniteBiTuple<GridHadoopShuffleMessage, GridFutureAdapterEx<?>> tup = sentMsgs.get(ack.id());
-
- if (tup != null)
- tup.get2().onDone();
- else
- log.warning("Received shuffle ack for not registered shuffle id: " + ack);
- }
-
- /**
- * Unsafe value.
- */
- private static class UnsafeValue implements GridHadoopMultimap.Value {
- /** */
- private final byte[] buf;
-
- /** */
- private int off;
-
- /** */
- private int size;
-
- /**
- * @param buf Buffer.
- */
- private UnsafeValue(byte[] buf) {
- assert buf != null;
-
- this.buf = buf;
- }
-
- /** */
- @Override public int size() {
- return size;
- }
-
- /** */
- @Override public void copyTo(long ptr) {
- UNSAFE.copyMemory(buf, BYTE_ARR_OFF + off, null, ptr, size);
- }
- }
-
- /**
- * Sends map updates to remote reducers.
- */
- private void collectUpdatesAndSend(boolean flush) throws IgniteCheckedException {
- for (int i = 0; i < maps.length(); i++) {
- GridHadoopMultimap map = maps.get(i);
-
- if (map == null || locReduceAddr.equals(reduceAddrs[i]))
- continue; // Skip empty map and local node.
-
- if (msgs[i] == null)
- msgs[i] = new GridHadoopShuffleMessage(job.id(), i, MSG_BUF_SIZE);
-
- final int idx = i;
-
- map.visit(false, new GridHadoopMultimap.Visitor() {
- /** */
- private long keyPtr;
-
- /** */
- private int keySize;
-
- /** */
- private boolean keyAdded;
-
- /** {@inheritDoc} */
- @Override public void onKey(long keyPtr, int keySize) {
- this.keyPtr = keyPtr;
- this.keySize = keySize;
-
- keyAdded = false;
- }
-
- private boolean tryAdd(long valPtr, int valSize) {
- GridHadoopShuffleMessage msg = msgs[idx];
-
- if (!keyAdded) { // Add key and value.
- int size = keySize + valSize;
-
- if (!msg.available(size, false))
- return false;
-
- msg.addKey(keyPtr, keySize);
- msg.addValue(valPtr, valSize);
-
- keyAdded = true;
-
- return true;
- }
-
- if (!msg.available(valSize, true))
- return false;
-
- msg.addValue(valPtr, valSize);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public void onValue(long valPtr, int valSize) {
- if (tryAdd(valPtr, valSize))
- return;
-
- send(idx, keySize + valSize);
-
- keyAdded = false;
-
- if (!tryAdd(valPtr, valSize))
- throw new IllegalStateException();
- }
- });
-
- if (flush && msgs[i].offset() != 0)
- send(i, 0);
- }
- }
-
- /**
- * @param idx Index of message.
- * @param newBufMinSize Min new buffer size.
- */
- private void send(final int idx, int newBufMinSize) {
- final GridFutureAdapterEx<?> fut = new GridFutureAdapterEx<>();
-
- GridHadoopShuffleMessage msg = msgs[idx];
-
- final long msgId = msg.id();
-
- IgniteBiTuple<GridHadoopShuffleMessage, GridFutureAdapterEx<?>> old = sentMsgs.putIfAbsent(msgId,
- new IgniteBiTuple<GridHadoopShuffleMessage, GridFutureAdapterEx<?>>(msg, fut));
-
- assert old == null;
-
- try {
- io.apply(reduceAddrs[idx], msg);
- }
- catch (GridClosureException e) {
- fut.onDone(U.unwrap(e));
- }
-
- fut.listenAsync(new IgniteInClosure<IgniteInternalFuture<?>>() {
- @Override public void apply(IgniteInternalFuture<?> f) {
- try {
- f.get();
-
- // Clean up the future from map only if there was no exception.
- // Otherwise flush() should fail.
- sentMsgs.remove(msgId);
- }
- catch (IgniteCheckedException e) {
- log.error("Failed to send message.", e);
- }
- }
- });
-
- msgs[idx] = newBufMinSize == 0 ? null : new GridHadoopShuffleMessage(job.id(), idx,
- Math.max(MSG_BUF_SIZE, newBufMinSize));
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteCheckedException {
- if (snd != null) {
- snd.cancel();
-
- try {
- snd.join();
- }
- catch (InterruptedException e) {
- throw new IgniteInterruptedCheckedException(e);
- }
- }
-
- close(maps);
- }
-
- /**
- * @param maps Maps.
- */
- private void close(AtomicReferenceArray<GridHadoopMultimap> maps) {
- for (int i = 0; i < maps.length(); i++) {
- GridHadoopMultimap map = maps.get(i);
-
- if (map != null)
- map.close();
- }
- }
-
- /**
- * @return Future.
- */
- @SuppressWarnings("unchecked")
- public IgniteInternalFuture<?> flush() throws IgniteCheckedException {
- if (log.isDebugEnabled())
- log.debug("Flushing job " + job.id() + " on address " + locReduceAddr);
-
- flushed = true;
-
- if (maps.length() == 0)
- return new GridFinishedFutureEx<>();
-
- U.await(ioInitLatch);
-
- GridWorker snd0 = snd;
-
- if (snd0 != null) {
- if (log.isDebugEnabled())
- log.debug("Cancelling sender thread.");
-
- snd0.cancel();
-
- try {
- snd0.join();
-
- if (log.isDebugEnabled())
- log.debug("Finished waiting for sending thread to complete on shuffle job flush: " + job.id());
- }
- catch (InterruptedException e) {
- throw new IgniteInterruptedCheckedException(e);
- }
- }
-
- collectUpdatesAndSend(true); // With flush.
-
- if (log.isDebugEnabled())
- log.debug("Finished sending collected updates to remote reducers: " + job.id());
-
- GridCompoundFuture fut = new GridCompoundFuture<>();
-
- for (IgniteBiTuple<GridHadoopShuffleMessage, GridFutureAdapterEx<?>> tup : sentMsgs.values())
- fut.add(tup.get2());
-
- fut.markInitialized();
-
- if (log.isDebugEnabled())
- log.debug("Collected futures to compound futures for flush: " + sentMsgs.size());
-
- return fut;
- }
-
- /**
- * @param taskCtx Task context.
- * @return Output.
- * @throws IgniteCheckedException If failed.
- */
- public GridHadoopTaskOutput output(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- switch (taskCtx.taskInfo().type()) {
- case MAP:
- assert !job.info().hasCombiner() : "The output creation is allowed if combiner has not been defined.";
-
- case COMBINE:
- return new PartitionedOutput(taskCtx);
-
- default:
- throw new IllegalStateException("Illegal type: " + taskCtx.taskInfo().type());
- }
- }
-
- /**
- * @param taskCtx Task context.
- * @return Input.
- * @throws IgniteCheckedException If failed.
- */
- @SuppressWarnings("unchecked")
- public GridHadoopTaskInput input(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- switch (taskCtx.taskInfo().type()) {
- case REDUCE:
- int reducer = taskCtx.taskInfo().taskNumber();
-
- GridHadoopMultimap m = maps.get(reducer);
-
- if (m != null)
- return m.input(taskCtx);
-
- return new GridHadoopTaskInput() { // Empty input.
- @Override public boolean next() {
- return false;
- }
-
- @Override public Object key() {
- throw new IllegalStateException();
- }
-
- @Override public Iterator<?> values() {
- throw new IllegalStateException();
- }
-
- @Override public void close() {
- // No-op.
- }
- };
-
- default:
- throw new IllegalStateException("Illegal type: " + taskCtx.taskInfo().type());
- }
- }
-
- /**
- * Partitioned output.
- */
- private class PartitionedOutput implements GridHadoopTaskOutput {
- /** */
- private final GridHadoopTaskOutput[] adders = new GridHadoopTaskOutput[maps.length()];
-
- /** */
- private GridHadoopPartitioner partitioner;
-
- /** */
- private final GridHadoopTaskContext taskCtx;
-
- /**
- * Constructor.
- * @param taskCtx Task context.
- */
- private PartitionedOutput(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- this.taskCtx = taskCtx;
-
- if (needPartitioner)
- partitioner = taskCtx.partitioner();
- }
-
- /** {@inheritDoc} */
- @Override public void write(Object key, Object val) throws IgniteCheckedException {
- int part = 0;
-
- if (partitioner != null) {
- part = partitioner.partition(key, val, adders.length);
-
- if (part < 0 || part >= adders.length)
- throw new IgniteCheckedException("Invalid partition: " + part);
- }
-
- GridHadoopTaskOutput out = adders[part];
-
- if (out == null)
- adders[part] = out = getOrCreateMap(maps, part).startAdding(taskCtx);
-
- out.write(key, val);
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteCheckedException {
- for (GridHadoopTaskOutput adder : adders) {
- if (adder != null)
- adder.close();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleMessage.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleMessage.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleMessage.java
deleted file mode 100644
index 24ebc0c..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleMessage.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.tostring.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.util.concurrent.atomic.*;
-
-import static org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory.*;
-
-/**
- * Shuffle message.
- */
-public class GridHadoopShuffleMessage implements GridHadoopMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private static final AtomicLong ids = new AtomicLong();
-
- /** */
- private static final byte MARKER_KEY = (byte)17;
-
- /** */
- private static final byte MARKER_VALUE = (byte)31;
-
- /** */
- @GridToStringInclude
- private long msgId;
-
- /** */
- @GridToStringInclude
- private GridHadoopJobId jobId;
-
- /** */
- @GridToStringInclude
- private int reducer;
-
- /** */
- private byte[] buf;
-
- /** */
- @GridToStringInclude
- private int off;
-
- /**
- *
- */
- public GridHadoopShuffleMessage() {
- // No-op.
- }
-
- /**
- * @param size Size.
- */
- public GridHadoopShuffleMessage(GridHadoopJobId jobId, int reducer, int size) {
- assert jobId != null;
-
- buf = new byte[size];
-
- this.jobId = jobId;
- this.reducer = reducer;
-
- msgId = ids.incrementAndGet();
- }
-
- /**
- * @return Message ID.
- */
- public long id() {
- return msgId;
- }
-
- /**
- * @return Job ID.
- */
- public GridHadoopJobId jobId() {
- return jobId;
- }
-
- /**
- * @return Reducer.
- */
- public int reducer() {
- return reducer;
- }
-
- /**
- * @return Buffer.
- */
- public byte[] buffer() {
- return buf;
- }
-
- /**
- * @return Offset.
- */
- public int offset() {
- return off;
- }
-
- /**
- * @param size Size.
- * @param valOnly Only value wll be added.
- * @return {@code true} If this message can fit additional data of this size
- */
- public boolean available(int size, boolean valOnly) {
- size += valOnly ? 5 : 10;
-
- if (off + size > buf.length) {
- if (off == 0) { // Resize if requested size is too big.
- buf = new byte[size];
-
- return true;
- }
-
- return false;
- }
-
- return true;
- }
-
- /**
- * @param keyPtr Key pointer.
- * @param keySize Key size.
- */
- public void addKey(long keyPtr, int keySize) {
- add(MARKER_KEY, keyPtr, keySize);
- }
-
- /**
- * @param valPtr Value pointer.
- * @param valSize Value size.
- */
- public void addValue(long valPtr, int valSize) {
- add(MARKER_VALUE, valPtr, valSize);
- }
-
- /**
- * @param marker Marker.
- * @param ptr Pointer.
- * @param size Size.
- */
- private void add(byte marker, long ptr, int size) {
- buf[off++] = marker;
-
- UNSAFE.putInt(buf, BYTE_ARR_OFF + off, size);
-
- off += 4;
-
- UNSAFE.copyMemory(null, ptr, buf, BYTE_ARR_OFF + off, size);
-
- off += size;
- }
-
- /**
- * @param v Visitor.
- */
- public void visit(Visitor v) throws IgniteCheckedException {
- for (int i = 0; i < off;) {
- byte marker = buf[i++];
-
- int size = UNSAFE.getInt(buf, BYTE_ARR_OFF + i);
-
- i += 4;
-
- if (marker == MARKER_VALUE)
- v.onValue(buf, i, size);
- else if (marker == MARKER_KEY)
- v.onKey(buf, i, size);
- else
- throw new IllegalStateException();
-
- i += size;
- }
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- jobId.writeExternal(out);
- out.writeLong(msgId);
- out.writeInt(reducer);
- out.writeInt(off);
- U.writeByteArray(out, buf);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- jobId = new GridHadoopJobId();
-
- jobId.readExternal(in);
- msgId = in.readLong();
- reducer = in.readInt();
- off = in.readInt();
- buf = U.readByteArray(in);
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopShuffleMessage.class, this);
- }
-
- /**
- * Visitor.
- */
- public static interface Visitor {
- /**
- * @param buf Buffer.
- * @param off Offset.
- * @param len Length.
- */
- public void onKey(byte[] buf, int off, int len) throws IgniteCheckedException;
-
- /**
- * @param buf Buffer.
- * @param off Offset.
- * @param len Length.
- */
- public void onValue(byte[] buf, int off, int len) throws IgniteCheckedException;
- }
-}
[10/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemHandshakeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemHandshakeSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemHandshakeSelfTest.java
deleted file mode 100644
index 137db6d..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemHandshakeSelfTest.java
+++ /dev/null
@@ -1,311 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.*;
-import org.apache.ignite.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.spi.communication.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
-import org.apache.ignite.testframework.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.cache.CacheDistributionMode.*;
-import static org.apache.ignite.cache.CacheMode.*;
-import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*;
-import static org.apache.ignite.igfs.IgfsMode.*;
-import static org.apache.ignite.internal.igfs.hadoop.IgfsHadoopUtils.*;
-import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
-
-/**
- * Tests for IGFS file system handshake.
- */
-public class IgfsHadoopFileSystemHandshakeSelfTest extends IgfsCommonAbstractTest {
- /** IP finder. */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** Grid name. */
- private static final String GRID_NAME = "grid";
-
- /** IGFS name. */
- private static final String IGFS_NAME = "igfs";
-
- /** IGFS path. */
- private static final IgfsPath PATH = new IgfsPath("/path");
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids(true);
- }
-
- /**
- * Tests for Grid and IGFS having normal names.
- *
- * @throws Exception If failed.
- */
- public void testHandshake() throws Exception {
- startUp(false, false);
-
- checkValid(IGFS_NAME + ":" + GRID_NAME + "@");
- checkValid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1");
- checkValid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
-
- checkInvalid(IGFS_NAME + "@");
- checkInvalid(IGFS_NAME + "@127.0.0.1");
- checkInvalid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
-
- checkInvalid(":" + GRID_NAME + "@");
- checkInvalid(":" + GRID_NAME + "@127.0.0.1");
- checkInvalid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
-
- checkInvalid("");
- checkInvalid("127.0.0.1");
- checkInvalid("127.0.0.1:" + DFLT_IPC_PORT);
- }
-
- /**
- * Tests for Grid having {@code null} name and IGFS having normal name.
- *
- * @throws Exception If failed.
- */
- public void testHandshakeDefaultGrid() throws Exception {
- startUp(true, false);
-
- checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@");
- checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1");
- checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
-
- checkValid(IGFS_NAME + "@");
- checkValid(IGFS_NAME + "@127.0.0.1");
- checkValid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
-
- checkInvalid(":" + GRID_NAME + "@");
- checkInvalid(":" + GRID_NAME + "@127.0.0.1");
- checkInvalid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
-
- checkInvalid("");
- checkInvalid("127.0.0.1");
- checkInvalid("127.0.0.1:" + DFLT_IPC_PORT);
- }
-
- /**
- * Tests for Grid having normal name and IGFS having {@code null} name.
- *
- * @throws Exception If failed.
- */
- public void testHandshakeDefaultIgfs() throws Exception {
- startUp(false, true);
-
- checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@");
- checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1");
- checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
-
- checkInvalid(IGFS_NAME + "@");
- checkInvalid(IGFS_NAME + "@127.0.0.1");
- checkInvalid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
-
- checkValid(":" + GRID_NAME + "@");
- checkValid(":" + GRID_NAME + "@127.0.0.1");
- checkValid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
-
- checkInvalid("");
- checkInvalid("127.0.0.1");
- checkInvalid("127.0.0.1:" + DFLT_IPC_PORT);
- }
-
- /**
- * Tests for Grid having {@code null} name and IGFS having {@code null} name.
- *
- * @throws Exception If failed.
- */
- public void testHandshakeDefaultGridDefaultIgfs() throws Exception {
- startUp(true, true);
-
- checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@");
- checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1");
- checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
-
- checkInvalid(IGFS_NAME + "@");
- checkInvalid(IGFS_NAME + "@127.0.0.1");
- checkInvalid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
-
- checkInvalid(":" + GRID_NAME + "@");
- checkInvalid(":" + GRID_NAME + "@127.0.0.1");
- checkInvalid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
-
- checkValid("");
- checkValid("127.0.0.1");
- checkValid("127.0.0.1:" + DFLT_IPC_PORT);
- }
-
- /**
- * Perform startup.
- *
- * @param dfltGridName Default Grid name.
- * @param dfltIgfsName Default IGFS name.
- * @throws Exception If failed.
- */
- private void startUp(boolean dfltGridName, boolean dfltIgfsName) throws Exception {
- Ignite ignite = G.start(gridConfiguration(dfltGridName, dfltIgfsName));
-
- IgniteFs igfs = ignite.fileSystem(dfltIgfsName ? null : IGFS_NAME);
-
- igfs.mkdirs(PATH);
- }
-
- /**
- * Create Grid configuration.
- *
- * @param dfltGridName Default Grid name.
- * @param dfltIgfsName Default IGFS name.
- * @return Grid configuration.
- * @throws Exception If failed.
- */
- private IgniteConfiguration gridConfiguration(boolean dfltGridName, boolean dfltIgfsName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(dfltGridName ? null : GRID_NAME);
-
- cfg.setLocalHost("127.0.0.1");
- cfg.setConnectorConfiguration(null);
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(IP_FINDER);
-
- cfg.setDiscoverySpi(discoSpi);
-
- TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
-
- commSpi.setSharedMemoryPort(-1);
-
- cfg.setCommunicationSpi(commSpi);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("replicated");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setWriteSynchronizationMode(FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
-
- dataCacheCfg.setName("partitioned");
- dataCacheCfg.setCacheMode(PARTITIONED);
- dataCacheCfg.setDistributionMode(PARTITIONED_ONLY);
- dataCacheCfg.setWriteSynchronizationMode(FULL_SYNC);
- dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
- dataCacheCfg.setBackups(0);
- dataCacheCfg.setQueryIndexEnabled(false);
- dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- cfg.setCacheConfiguration(metaCacheCfg, dataCacheCfg);
-
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("partitioned");
- igfsCfg.setMetaCacheName("replicated");
- igfsCfg.setName(dfltIgfsName ? null : IGFS_NAME);
- igfsCfg.setPrefetchBlocks(1);
- igfsCfg.setDefaultMode(PRIMARY);
- igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
- put("type", "tcp");
- put("port", String.valueOf(DFLT_IPC_PORT));
- }});
-
- igfsCfg.setManagementPort(-1);
- igfsCfg.setBlockSize(512 * 1024);
-
- cfg.setIgfsConfiguration(igfsCfg);
-
- return cfg;
- }
-
- /**
- * Check valid file system endpoint.
- *
- * @param authority Authority.
- * @throws Exception If failed.
- */
- private void checkValid(String authority) throws Exception {
- FileSystem fs = fileSystem(authority);
-
- assert fs.exists(new Path(PATH.toString()));
- }
-
- /**
- * Check invalid file system endpoint.
- *
- * @param authority Authority.
- * @throws Exception If failed.
- */
- @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
- private void checkInvalid(final String authority) throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fileSystem(authority);
-
- return null;
- }
- }, IOException.class, null);
- }
-
- /**
- *
- *
- * @param authority Authority.
- * @return File system.
- * @throws Exception If failed.
- */
- private static FileSystem fileSystem(String authority) throws Exception {
- return FileSystem.get(new URI("igfs://" + authority + "/"), configuration(authority));
- }
-
- /**
- * Create configuration for test.
- *
- * @param authority Authority.
- * @return Configuration.
- */
- private static Configuration configuration(String authority) {
- Configuration cfg = new Configuration();
-
- cfg.set("fs.defaultFS", "igfs://" + authority + "/");
- cfg.set("fs.igfs.impl", org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem.class.getName());
- cfg.set("fs.AbstractFileSystem.igfs.impl",
- IgfsHadoopFileSystem.class.getName());
-
- cfg.setBoolean("fs.igfs.impl.disable.cache", true);
-
- cfg.setBoolean(String.format(PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);
- cfg.setBoolean(String.format(PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);
-
- return cfg;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemIpcCacheSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemIpcCacheSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemIpcCacheSelfTest.java
deleted file mode 100644
index a6357f8..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemIpcCacheSelfTest.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.ignite.cache.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.internal.igfs.hadoop.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.ipc.shmem.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.spi.communication.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
-
-import java.lang.reflect.*;
-import java.net.*;
-import java.util.*;
-import java.util.concurrent.atomic.*;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.cache.CacheMode.*;
-import static org.apache.ignite.events.EventType.*;
-
-/**
- * IPC cache test.
- */
-public class IgfsHadoopFileSystemIpcCacheSelfTest extends IgfsCommonAbstractTest {
- /** IP finder. */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** Path to test hadoop configuration. */
- private static final String HADOOP_FS_CFG = "modules/core/src/test/config/hadoop/core-site.xml";
-
- /** Group size. */
- public static final int GRP_SIZE = 128;
-
- /** Started grid counter. */
- private static int cnt;
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
- discoSpi.setIpFinder(IP_FINDER);
-
- cfg.setDiscoverySpi(discoSpi);
-
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("partitioned");
- igfsCfg.setMetaCacheName("replicated");
- igfsCfg.setName("igfs");
- igfsCfg.setManagementPort(IgfsConfiguration.DFLT_MGMT_PORT + cnt);
-
- igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
- put("type", "shmem");
- put("port", String.valueOf(IpcSharedMemoryServerEndpoint.DFLT_IPC_PORT + cnt));
- }});
-
- igfsCfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups.
-
- cfg.setIgfsConfiguration(igfsCfg);
-
- cfg.setCacheConfiguration(cacheConfiguration());
-
- cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
-
- TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
-
- commSpi.setSharedMemoryPort(-1);
-
- cfg.setCommunicationSpi(commSpi);
-
- cnt++;
-
- return cfg;
- }
-
- /**
- * Gets cache configuration.
- *
- * @return Cache configuration.
- */
- private CacheConfiguration[] cacheConfiguration() {
- CacheConfiguration cacheCfg = defaultCacheConfiguration();
-
- cacheCfg.setName("partitioned");
- cacheCfg.setCacheMode(PARTITIONED);
- cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
- cacheCfg.setBackups(0);
- cacheCfg.setQueryIndexEnabled(false);
- cacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("replicated");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- return new CacheConfiguration[] {metaCacheCfg, cacheCfg};
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- startGrids(4);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- G.stopAll(true);
- }
-
- /**
- * Test how IPC cache map works.
- *
- * @throws Exception If failed.
- */
- @SuppressWarnings("unchecked")
- public void testIpcCache() throws Exception {
- Field cacheField = IgfsHadoopIpcIo.class.getDeclaredField("ipcCache");
-
- cacheField.setAccessible(true);
-
- Field activeCntField = IgfsHadoopIpcIo.class.getDeclaredField("activeCnt");
-
- activeCntField.setAccessible(true);
-
- Map<String, IgfsHadoopIpcIo> cache = (Map<String, IgfsHadoopIpcIo>)cacheField.get(null);
-
- String name = "igfs:" + getTestGridName(0) + "@";
-
- Configuration cfg = new Configuration();
-
- cfg.addResource(U.resolveIgniteUrl(HADOOP_FS_CFG));
- cfg.setBoolean("fs.igfs.impl.disable.cache", true);
- cfg.setBoolean(String.format(IgfsHadoopUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, name), true);
-
- // Ensure that existing IO is reused.
- FileSystem fs1 = FileSystem.get(new URI("igfs://" + name + "/"), cfg);
-
- assertEquals(1, cache.size());
-
- IgfsHadoopIpcIo io = null;
-
- System.out.println("CACHE: " + cache);
-
- for (String key : cache.keySet()) {
- if (key.contains("10500")) {
- io = cache.get(key);
-
- break;
- }
- }
-
- assert io != null;
-
- assertEquals(1, ((AtomicInteger)activeCntField.get(io)).get());
-
- // Ensure that when IO is used by multiple file systems and one of them is closed, IO is not stopped.
- FileSystem fs2 = FileSystem.get(new URI("igfs://" + name + "/abc"), cfg);
-
- assertEquals(1, cache.size());
- assertEquals(2, ((AtomicInteger)activeCntField.get(io)).get());
-
- fs2.close();
-
- assertEquals(1, cache.size());
- assertEquals(1, ((AtomicInteger)activeCntField.get(io)).get());
-
- Field stopField = IgfsHadoopIpcIo.class.getDeclaredField("stopping");
-
- stopField.setAccessible(true);
-
- assert !(Boolean)stopField.get(io);
-
- // Ensure that IO is stopped when nobody else is need it.
- fs1.close();
-
- assert cache.isEmpty();
-
- assert (Boolean)stopField.get(io);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoggerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoggerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoggerSelfTest.java
deleted file mode 100644
index 2e3a3ec..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoggerSelfTest.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.ignite.internal.igfs.common.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.util.*;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-import static org.apache.ignite.internal.igfs.common.IgfsLogger.*;
-
-/**
- * Grid IGFS client logger test.
- */
-public class IgfsHadoopFileSystemLoggerSelfTest extends IgfsCommonAbstractTest {
- /** Path string. */
- private static final String PATH_STR = "/dir1/dir2/file;test";
-
- /** Path string with escaped semicolons. */
- private static final String PATH_STR_ESCAPED = PATH_STR.replace(';', '~');
-
- /** Path. */
- private static final IgfsPath PATH = new IgfsPath(PATH_STR);
-
- /** IGFS name. */
- private static final String IGFS_NAME = "igfs";
-
- /** Log file path. */
- private static final String LOG_DIR = U.getIgniteHome();
-
- /** Endpoint address. */
- private static final String ENDPOINT = "localhost:10500";
-
- /** Log file name. */
- private static final String LOG_FILE = LOG_DIR + File.separator + "igfs-log-" + IGFS_NAME + "-" + U.jvmPid() +
- ".csv";
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- removeLogs();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- removeLogs();
- }
-
- /**
- * Remove existing logs.
- *
- * @throws Exception If failed.
- */
- private void removeLogs() throws Exception {
- File dir = new File(LOG_DIR);
-
- File[] logs = dir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return name.startsWith("igfs-log-");
- }
- });
-
- for (File log : logs)
- log.delete();
- }
-
- /**
- * Ensure correct static loggers creation/removal as well as file creation.
- *
- * @throws Exception If failed.
- */
- public void testCreateDelete() throws Exception {
- IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
-
- IgfsLogger sameLog0 = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
-
- // Loggers for the same endpoint must be the same object.
- assert log == sameLog0;
-
- IgfsLogger otherLog = IgfsLogger.logger("other" + ENDPOINT, IGFS_NAME, LOG_DIR, 10);
-
- // Logger for another endpoint must be different.
- assert log != otherLog;
-
- otherLog.close();
-
- log.logDelete(PATH, PRIMARY, false);
-
- log.close();
-
- File logFile = new File(LOG_FILE);
-
- // When there are multiple loggers, closing one must not force flushing.
- assert !logFile.exists();
-
- IgfsLogger sameLog1 = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
-
- assert sameLog0 == sameLog1;
-
- sameLog0.close();
-
- assert !logFile.exists();
-
- sameLog1.close();
-
- // When we cloe the last logger, it must flush data to disk.
- assert logFile.exists();
-
- logFile.delete();
-
- IgfsLogger sameLog2 = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
-
- // This time we expect new logger instance to be created.
- assert sameLog0 != sameLog2;
-
- sameLog2.close();
-
- // As we do not add any records to the logger, we do not expect flushing.
- assert !logFile.exists();
- }
-
- /**
- * Test read operations logging.
- *
- * @throws Exception If failed.
- */
- public void testLogRead() throws Exception {
- IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
-
- log.logOpen(1, PATH, PRIMARY, 2, 3L);
- log.logRandomRead(1, 4L, 5);
- log.logSeek(1, 6L);
- log.logSkip(1, 7L);
- log.logMark(1, 8L);
- log.logReset(1);
- log.logCloseIn(1, 9L, 10L, 11);
-
- log.close();
-
- checkLog(
- new SB().a(U.jvmPid() + d() + TYPE_OPEN_IN + d() + PATH_STR_ESCAPED + d() + PRIMARY + d() + 1 + d() + 2 +
- d() + 3 + d(14)).toString(),
- new SB().a(U.jvmPid() + d() + TYPE_RANDOM_READ + d(3) + 1 + d(7) + 4 + d() + 5 + d(8)).toString(),
- new SB().a(U.jvmPid() + d() + TYPE_SEEK + d(3) + 1 + d(7) + 6 + d(9)).toString(),
- new SB().a(U.jvmPid() + d() + TYPE_SKIP + d(3) + 1 + d(9) + 7 + d(7)).toString(),
- new SB().a(U.jvmPid() + d() + TYPE_MARK + d(3) + 1 + d(10) + 8 + d(6)).toString(),
- new SB().a(U.jvmPid() + d() + TYPE_RESET + d(3) + 1 + d(16)).toString(),
- new SB().a(U.jvmPid() + d() + TYPE_CLOSE_IN + d(3) + 1 + d(11) + 9 + d() + 10 + d() + 11 + d(3)).toString()
- );
- }
-
- /**
- * Test write operations logging.
- *
- * @throws Exception If failed.
- */
- public void testLogWrite() throws Exception {
- IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
-
- log.logCreate(1, PATH, PRIMARY, true, 2, new Integer(3).shortValue(), 4L);
- log.logAppend(2, PATH, PRIMARY, 8);
- log.logCloseOut(2, 9L, 10L, 11);
-
- log.close();
-
- checkLog(
- new SB().a(U.jvmPid() + d() + TYPE_OPEN_OUT + d() + PATH_STR_ESCAPED + d() + PRIMARY + d() + 1 + d() +
- 2 + d(2) + 0 + d() + 1 + d() + 3 + d() + 4 + d(10)).toString(),
- new SB().a(U.jvmPid() + d() + TYPE_OPEN_OUT + d() + PATH_STR_ESCAPED + d() + PRIMARY + d() + 2 + d() +
- 8 + d(2) + 1 + d(13)).toString(),
- new SB().a(U.jvmPid() + d() + TYPE_CLOSE_OUT + d(3) + 2 + d(11) + 9 + d() + 10 + d() + 11 + d(3))
- .toString()
- );
- }
-
- /**
- * Test miscellaneous operations logging.
- *
- * @throws Exception If failed.
- */
- @SuppressWarnings("TooBroadScope")
- public void testLogMisc() throws Exception {
- IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
-
- String newFile = "/dir3/file.test";
- String file1 = "/dir3/file1.test";
- String file2 = "/dir3/file1.test";
-
- log.logMakeDirectory(PATH, PRIMARY);
- log.logRename(PATH, PRIMARY, new IgfsPath(newFile));
- log.logListDirectory(PATH, PRIMARY, new String[] { file1, file2 });
- log.logDelete(PATH, PRIMARY, false);
-
- log.close();
-
- checkLog(
- new SB().a(U.jvmPid() + d() + TYPE_DIR_MAKE + d() + PATH_STR_ESCAPED + d() + PRIMARY + d(17)).toString(),
- new SB().a(U.jvmPid() + d() + TYPE_RENAME + d() + PATH_STR_ESCAPED + d() + PRIMARY + d(15) + newFile +
- d(2)).toString(),
- new SB().a(U.jvmPid() + d() + TYPE_DIR_LIST + d() + PATH_STR_ESCAPED + d() + PRIMARY + d(17) + file1 +
- DELIM_FIELD_VAL + file2).toString(),
- new SB().a(U.jvmPid() + d() + TYPE_DELETE + d(1) + PATH_STR_ESCAPED + d() + PRIMARY + d(16) + 0 +
- d()).toString()
- );
- }
-
- /**
- * Create IGFS file with the given path.
- *
- * @param path File path.
- * @return IGFS file instance.
- */
- private IgfsFile file(String path) {
- return new IgfsFileImpl(new IgfsPath(path), new IgfsFileInfo(), 64 * 1024 * 1024);
- }
-
- /**
- * Ensure that log file has only the following lines.
- *
- * @param lines Expected lines.
- */
- private void checkLog(String... lines) throws Exception {
- BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(LOG_FILE)));
-
- List<String> logLines = new ArrayList<>(lines.length);
-
- String nextLogLine;
-
- while ((nextLogLine = br.readLine()) != null)
- logLines.add(nextLogLine);
-
- U.closeQuiet(br);
-
- assertEquals(lines.length + 1, logLines.size());
-
- assertEquals(logLines.get(0), HDR);
-
- for (int i = 0; i < lines.length; i++) {
- String logLine = logLines.get(i + 1);
-
- logLine = logLine.substring(logLine.indexOf(DELIM_FIELD, logLine.indexOf(DELIM_FIELD) + 1) + 1);
-
- assertEquals(lines[i], logLine);
- }
- }
-
- /**
- * Return single field delimiter.
- *
- * @return Single field delimiter.
- */
- private String d() {
- return d(1);
- }
-
- /**
- * Return a bunch of field delimiters.
- *
- * @param cnt Amount of field delimiters.
- * @return Field delimiters.
- */
- private String d(int cnt) {
- SB buf = new SB();
-
- for (int i = 0; i < cnt; i++)
- buf.a(DELIM_FIELD);
-
- return buf.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoggerStateSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoggerStateSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoggerStateSelfTest.java
deleted file mode 100644
index cbe83f0..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoggerStateSelfTest.java
+++ /dev/null
@@ -1,325 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.ignite.*;
-import org.apache.ignite.cache.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.hadoop.v1.*;
-import org.apache.ignite.internal.igfs.common.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.spi.discovery.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
-
-import java.lang.reflect.*;
-import java.net.*;
-import java.nio.file.*;
-import java.util.*;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.cache.CacheMode.*;
-import static org.apache.ignite.igfs.IgfsMode.*;
-import static org.apache.ignite.igfs.hadoop.IgfsHadoopParameters.*;
-
-/**
- * Ensures that sampling is really turned on/off.
- */
-public class IgfsHadoopFileSystemLoggerStateSelfTest extends IgfsCommonAbstractTest {
- /** IGFS. */
- private IgfsEx igfs;
-
- /** File system. */
- private FileSystem fs;
-
- /** Whether logging is enabled in FS configuration. */
- private boolean logging;
-
- /** whether sampling is enabled. */
- private Boolean sampling;
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- U.closeQuiet(fs);
-
- igfs = null;
- fs = null;
-
- G.stopAll(true);
-
- logging = false;
- sampling = null;
- }
-
- /**
- * Startup the grid and instantiate the file system.
- *
- * @throws Exception If failed.
- */
- private void startUp() throws Exception {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("partitioned");
- igfsCfg.setMetaCacheName("replicated");
- igfsCfg.setName("igfs");
- igfsCfg.setBlockSize(512 * 1024);
- igfsCfg.setDefaultMode(PRIMARY);
- igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
- put("type", "tcp");
- put("port", "10500");
- }});
-
- CacheConfiguration cacheCfg = defaultCacheConfiguration();
-
- cacheCfg.setName("partitioned");
- cacheCfg.setCacheMode(PARTITIONED);
- cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
- cacheCfg.setBackups(0);
- cacheCfg.setQueryIndexEnabled(false);
- cacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("replicated");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- IgniteConfiguration cfg = new IgniteConfiguration();
-
- cfg.setGridName("igfs-grid");
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
-
- cfg.setLocalHost("127.0.0.1");
- cfg.setConnectorConfiguration(null);
-
- Ignite g = G.start(cfg);
-
- igfs = (IgfsEx)g.fileSystem("igfs");
-
- igfs.globalSampling(sampling);
-
- fs = fileSystem();
- }
-
- /**
- * When logging is disabled and sampling is not set no-op logger must be used.
- *
- * @throws Exception If failed.
- */
- public void testLoggingDisabledSamplingNotSet() throws Exception {
- startUp();
-
- assert !logEnabled();
- }
-
- /**
- * When logging is enabled and sampling is not set file logger must be used.
- *
- * @throws Exception If failed.
- */
- public void testLoggingEnabledSamplingNotSet() throws Exception {
- logging = true;
-
- startUp();
-
- assert logEnabled();
- }
-
- /**
- * When logging is disabled and sampling is disabled no-op logger must be used.
- *
- * @throws Exception If failed.
- */
- public void testLoggingDisabledSamplingDisabled() throws Exception {
- sampling = false;
-
- startUp();
-
- assert !logEnabled();
- }
-
- /**
- * When logging is enabled and sampling is disabled no-op logger must be used.
- *
- * @throws Exception If failed.
- */
- public void testLoggingEnabledSamplingDisabled() throws Exception {
- logging = true;
- sampling = false;
-
- startUp();
-
- assert !logEnabled();
- }
-
- /**
- * When logging is disabled and sampling is enabled file logger must be used.
- *
- * @throws Exception If failed.
- */
- public void testLoggingDisabledSamplingEnabled() throws Exception {
- sampling = true;
-
- startUp();
-
- assert logEnabled();
- }
-
- /**
- * When logging is enabled and sampling is enabled file logger must be used.
- *
- * @throws Exception If failed.
- */
- public void testLoggingEnabledSamplingEnabled() throws Exception {
- logging = true;
- sampling = true;
-
- startUp();
-
- assert logEnabled();
- }
-
- /**
- * Ensure sampling change through API causes changes in logging on subsequent client connections.
- *
- * @throws Exception If failed.
- */
- public void testSamplingChange() throws Exception {
- // Start with sampling not set.
- startUp();
-
- assert !logEnabled();
-
- fs.close();
-
- // "Not set" => true transition.
- igfs.globalSampling(true);
-
- fs = fileSystem();
-
- assert logEnabled();
-
- fs.close();
-
- // True => "not set" transition.
- igfs.globalSampling(null);
-
- fs = fileSystem();
-
- assert !logEnabled();
-
- // "Not-set" => false transition.
- igfs.globalSampling(false);
-
- fs = fileSystem();
-
- assert !logEnabled();
-
- fs.close();
-
- // False => "not=set" transition.
- igfs.globalSampling(null);
-
- fs = fileSystem();
-
- assert !logEnabled();
-
- fs.close();
-
- // True => false transition.
- igfs.globalSampling(true);
- igfs.globalSampling(false);
-
- fs = fileSystem();
-
- assert !logEnabled();
-
- fs.close();
-
- // False => true transition.
- igfs.globalSampling(true);
-
- fs = fileSystem();
-
- assert logEnabled();
- }
-
- /**
- * Ensure that log directory is set to IGFS when client FS connects.
- *
- * @throws Exception If failed.
- */
- @SuppressWarnings("ConstantConditions")
- public void testLogDirectory() throws Exception {
- startUp();
-
- assertEquals(Paths.get(U.getIgniteHome()).normalize().toString(),
- igfs.clientLogDirectory());
- }
-
- /**
- * Instantiate new file system.
- *
- * @return New file system.
- * @throws Exception If failed.
- */
- private IgfsHadoopFileSystem fileSystem() throws Exception {
- Configuration fsCfg = new Configuration();
-
- fsCfg.addResource(U.resolveIgniteUrl("modules/core/src/test/config/hadoop/core-site-loopback.xml"));
-
- fsCfg.setBoolean("fs.igfs.impl.disable.cache", true);
-
- if (logging)
- fsCfg.setBoolean(String.format(PARAM_IGFS_LOG_ENABLED, "igfs:igfs-grid@"), logging);
-
- fsCfg.setStrings(String.format(PARAM_IGFS_LOG_DIR, "igfs:igfs-grid@"), U.getIgniteHome());
-
- return (IgfsHadoopFileSystem)FileSystem.get(new URI("igfs://igfs:igfs-grid@/"), fsCfg);
- }
-
- /**
- * Ensure that real logger is used by the file system.
- *
- * @return {@code True} in case path is secondary.
- * @throws Exception If failed.
- */
- private boolean logEnabled() throws Exception {
- assert fs != null;
-
- Field field = fs.getClass().getDeclaredField("clientLog");
-
- field.setAccessible(true);
-
- return ((IgfsLogger)field.get(fs)).isLogEnabled();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackAbstractSelfTest.java
deleted file mode 100644
index bd9b031..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackAbstractSelfTest.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import java.util.*;
-
-import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
-
-/**
- * IGFS Hadoop file system IPC loopback self test.
- */
-public abstract class IgfsHadoopFileSystemLoopbackAbstractSelfTest extends
- IgfsHadoopFileSystemAbstractSelfTest {
- /**
- * Constructor.
- *
- * @param mode IGFS mode.
- * @param skipEmbed Skip embedded mode flag.
- */
- protected IgfsHadoopFileSystemLoopbackAbstractSelfTest(IgfsMode mode, boolean skipEmbed) {
- super(mode, skipEmbed, true);
- }
-
- /** {@inheritDoc} */
- @Override protected Map<String, String> primaryIpcEndpointConfiguration(final String gridName) {
- return new HashMap<String, String>() {{
- put("type", "tcp");
- put("port", String.valueOf(DFLT_IPC_PORT + getTestGridIndex(gridName)));
- }};
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java
deleted file mode 100644
index 6035c7a..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC loopback self test in DUAL_ASYNC mode.
- */
-public class IgfsHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest extends
- IgfsHadoopFileSystemLoopbackAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest() {
- super(DUAL_ASYNC, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java
deleted file mode 100644
index 3900b0b..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC loopback self test in DUAL_SYNC mode.
- */
-public class IgfsHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest
- extends IgfsHadoopFileSystemLoopbackAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest() {
- super(DUAL_SYNC, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java
deleted file mode 100644
index dee717e..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC loopback self test in PRIMARY mode.
- */
-public class IgfsHadoopFileSystemLoopbackEmbeddedPrimarySelfTest
- extends IgfsHadoopFileSystemLoopbackAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemLoopbackEmbeddedPrimarySelfTest() {
- super(PRIMARY, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java
deleted file mode 100644
index 1f13019..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC loopback self test in SECONDARY mode.
- */
-public class IgfsHadoopFileSystemLoopbackEmbeddedSecondarySelfTest extends
- IgfsHadoopFileSystemLoopbackAbstractSelfTest {
-
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemLoopbackEmbeddedSecondarySelfTest() {
- super(PROXY, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java
deleted file mode 100644
index 4a66da6..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC loopback self test in DUAL_ASYNC mode.
- */
-public class IgfsHadoopFileSystemLoopbackExternalDualAsyncSelfTest extends
- IgfsHadoopFileSystemLoopbackAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemLoopbackExternalDualAsyncSelfTest() {
- super(DUAL_ASYNC, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalDualSyncSelfTest.java
deleted file mode 100644
index f5f49ed..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalDualSyncSelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC loopback self test in DUAL_SYNC mode.
- */
-public class IgfsHadoopFileSystemLoopbackExternalDualSyncSelfTest
- extends IgfsHadoopFileSystemLoopbackAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemLoopbackExternalDualSyncSelfTest() {
- super(DUAL_SYNC, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalPrimarySelfTest.java
deleted file mode 100644
index 5903704..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalPrimarySelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC loopback self test in PRIMARY mode.
- */
-public class IgfsHadoopFileSystemLoopbackExternalPrimarySelfTest
- extends IgfsHadoopFileSystemLoopbackAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemLoopbackExternalPrimarySelfTest() {
- super(PRIMARY, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalSecondarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalSecondarySelfTest.java
deleted file mode 100644
index 0e95f83..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemLoopbackExternalSecondarySelfTest.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC loopback self test in SECONDARY mode.
- */
-public class IgfsHadoopFileSystemLoopbackExternalSecondarySelfTest extends
- IgfsHadoopFileSystemLoopbackAbstractSelfTest {
-
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemLoopbackExternalSecondarySelfTest() {
- super(PROXY, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemSecondaryModeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemSecondaryModeSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemSecondaryModeSelfTest.java
deleted file mode 100644
index b88816a..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemSecondaryModeSelfTest.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.ignite.cache.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.hadoop.*;
-import org.apache.ignite.igfs.hadoop.v1.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.lang.*;
-import org.apache.ignite.spi.discovery.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
-
-import java.net.*;
-import java.util.*;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.cache.CacheMode.*;
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * Ensures correct modes resolution for SECONDARY paths.
- */
-public class IgfsHadoopFileSystemSecondaryModeSelfTest extends IgfsCommonAbstractTest {
- /** Path to check. */
- private static final Path PATH = new Path("/dir");
-
- /** Pattern matching the path. */
- private static final String PATTERN_MATCHES = "/dir";
-
- /** Pattern doesn't matching the path. */
- private static final String PATTERN_NOT_MATCHES = "/files";
-
- /** Default IGFS mode. */
- private IgfsMode mode;
-
- /** Path modes. */
- private Map<String, IgfsMode> pathModes;
-
- /** File system. */
- private IgfsHadoopFileSystem fs;
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- mode = null;
- pathModes = null;
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- U.closeQuiet(fs);
-
- fs = null;
-
- G.stopAll(true);
- }
-
- /**
- * Perform initial startup.
- *
- * @throws Exception If failed.
- */
- @SuppressWarnings("NullableProblems")
- private void startUp() throws Exception {
- startUpSecondary();
-
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("partitioned");
- igfsCfg.setMetaCacheName("replicated");
- igfsCfg.setName("igfs");
- igfsCfg.setBlockSize(512 * 1024);
- igfsCfg.setDefaultMode(mode);
- igfsCfg.setPathModes(pathModes);
- igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
- put("type", "tcp");
- put("port", "10500");
- }});
-
- igfsCfg.setManagementPort(-1);
- igfsCfg.setSecondaryFileSystem(new IgfsHadoopFileSystemWrapper(
- "igfs://igfs-secondary:igfs-grid-secondary@127.0.0.1:11500/",
- "modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml"));
-
- CacheConfiguration cacheCfg = defaultCacheConfiguration();
-
- cacheCfg.setName("partitioned");
- cacheCfg.setCacheMode(PARTITIONED);
- cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
- cacheCfg.setBackups(0);
- cacheCfg.setQueryIndexEnabled(false);
- cacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("replicated");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- IgniteConfiguration cfg = new IgniteConfiguration();
-
- cfg.setGridName("igfs-grid");
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
-
- cfg.setLocalHost("127.0.0.1");
-
- G.start(cfg);
-
- Configuration fsCfg = new Configuration();
-
- fsCfg.addResource(U.resolveIgniteUrl("modules/core/src/test/config/hadoop/core-site-loopback.xml"));
-
- fsCfg.setBoolean("fs.igfs.impl.disable.cache", true);
-
- fs = (IgfsHadoopFileSystem)FileSystem.get(new URI("igfs://igfs:igfs-grid@/"), fsCfg);
- }
-
- /**
- * Startup secondary file system.
- *
- * @throws Exception If failed.
- */
- private void startUpSecondary() throws Exception {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("partitioned");
- igfsCfg.setMetaCacheName("replicated");
- igfsCfg.setName("igfs-secondary");
- igfsCfg.setBlockSize(512 * 1024);
- igfsCfg.setDefaultMode(PRIMARY);
- igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
- put("type", "tcp");
- put("port", "11500");
- }});
-
- CacheConfiguration cacheCfg = defaultCacheConfiguration();
-
- cacheCfg.setName("partitioned");
- cacheCfg.setCacheMode(PARTITIONED);
- cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
- cacheCfg.setBackups(0);
- cacheCfg.setQueryIndexEnabled(false);
- cacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("replicated");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- IgniteConfiguration cfg = new IgniteConfiguration();
-
- cfg.setGridName("igfs-grid-secondary");
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
-
- cfg.setLocalHost("127.0.0.1");
-
- G.start(cfg);
- }
-
- /**
- * Check path resolution when secondary mode is not default and there are no other exclusion paths.
- *
- * @throws Exception If failed.
- */
- public void testSecondaryNotDefaultNoExclusions() throws Exception {
- mode = PRIMARY;
-
- startUp();
-
- assert !secondary(PATH);
- assert !secondary(PATH);
- }
-
- /**
- * Check path resolution when secondary mode is not default and there is no matching exclusion paths.
- *
- * @throws Exception If failed.
- */
- public void testSecondaryNotDefaultNonMatchingExclusion() throws Exception {
- mode = PRIMARY;
-
- pathModes(F.t(PATTERN_NOT_MATCHES, PROXY));
-
- startUp();
-
- assert !secondary(PATH);
- assert !secondary(PATH);
- }
-
- /**
- * Check path resolution when secondary mode is not default and there is matching exclusion path.
- *
- * @throws Exception If failed.
- */
- public void testSecondaryNotDefaultMatchingExclusion() throws Exception {
- mode = PRIMARY;
-
- pathModes(F.t(PATTERN_NOT_MATCHES, PROXY), F.t(PATTERN_MATCHES, PROXY));
-
- startUp();
-
- assert secondary(PATH);
- assert secondary(PATH);
- }
-
- /**
- * Check path resolution when secondary mode is default and there is no exclusion paths.
- *
- * @throws Exception If failed.
- */
- public void testSecondaryDefaultNoExclusions() throws Exception {
- mode = PROXY;
-
- startUp();
-
- assert secondary(PATH);
- assert secondary(PATH);
- }
-
- /**
- * Check path resolution when secondary mode is default and there is no matching exclusion paths.
- *
- * @throws Exception If failed.
- */
- public void testSecondaryDefaultNonMatchingExclusion() throws Exception {
- mode = PROXY;
-
- pathModes(F.t(PATTERN_NOT_MATCHES, PRIMARY));
-
- startUp();
-
- assert secondary(PATH);
- assert secondary(PATH);
- }
-
- /**
- * Check path resolution when secondary mode is default and there is no matching exclusion paths.
- *
- * @throws Exception If failed.
- */
- public void testSecondaryDefaultMatchingExclusion() throws Exception {
- mode = PROXY;
-
- pathModes(F.t(PATTERN_NOT_MATCHES, PRIMARY), F.t(PATTERN_MATCHES, PRIMARY));
-
- startUp();
-
- assert !secondary(PATH);
- assert !secondary(PATH);
- }
-
- /**
- * Set IGFS modes for particular paths.
- *
- * @param modes Modes.
- */
- @SafeVarargs
- final void pathModes(IgniteBiTuple<String, IgfsMode>... modes) {
- assert modes != null;
-
- pathModes = new LinkedHashMap<>(modes.length, 1.0f);
-
- for (IgniteBiTuple<String, IgfsMode> mode : modes)
- pathModes.put(mode.getKey(), mode.getValue());
- }
-
- /**
- * Check whether the given path is threaten as SECONDARY in the file system.
- *
- * @param path Path to check.
- * @return {@code True} in case path is secondary.
- * @throws Exception If failed.
- */
- private boolean secondary(Path path) throws Exception {
- return fs.mode(path) == PROXY;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemAbstractSelfTest.java
deleted file mode 100644
index ed7b7cc..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemAbstractSelfTest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.util.ipc.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.testframework.*;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
-
-/**
- * IGFS Hadoop file system IPC self test.
- */
-public abstract class IgfsHadoopFileSystemShmemAbstractSelfTest extends IgfsHadoopFileSystemAbstractSelfTest {
- /**
- * Constructor.
- *
- * @param mode IGFS mode.
- * @param skipEmbed Skip embedded mode flag.
- */
- protected IgfsHadoopFileSystemShmemAbstractSelfTest(IgfsMode mode, boolean skipEmbed) {
- super(mode, skipEmbed, false);
- }
-
- /** {@inheritDoc} */
- @Override protected Map<String, String> primaryIpcEndpointConfiguration(final String gridName) {
- return new HashMap<String, String>() {{
- put("type", "shmem");
- put("port", String.valueOf(DFLT_IPC_PORT + getTestGridIndex(gridName)));
- }};
- }
-
- /**
- * Checks correct behaviour in case when we run out of system
- * resources.
- *
- * @throws Exception If error occurred.
- */
- @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
- public void testOutOfResources() throws Exception {
- final Collection<IpcEndpoint> eps = new LinkedList<>();
-
- try {
- IgniteCheckedException e = (IgniteCheckedException)GridTestUtils.assertThrows(log, new Callable<Object>() {
- @SuppressWarnings("InfiniteLoopStatement")
- @Override public Object call() throws Exception {
- while (true) {
- IpcEndpoint ep = IpcEndpointFactory.connectEndpoint("shmem:10500", log);
-
- eps.add(ep);
- }
- }
- }, IgniteCheckedException.class, null);
-
- assertNotNull(e);
-
- String msg = e.getMessage();
-
- assertTrue("Invalid exception: " + X.getFullStackTrace(e),
- msg.contains("(error code: 28)") ||
- msg.contains("(error code: 24)") ||
- msg.contains("(error code: 12)"));
- }
- finally {
- for (IpcEndpoint ep : eps)
- ep.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java
deleted file mode 100644
index d11e8d1..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC shmem self test in DUAL_ASYNC mode.
- */
-public class IgfsHadoopFileSystemShmemEmbeddedDualAsyncSelfTest
- extends IgfsHadoopFileSystemShmemAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemShmemEmbeddedDualAsyncSelfTest() {
- super(DUAL_ASYNC, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java
deleted file mode 100644
index 6138215..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC shmem self test in DUAL_SYNC mode.
- */
-public class IgfsHadoopFileSystemShmemEmbeddedDualSyncSelfTest
- extends IgfsHadoopFileSystemShmemAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemShmemEmbeddedDualSyncSelfTest() {
- super(DUAL_SYNC, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedPrimarySelfTest.java
deleted file mode 100644
index 9fc2d33..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedPrimarySelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC shmem self test in PRIMARY mode.
- */
-public class IgfsHadoopFileSystemShmemEmbeddedPrimarySelfTest
- extends IgfsHadoopFileSystemShmemAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemShmemEmbeddedPrimarySelfTest() {
- super(PRIMARY, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedSecondarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedSecondarySelfTest.java
deleted file mode 100644
index a05158c..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemEmbeddedSecondarySelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC shmem self test in SECONDARY mode.
- */
-public class IgfsHadoopFileSystemShmemEmbeddedSecondarySelfTest
- extends IgfsHadoopFileSystemShmemAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemShmemEmbeddedSecondarySelfTest() {
- super(PROXY, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalDualAsyncSelfTest.java
deleted file mode 100644
index c132e5b..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalDualAsyncSelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC shmem self test in DUAL_ASYNC mode.
- */
-public class IgfsHadoopFileSystemShmemExternalDualAsyncSelfTest
- extends IgfsHadoopFileSystemShmemAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemShmemExternalDualAsyncSelfTest() {
- super(DUAL_ASYNC, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalDualSyncSelfTest.java
deleted file mode 100644
index 21b393d..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalDualSyncSelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC shmem self test in DUAL_SYNC mode.
- */
-public class IgfsHadoopFileSystemShmemExternalDualSyncSelfTest
- extends IgfsHadoopFileSystemShmemAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemShmemExternalDualSyncSelfTest() {
- super(DUAL_SYNC, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalPrimarySelfTest.java
deleted file mode 100644
index f147929..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalPrimarySelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC shmem self test in PRIMARY mode.
- */
-public class IgfsHadoopFileSystemShmemExternalPrimarySelfTest
- extends IgfsHadoopFileSystemShmemAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemShmemExternalPrimarySelfTest() {
- super(PRIMARY, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalSecondarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalSecondarySelfTest.java
deleted file mode 100644
index 7c77740..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopFileSystemShmemExternalSecondarySelfTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * IGFS Hadoop file system IPC shmem self test in SECONDARY mode.
- */
-public class IgfsHadoopFileSystemShmemExternalSecondarySelfTest
- extends IgfsHadoopFileSystemShmemAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopFileSystemShmemExternalSecondarySelfTest() {
- super(PROXY, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsNearOnlyMultiNodeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsNearOnlyMultiNodeSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsNearOnlyMultiNodeSelfTest.java
index d27d93d..d128731 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsNearOnlyMultiNodeSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsNearOnlyMultiNodeSelfTest.java
@@ -75,7 +75,7 @@ public class IgfsNearOnlyMultiNodeSelfTest extends GridCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName("partitioned");
igfsCfg.setMetaCacheName("partitioned");
@@ -88,7 +88,7 @@ public class IgfsNearOnlyMultiNodeSelfTest extends GridCommonAbstractTest {
igfsCfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups.
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
cfg.setCacheConfiguration(cacheConfiguration(gridName));
[28/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobTracker.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobTracker.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobTracker.java
deleted file mode 100644
index 0beaf32..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobTracker.java
+++ /dev/null
@@ -1,1625 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.jobtracker;
-
-import org.apache.ignite.*;
-import org.apache.ignite.events.*;
-import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.managers.eventstorage.*;
-import org.apache.ignite.internal.processors.cache.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.counter.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.future.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.lang.*;
-import org.jdk8.backport.*;
-import org.jetbrains.annotations.*;
-
-import javax.cache.event.*;
-import javax.cache.expiry.*;
-import javax.cache.processor.*;
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-
-import static java.util.concurrent.TimeUnit.*;
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopJobPhase.*;
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopTaskType.*;
-import static org.apache.ignite.internal.processors.hadoop.taskexecutor.GridHadoopTaskState.*;
-
-/**
- * Hadoop job tracker.
- */
-public class GridHadoopJobTracker extends GridHadoopComponent {
- /** */
- private final GridMutex mux = new GridMutex();
-
- /** */
- private volatile GridCacheProjectionEx<GridHadoopJobId, GridHadoopJobMetadata> jobMetaPrj;
-
- /** Projection with expiry policy for finished job updates. */
- private volatile GridCacheProjectionEx<GridHadoopJobId, GridHadoopJobMetadata> finishedJobMetaPrj;
-
- /** Map-reduce execution planner. */
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private GridHadoopMapReducePlanner mrPlanner;
-
- /** All the known jobs. */
- private final ConcurrentMap<GridHadoopJobId, GridFutureAdapterEx<GridHadoopJob>> jobs = new ConcurrentHashMap8<>();
-
- /** Locally active jobs. */
- private final ConcurrentMap<GridHadoopJobId, JobLocalState> activeJobs = new ConcurrentHashMap8<>();
-
- /** Locally requested finish futures. */
- private final ConcurrentMap<GridHadoopJobId, GridFutureAdapter<GridHadoopJobId>> activeFinishFuts =
- new ConcurrentHashMap8<>();
-
- /** Event processing service. */
- private ExecutorService evtProcSvc;
-
- /** Component busy lock. */
- private GridSpinReadWriteLock busyLock;
-
- /** Closure to check result of async transform of system cache. */
- private final IgniteInClosure<IgniteInternalFuture<?>> failsLog = new CI1<IgniteInternalFuture<?>>() {
- @Override public void apply(IgniteInternalFuture<?> gridFut) {
- try {
- gridFut.get();
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to transform system cache.", e);
- }
- }
- };
-
- /** {@inheritDoc} */
- @Override public void start(GridHadoopContext ctx) throws IgniteCheckedException {
- super.start(ctx);
-
- busyLock = new GridSpinReadWriteLock();
-
- evtProcSvc = Executors.newFixedThreadPool(1);
- }
-
- /**
- * @return Job meta projection.
- */
- @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext")
- private GridCacheProjectionEx<GridHadoopJobId, GridHadoopJobMetadata> jobMetaCache() {
- GridCacheProjectionEx<GridHadoopJobId, GridHadoopJobMetadata> prj = jobMetaPrj;
-
- if (prj == null) {
- synchronized (mux) {
- if ((prj = jobMetaPrj) == null) {
- CacheProjection<Object, Object> sysCache = ctx.kernalContext().cache()
- .cache(CU.SYS_CACHE_HADOOP_MR);
-
- assert sysCache != null;
-
- mrPlanner = ctx.planner();
-
- try {
- ctx.kernalContext().resource().injectGeneric(mrPlanner);
- }
- catch (IgniteCheckedException e) { // Must not happen.
- U.error(log, "Failed to inject resources.", e);
-
- throw new IllegalStateException(e);
- }
-
- jobMetaPrj = prj = (GridCacheProjectionEx<GridHadoopJobId, GridHadoopJobMetadata>)
- sysCache.projection(GridHadoopJobId.class, GridHadoopJobMetadata.class);
-
- if (ctx.configuration().getFinishedJobInfoTtl() > 0) {
- ExpiryPolicy finishedJobPlc = new ModifiedExpiryPolicy(
- new Duration(MILLISECONDS, ctx.configuration().getFinishedJobInfoTtl()));
-
- finishedJobMetaPrj = prj.withExpiryPolicy(finishedJobPlc);
- }
- else
- finishedJobMetaPrj = jobMetaPrj;
- }
- }
- }
-
- return prj;
- }
-
- /**
- * @return Projection with expiry policy for finished job updates.
- */
- private GridCacheProjectionEx<GridHadoopJobId, GridHadoopJobMetadata> finishedJobMetaCache() {
- GridCacheProjectionEx<GridHadoopJobId, GridHadoopJobMetadata> prj = finishedJobMetaPrj;
-
- if (prj == null) {
- jobMetaCache();
-
- prj = finishedJobMetaPrj;
-
- assert prj != null;
- }
-
- return prj;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("deprecation")
- @Override public void onKernalStart() throws IgniteCheckedException {
- super.onKernalStart();
-
- jobMetaCache().context().continuousQueries().executeInternalQuery(
- new CacheEntryUpdatedListener<GridHadoopJobId, GridHadoopJobMetadata>() {
- @Override public void onUpdated(final Iterable<CacheEntryEvent<? extends GridHadoopJobId,
- ? extends GridHadoopJobMetadata>> evts) {
- if (!busyLock.tryReadLock())
- return;
-
- try {
- // Must process query callback in a separate thread to avoid deadlocks.
- evtProcSvc.submit(new EventHandler() {
- @Override protected void body() throws IgniteCheckedException {
- processJobMetadataUpdates(evts);
- }
- });
- }
- finally {
- busyLock.readUnlock();
- }
- }
- },
- null,
- true,
- true
- );
-
- ctx.kernalContext().event().addLocalEventListener(new GridLocalEventListener() {
- @Override public void onEvent(final Event evt) {
- if (!busyLock.tryReadLock())
- return;
-
- try {
- // Must process discovery callback in a separate thread to avoid deadlock.
- evtProcSvc.submit(new EventHandler() {
- @Override protected void body() {
- processNodeLeft((DiscoveryEvent)evt);
- }
- });
- }
- finally {
- busyLock.readUnlock();
- }
- }
- }, EventType.EVT_NODE_FAILED, EventType.EVT_NODE_LEFT);
- }
-
- /** {@inheritDoc} */
- @Override public void onKernalStop(boolean cancel) {
- super.onKernalStop(cancel);
-
- busyLock.writeLock();
-
- evtProcSvc.shutdown();
-
- // Fail all pending futures.
- for (GridFutureAdapter<GridHadoopJobId> fut : activeFinishFuts.values())
- fut.onDone(new IgniteCheckedException("Failed to execute Hadoop map-reduce job (grid is stopping)."));
- }
-
- /**
- * Submits execution of Hadoop job to grid.
- *
- * @param jobId Job ID.
- * @param info Job info.
- * @return Job completion future.
- */
- @SuppressWarnings("unchecked")
- public IgniteInternalFuture<GridHadoopJobId> submit(GridHadoopJobId jobId, GridHadoopJobInfo info) {
- if (!busyLock.tryReadLock()) {
- return new GridFinishedFutureEx<>(new IgniteCheckedException("Failed to execute map-reduce job " +
- "(grid is stopping): " + info));
- }
-
- try {
- long jobPrepare = U.currentTimeMillis();
-
- if (jobs.containsKey(jobId) || jobMetaCache().containsKey(jobId))
- throw new IgniteCheckedException("Failed to submit job. Job with the same ID already exists: " + jobId);
-
- GridHadoopJob job = job(jobId, info);
-
- GridHadoopMapReducePlan mrPlan = mrPlanner.preparePlan(job, ctx.nodes(), null);
-
- GridHadoopJobMetadata meta = new GridHadoopJobMetadata(ctx.localNodeId(), jobId, info);
-
- meta.mapReducePlan(mrPlan);
-
- meta.pendingSplits(allSplits(mrPlan));
- meta.pendingReducers(allReducers(mrPlan));
-
- GridFutureAdapter<GridHadoopJobId> completeFut = new GridFutureAdapter<>();
-
- GridFutureAdapter<GridHadoopJobId> old = activeFinishFuts.put(jobId, completeFut);
-
- assert old == null : "Duplicate completion future [jobId=" + jobId + ", old=" + old + ']';
-
- if (log.isDebugEnabled())
- log.debug("Submitting job metadata [jobId=" + jobId + ", meta=" + meta + ']');
-
- long jobStart = U.currentTimeMillis();
-
- GridHadoopPerformanceCounter perfCntr = GridHadoopPerformanceCounter.getCounter(meta.counters(),
- ctx.localNodeId());
-
- perfCntr.clientSubmissionEvents(info);
- perfCntr.onJobPrepare(jobPrepare);
- perfCntr.onJobStart(jobStart);
-
- if (jobMetaCache().putIfAbsent(jobId, meta) != null)
- throw new IgniteCheckedException("Failed to submit job. Job with the same ID already exists: " + jobId);
-
- return completeFut;
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to submit job: " + jobId, e);
-
- return new GridFinishedFutureEx<>(e);
- }
- finally {
- busyLock.readUnlock();
- }
- }
-
- /**
- * Convert Hadoop job metadata to job status.
- *
- * @param meta Metadata.
- * @return Status.
- */
- @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
- public static GridHadoopJobStatus status(GridHadoopJobMetadata meta) {
- GridHadoopJobInfo jobInfo = meta.jobInfo();
-
- return new GridHadoopJobStatus(
- meta.jobId(),
- jobInfo.jobName(),
- jobInfo.user(),
- meta.pendingSplits() != null ? meta.pendingSplits().size() : 0,
- meta.pendingReducers() != null ? meta.pendingReducers().size() : 0,
- meta.mapReducePlan().mappers(),
- meta.mapReducePlan().reducers(),
- meta.phase(),
- meta.failCause() != null,
- meta.version()
- );
- }
-
- /**
- * Gets hadoop job status for given job ID.
- *
- * @param jobId Job ID to get status for.
- * @return Job status for given job ID or {@code null} if job was not found.
- */
- @Nullable public GridHadoopJobStatus status(GridHadoopJobId jobId) throws IgniteCheckedException {
- if (!busyLock.tryReadLock())
- return null; // Grid is stopping.
-
- try {
- GridHadoopJobMetadata meta = jobMetaCache().get(jobId);
-
- return meta != null ? status(meta) : null;
- }
- finally {
- busyLock.readUnlock();
- }
- }
-
- /**
- * Gets job finish future.
- *
- * @param jobId Job ID.
- * @return Finish future or {@code null}.
- * @throws IgniteCheckedException If failed.
- */
- @Nullable public IgniteInternalFuture<?> finishFuture(GridHadoopJobId jobId) throws IgniteCheckedException {
- if (!busyLock.tryReadLock())
- return null; // Grid is stopping.
-
- try {
- GridHadoopJobMetadata meta = jobMetaCache().get(jobId);
-
- if (meta == null)
- return null;
-
- if (log.isTraceEnabled())
- log.trace("Got job metadata for status check [locNodeId=" + ctx.localNodeId() + ", meta=" + meta + ']');
-
- if (meta.phase() == PHASE_COMPLETE) {
- if (log.isTraceEnabled())
- log.trace("Job is complete, returning finished future: " + jobId);
-
- return new GridFinishedFutureEx<>(jobId, meta.failCause());
- }
-
- GridFutureAdapter<GridHadoopJobId> fut = F.addIfAbsent(activeFinishFuts, jobId,
- new GridFutureAdapter<GridHadoopJobId>());
-
- // Get meta from cache one more time to close the window.
- meta = jobMetaCache().get(jobId);
-
- if (log.isTraceEnabled())
- log.trace("Re-checking job metadata [locNodeId=" + ctx.localNodeId() + ", meta=" + meta + ']');
-
- if (meta == null) {
- fut.onDone();
-
- activeFinishFuts.remove(jobId , fut);
- }
- else if (meta.phase() == PHASE_COMPLETE) {
- fut.onDone(jobId, meta.failCause());
-
- activeFinishFuts.remove(jobId , fut);
- }
-
- return fut;
- }
- finally {
- busyLock.readUnlock();
- }
- }
-
- /**
- * Gets job plan by job ID.
- *
- * @param jobId Job ID.
- * @return Job plan.
- * @throws IgniteCheckedException If failed.
- */
- public GridHadoopMapReducePlan plan(GridHadoopJobId jobId) throws IgniteCheckedException {
- if (!busyLock.tryReadLock())
- return null;
-
- try {
- GridHadoopJobMetadata meta = jobMetaCache().get(jobId);
-
- if (meta != null)
- return meta.mapReducePlan();
-
- return null;
- }
- finally {
- busyLock.readUnlock();
- }
- }
-
- /**
- * Callback from task executor invoked when a task has been finished.
- *
- * @param info Task info.
- * @param status Task status.
- */
- @SuppressWarnings({"ConstantConditions", "ThrowableResultOfMethodCallIgnored"})
- public void onTaskFinished(GridHadoopTaskInfo info, GridHadoopTaskStatus status) {
- if (!busyLock.tryReadLock())
- return;
-
- try {
- assert status.state() != RUNNING;
-
- if (log.isDebugEnabled())
- log.debug("Received task finished callback [info=" + info + ", status=" + status + ']');
-
- JobLocalState state = activeJobs.get(info.jobId());
-
- // Task CRASHes with null fail cause.
- assert (status.state() != FAILED) || status.failCause() != null :
- "Invalid task status [info=" + info + ", status=" + status + ']';
-
- assert state != null || (ctx.jobUpdateLeader() && (info.type() == COMMIT || info.type() == ABORT)):
- "Missing local state for finished task [info=" + info + ", status=" + status + ']';
-
- StackedProcessor incrCntrs = null;
-
- if (status.state() == COMPLETED)
- incrCntrs = new IncrementCountersProcessor(null, status.counters());
-
- switch (info.type()) {
- case SETUP: {
- state.onSetupFinished(info, status, incrCntrs);
-
- break;
- }
-
- case MAP: {
- state.onMapFinished(info, status, incrCntrs);
-
- break;
- }
-
- case REDUCE: {
- state.onReduceFinished(info, status, incrCntrs);
-
- break;
- }
-
- case COMBINE: {
- state.onCombineFinished(info, status, incrCntrs);
-
- break;
- }
-
- case COMMIT:
- case ABORT: {
- GridCacheProjectionEx<GridHadoopJobId, GridHadoopJobMetadata> cache = finishedJobMetaCache();
-
- cache.invokeAsync(info.jobId(), new UpdatePhaseProcessor(incrCntrs, PHASE_COMPLETE)).
- listenAsync(failsLog);
-
- break;
- }
- }
- }
- finally {
- busyLock.readUnlock();
- }
- }
-
- /**
- * @param jobId Job id.
- * @param c Closure of operation.
- */
- private void transform(GridHadoopJobId jobId, EntryProcessor<GridHadoopJobId, GridHadoopJobMetadata, Void> c) {
- jobMetaCache().invokeAsync(jobId, c).listenAsync(failsLog);
- }
-
- /**
- * Callback from task executor called when process is ready to received shuffle messages.
- *
- * @param jobId Job ID.
- * @param reducers Reducers.
- * @param desc Process descriptor.
- */
- public void onExternalMappersInitialized(GridHadoopJobId jobId, Collection<Integer> reducers,
- GridHadoopProcessDescriptor desc) {
- transform(jobId, new InitializeReducersProcessor(null, reducers, desc));
- }
-
- /**
- * Gets all input splits for given hadoop map-reduce plan.
- *
- * @param plan Map-reduce plan.
- * @return Collection of all input splits that should be processed.
- */
- @SuppressWarnings("ConstantConditions")
- private Map<GridHadoopInputSplit, Integer> allSplits(GridHadoopMapReducePlan plan) {
- Map<GridHadoopInputSplit, Integer> res = new HashMap<>();
-
- int taskNum = 0;
-
- for (UUID nodeId : plan.mapperNodeIds()) {
- for (GridHadoopInputSplit split : plan.mappers(nodeId)) {
- if (res.put(split, taskNum++) != null)
- throw new IllegalStateException("Split duplicate.");
- }
- }
-
- return res;
- }
-
- /**
- * Gets all reducers for this job.
- *
- * @param plan Map-reduce plan.
- * @return Collection of reducers.
- */
- private Collection<Integer> allReducers(GridHadoopMapReducePlan plan) {
- Collection<Integer> res = new HashSet<>();
-
- for (int i = 0; i < plan.reducers(); i++)
- res.add(i);
-
- return res;
- }
-
- /**
- * Processes node leave (or fail) event.
- *
- * @param evt Discovery event.
- */
- @SuppressWarnings("ConstantConditions")
- private void processNodeLeft(DiscoveryEvent evt) {
- if (log.isDebugEnabled())
- log.debug("Processing discovery event [locNodeId=" + ctx.localNodeId() + ", evt=" + evt + ']');
-
- // Check only if this node is responsible for job status updates.
- if (ctx.jobUpdateLeader()) {
- boolean checkSetup = evt.eventNode().order() < ctx.localNodeOrder();
-
- // Iteration over all local entries is correct since system cache is REPLICATED.
- for (Object metaObj : jobMetaCache().values()) {
- GridHadoopJobMetadata meta = (GridHadoopJobMetadata)metaObj;
-
- GridHadoopJobId jobId = meta.jobId();
-
- GridHadoopMapReducePlan plan = meta.mapReducePlan();
-
- GridHadoopJobPhase phase = meta.phase();
-
- try {
- if (checkSetup && phase == PHASE_SETUP && !activeJobs.containsKey(jobId)) {
- // Failover setup task.
- GridHadoopJob job = job(jobId, meta.jobInfo());
-
- Collection<GridHadoopTaskInfo> setupTask = setupTask(jobId);
-
- assert setupTask != null;
-
- ctx.taskExecutor().run(job, setupTask);
- }
- else if (phase == PHASE_MAP || phase == PHASE_REDUCE) {
- // Must check all nodes, even that are not event node ID due to
- // multiple node failure possibility.
- Collection<GridHadoopInputSplit> cancelSplits = null;
-
- for (UUID nodeId : plan.mapperNodeIds()) {
- if (ctx.kernalContext().discovery().node(nodeId) == null) {
- // Node has left the grid.
- Collection<GridHadoopInputSplit> mappers = plan.mappers(nodeId);
-
- if (cancelSplits == null)
- cancelSplits = new HashSet<>();
-
- cancelSplits.addAll(mappers);
- }
- }
-
- Collection<Integer> cancelReducers = null;
-
- for (UUID nodeId : plan.reducerNodeIds()) {
- if (ctx.kernalContext().discovery().node(nodeId) == null) {
- // Node has left the grid.
- int[] reducers = plan.reducers(nodeId);
-
- if (cancelReducers == null)
- cancelReducers = new HashSet<>();
-
- for (int rdc : reducers)
- cancelReducers.add(rdc);
- }
- }
-
- if (cancelSplits != null || cancelReducers != null)
- jobMetaCache().invoke(meta.jobId(), new CancelJobProcessor(null, new IgniteCheckedException(
- "One or more nodes participating in map-reduce job execution failed."), cancelSplits,
- cancelReducers));
- }
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to cancel job: " + meta, e);
- }
- }
- }
- }
-
- /**
- * @param updated Updated cache entries.
- * @throws IgniteCheckedException If failed.
- */
- private void processJobMetadataUpdates(
- Iterable<CacheEntryEvent<? extends GridHadoopJobId, ? extends GridHadoopJobMetadata>> updated)
- throws IgniteCheckedException {
- UUID locNodeId = ctx.localNodeId();
-
- for (CacheEntryEvent<? extends GridHadoopJobId, ? extends GridHadoopJobMetadata> entry : updated) {
- GridHadoopJobId jobId = entry.getKey();
- GridHadoopJobMetadata meta = entry.getValue();
-
- if (meta == null || !ctx.isParticipating(meta))
- continue;
-
- if (log.isDebugEnabled())
- log.debug("Processing job metadata update callback [locNodeId=" + locNodeId +
- ", meta=" + meta + ']');
-
- try {
- ctx.taskExecutor().onJobStateChanged(meta);
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to process job state changed callback (will fail the job) " +
- "[locNodeId=" + locNodeId + ", jobId=" + jobId + ", meta=" + meta + ']', e);
-
- transform(jobId, new CancelJobProcessor(null, e));
-
- continue;
- }
-
- processJobMetaUpdate(jobId, meta, locNodeId);
- }
- }
-
- /**
- * @param jobId Job ID.
- * @param plan Map-reduce plan.
- */
- private void printPlan(GridHadoopJobId jobId, GridHadoopMapReducePlan plan) {
- log.info("Plan for " + jobId);
-
- SB b = new SB();
-
- b.a(" Map: ");
-
- for (UUID nodeId : plan.mapperNodeIds())
- b.a(nodeId).a("=").a(plan.mappers(nodeId).size()).a(' ');
-
- log.info(b.toString());
-
- b = new SB();
-
- b.a(" Reduce: ");
-
- for (UUID nodeId : plan.reducerNodeIds())
- b.a(nodeId).a("=").a(Arrays.toString(plan.reducers(nodeId))).a(' ');
-
- log.info(b.toString());
- }
-
- /**
- * @param jobId Job ID.
- * @param meta Job metadata.
- * @param locNodeId Local node ID.
- * @throws IgniteCheckedException If failed.
- */
- private void processJobMetaUpdate(GridHadoopJobId jobId, GridHadoopJobMetadata meta, UUID locNodeId)
- throws IgniteCheckedException {
- JobLocalState state = activeJobs.get(jobId);
-
- GridHadoopJob job = job(jobId, meta.jobInfo());
-
- GridHadoopMapReducePlan plan = meta.mapReducePlan();
-
- switch (meta.phase()) {
- case PHASE_SETUP: {
- if (ctx.jobUpdateLeader()) {
- Collection<GridHadoopTaskInfo> setupTask = setupTask(jobId);
-
- if (setupTask != null)
- ctx.taskExecutor().run(job, setupTask);
- }
-
- break;
- }
-
- case PHASE_MAP: {
- // Check if we should initiate new task on local node.
- Collection<GridHadoopTaskInfo> tasks = mapperTasks(plan.mappers(locNodeId), meta);
-
- if (tasks != null)
- ctx.taskExecutor().run(job, tasks);
-
- break;
- }
-
- case PHASE_REDUCE: {
- if (meta.pendingReducers().isEmpty() && ctx.jobUpdateLeader()) {
- GridHadoopTaskInfo info = new GridHadoopTaskInfo(COMMIT, jobId, 0, 0, null);
-
- if (log.isDebugEnabled())
- log.debug("Submitting COMMIT task for execution [locNodeId=" + locNodeId +
- ", jobId=" + jobId + ']');
-
- ctx.taskExecutor().run(job, Collections.singletonList(info));
-
- break;
- }
-
- Collection<GridHadoopTaskInfo> tasks = reducerTasks(plan.reducers(locNodeId), job);
-
- if (tasks != null)
- ctx.taskExecutor().run(job, tasks);
-
- break;
- }
-
- case PHASE_CANCELLING: {
- // Prevent multiple task executor notification.
- if (state != null && state.onCancel()) {
- if (log.isDebugEnabled())
- log.debug("Cancelling local task execution for job: " + meta);
-
- ctx.taskExecutor().cancelTasks(jobId);
- }
-
- if (meta.pendingSplits().isEmpty() && meta.pendingReducers().isEmpty()) {
- if (ctx.jobUpdateLeader()) {
- if (state == null)
- state = initState(jobId);
-
- // Prevent running multiple abort tasks.
- if (state.onAborted()) {
- GridHadoopTaskInfo info = new GridHadoopTaskInfo(ABORT, jobId, 0, 0, null);
-
- if (log.isDebugEnabled())
- log.debug("Submitting ABORT task for execution [locNodeId=" + locNodeId +
- ", jobId=" + jobId + ']');
-
- ctx.taskExecutor().run(job, Collections.singletonList(info));
- }
- }
-
- break;
- }
- else {
- // Check if there are unscheduled mappers or reducers.
- Collection<GridHadoopInputSplit> cancelMappers = new ArrayList<>();
- Collection<Integer> cancelReducers = new ArrayList<>();
-
- Collection<GridHadoopInputSplit> mappers = plan.mappers(ctx.localNodeId());
-
- if (mappers != null) {
- for (GridHadoopInputSplit b : mappers) {
- if (state == null || !state.mapperScheduled(b))
- cancelMappers.add(b);
- }
- }
-
- int[] rdc = plan.reducers(ctx.localNodeId());
-
- if (rdc != null) {
- for (int r : rdc) {
- if (state == null || !state.reducerScheduled(r))
- cancelReducers.add(r);
- }
- }
-
- if (!cancelMappers.isEmpty() || !cancelReducers.isEmpty())
- transform(jobId, new CancelJobProcessor(null, cancelMappers, cancelReducers));
- }
-
- break;
- }
-
- case PHASE_COMPLETE: {
- if (log.isDebugEnabled())
- log.debug("Job execution is complete, will remove local state from active jobs " +
- "[jobId=" + jobId + ", meta=" + meta + ']');
-
- if (state != null) {
- state = activeJobs.remove(jobId);
-
- assert state != null;
-
- ctx.shuffle().jobFinished(jobId);
- }
-
- GridFutureAdapter<GridHadoopJobId> finishFut = activeFinishFuts.remove(jobId);
-
- if (finishFut != null) {
- if (log.isDebugEnabled())
- log.debug("Completing job future [locNodeId=" + locNodeId + ", meta=" + meta + ']');
-
- finishFut.onDone(jobId, meta.failCause());
- }
-
- if (ctx.jobUpdateLeader())
- job.cleanupStagingDirectory();
-
- jobs.remove(jobId);
-
- job.dispose(false);
-
- if (ctx.jobUpdateLeader()) {
- ClassLoader ldr = job.getClass().getClassLoader();
-
- try {
- String statWriterClsName = job.info().property(GridHadoopUtils.JOB_COUNTER_WRITER_PROPERTY);
-
- if (statWriterClsName != null) {
- Class<?> cls = ldr.loadClass(statWriterClsName);
-
- GridHadoopCounterWriter writer = (GridHadoopCounterWriter)cls.newInstance();
-
- GridHadoopCounters cntrs = meta.counters();
-
- writer.write(job.info(), jobId, cntrs);
- }
- }
- catch (Exception e) {
- log.error("Can't write statistic due to: ", e);
- }
- }
-
- break;
- }
-
- default:
- throw new IllegalStateException("Unknown phase: " + meta.phase());
- }
- }
-
- /**
- * Creates setup task based on job information.
- *
- * @param jobId Job ID.
- * @return Setup task wrapped in collection.
- */
- @Nullable private Collection<GridHadoopTaskInfo> setupTask(GridHadoopJobId jobId) {
- if (activeJobs.containsKey(jobId))
- return null;
- else {
- initState(jobId);
-
- return Collections.singleton(new GridHadoopTaskInfo(SETUP, jobId, 0, 0, null));
- }
- }
-
- /**
- * Creates mapper tasks based on job information.
- *
- * @param mappers Mapper blocks.
- * @param meta Job metadata.
- * @return Collection of created task infos or {@code null} if no mapper tasks scheduled for local node.
- */
- private Collection<GridHadoopTaskInfo> mapperTasks(Iterable<GridHadoopInputSplit> mappers, GridHadoopJobMetadata meta) {
- UUID locNodeId = ctx.localNodeId();
- GridHadoopJobId jobId = meta.jobId();
-
- JobLocalState state = activeJobs.get(jobId);
-
- Collection<GridHadoopTaskInfo> tasks = null;
-
- if (mappers != null) {
- if (state == null)
- state = initState(jobId);
-
- for (GridHadoopInputSplit split : mappers) {
- if (state.addMapper(split)) {
- if (log.isDebugEnabled())
- log.debug("Submitting MAP task for execution [locNodeId=" + locNodeId +
- ", split=" + split + ']');
-
- GridHadoopTaskInfo taskInfo = new GridHadoopTaskInfo(MAP, jobId, meta.taskNumber(split), 0, split);
-
- if (tasks == null)
- tasks = new ArrayList<>();
-
- tasks.add(taskInfo);
- }
- }
- }
-
- return tasks;
- }
-
- /**
- * Creates reducer tasks based on job information.
- *
- * @param reducers Reducers (may be {@code null}).
- * @param job Job instance.
- * @return Collection of task infos.
- */
- private Collection<GridHadoopTaskInfo> reducerTasks(int[] reducers, GridHadoopJob job) {
- UUID locNodeId = ctx.localNodeId();
- GridHadoopJobId jobId = job.id();
-
- JobLocalState state = activeJobs.get(jobId);
-
- Collection<GridHadoopTaskInfo> tasks = null;
-
- if (reducers != null) {
- if (state == null)
- state = initState(job.id());
-
- for (int rdc : reducers) {
- if (state.addReducer(rdc)) {
- if (log.isDebugEnabled())
- log.debug("Submitting REDUCE task for execution [locNodeId=" + locNodeId +
- ", rdc=" + rdc + ']');
-
- GridHadoopTaskInfo taskInfo = new GridHadoopTaskInfo(REDUCE, jobId, rdc, 0, null);
-
- if (tasks == null)
- tasks = new ArrayList<>();
-
- tasks.add(taskInfo);
- }
- }
- }
-
- return tasks;
- }
-
- /**
- * Initializes local state for given job metadata.
- *
- * @param jobId Job ID.
- * @return Local state.
- */
- private JobLocalState initState(GridHadoopJobId jobId) {
- return F.addIfAbsent(activeJobs, jobId, new JobLocalState());
- }
-
- /**
- * Gets or creates job instance.
- *
- * @param jobId Job ID.
- * @param jobInfo Job info.
- * @return Job.
- * @throws IgniteCheckedException If failed.
- */
- @Nullable public GridHadoopJob job(GridHadoopJobId jobId, @Nullable GridHadoopJobInfo jobInfo) throws IgniteCheckedException {
- GridFutureAdapterEx<GridHadoopJob> fut = jobs.get(jobId);
-
- if (fut != null || (fut = jobs.putIfAbsent(jobId, new GridFutureAdapterEx<GridHadoopJob>())) != null)
- return fut.get();
-
- fut = jobs.get(jobId);
-
- GridHadoopJob job = null;
-
- try {
- if (jobInfo == null) {
- GridHadoopJobMetadata meta = jobMetaCache().get(jobId);
-
- if (meta == null)
- throw new IgniteCheckedException("Failed to find job metadata for ID: " + jobId);
-
- jobInfo = meta.jobInfo();
- }
-
- job = jobInfo.createJob(jobId, log);
-
- job.initialize(false, ctx.localNodeId());
-
- fut.onDone(job);
-
- return job;
- }
- catch (IgniteCheckedException e) {
- fut.onDone(e);
-
- jobs.remove(jobId, fut);
-
- if (job != null) {
- try {
- job.dispose(false);
- }
- catch (IgniteCheckedException e0) {
- U.error(log, "Failed to dispose job: " + jobId, e0);
- }
- }
-
- throw e;
- }
- }
-
- /**
- * Kills job.
- *
- * @param jobId Job ID.
- * @return {@code True} if job was killed.
- * @throws IgniteCheckedException If failed.
- */
- public boolean killJob(GridHadoopJobId jobId) throws IgniteCheckedException {
- if (!busyLock.tryReadLock())
- return false; // Grid is stopping.
-
- try {
- GridHadoopJobMetadata meta = jobMetaCache().get(jobId);
-
- if (meta != null && meta.phase() != PHASE_COMPLETE && meta.phase() != PHASE_CANCELLING) {
- GridHadoopTaskCancelledException err = new GridHadoopTaskCancelledException("Job cancelled.");
-
- jobMetaCache().invoke(jobId, new CancelJobProcessor(null, err));
- }
- }
- finally {
- busyLock.readUnlock();
- }
-
- IgniteInternalFuture<?> fut = finishFuture(jobId);
-
- if (fut != null) {
- try {
- fut.get();
- }
- catch (Throwable e) {
- if (e.getCause() instanceof GridHadoopTaskCancelledException)
- return true;
- }
- }
-
- return false;
- }
-
- /**
- * Returns job counters.
- *
- * @param jobId Job identifier.
- * @return Job counters or {@code null} if job cannot be found.
- * @throws IgniteCheckedException If failed.
- */
- @Nullable public GridHadoopCounters jobCounters(GridHadoopJobId jobId) throws IgniteCheckedException {
- if (!busyLock.tryReadLock())
- return null;
-
- try {
- final GridHadoopJobMetadata meta = jobMetaCache().get(jobId);
-
- return meta != null ? meta.counters() : null;
- }
- finally {
- busyLock.readUnlock();
- }
- }
-
- /**
- * Event handler protected by busy lock.
- */
- private abstract class EventHandler implements Runnable {
- /** {@inheritDoc} */
- @Override public void run() {
- if (!busyLock.tryReadLock())
- return;
-
- try {
- body();
- }
- catch (Throwable e) {
- U.error(log, "Unhandled exception while processing event.", e);
- }
- finally {
- busyLock.readUnlock();
- }
- }
-
- /**
- * Handler body.
- */
- protected abstract void body() throws Exception;
- }
-
- /**
- *
- */
- private class JobLocalState {
- /** Mappers. */
- private final Collection<GridHadoopInputSplit> currMappers = new HashSet<>();
-
- /** Reducers. */
- private final Collection<Integer> currReducers = new HashSet<>();
-
- /** Number of completed mappers. */
- private final AtomicInteger completedMappersCnt = new AtomicInteger();
-
- /** Cancelled flag. */
- private boolean cancelled;
-
- /** Aborted flag. */
- private boolean aborted;
-
- /**
- * @param mapSplit Map split to add.
- * @return {@code True} if mapper was added.
- */
- private boolean addMapper(GridHadoopInputSplit mapSplit) {
- return currMappers.add(mapSplit);
- }
-
- /**
- * @param rdc Reducer number to add.
- * @return {@code True} if reducer was added.
- */
- private boolean addReducer(int rdc) {
- return currReducers.add(rdc);
- }
-
- /**
- * Checks whether this split was scheduled for given attempt.
- *
- * @param mapSplit Map split to check.
- * @return {@code True} if mapper was scheduled.
- */
- public boolean mapperScheduled(GridHadoopInputSplit mapSplit) {
- return currMappers.contains(mapSplit);
- }
-
- /**
- * Checks whether this split was scheduled for given attempt.
- *
- * @param rdc Reducer number to check.
- * @return {@code True} if reducer was scheduled.
- */
- public boolean reducerScheduled(int rdc) {
- return currReducers.contains(rdc);
- }
-
- /**
- * @param taskInfo Task info.
- * @param status Task status.
- * @param prev Previous closure.
- */
- private void onSetupFinished(final GridHadoopTaskInfo taskInfo, GridHadoopTaskStatus status, StackedProcessor prev) {
- final GridHadoopJobId jobId = taskInfo.jobId();
-
- if (status.state() == FAILED || status.state() == CRASHED)
- transform(jobId, new CancelJobProcessor(prev, status.failCause()));
- else
- transform(jobId, new UpdatePhaseProcessor(prev, PHASE_MAP));
- }
-
- /**
- * @param taskInfo Task info.
- * @param status Task status.
- * @param prev Previous closure.
- */
- private void onMapFinished(final GridHadoopTaskInfo taskInfo, GridHadoopTaskStatus status,
- final StackedProcessor prev) {
- final GridHadoopJobId jobId = taskInfo.jobId();
-
- boolean lastMapperFinished = completedMappersCnt.incrementAndGet() == currMappers.size();
-
- if (status.state() == FAILED || status.state() == CRASHED) {
- // Fail the whole job.
- transform(jobId, new RemoveMappersProcessor(prev, taskInfo.inputSplit(), status.failCause()));
-
- return;
- }
-
- IgniteInClosure<IgniteInternalFuture<?>> cacheUpdater = new CIX1<IgniteInternalFuture<?>>() {
- @Override public void applyx(IgniteInternalFuture<?> f) {
- Throwable err = null;
-
- if (f != null) {
- try {
- f.get();
- }
- catch (IgniteCheckedException e) {
- err = e;
- }
- }
-
- transform(jobId, new RemoveMappersProcessor(prev, taskInfo.inputSplit(), err));
- }
- };
-
- if (lastMapperFinished)
- ctx.shuffle().flush(jobId).listenAsync(cacheUpdater);
- else
- cacheUpdater.apply(null);
- }
-
- /**
- * @param taskInfo Task info.
- * @param status Task status.
- * @param prev Previous closure.
- */
- private void onReduceFinished(GridHadoopTaskInfo taskInfo, GridHadoopTaskStatus status, StackedProcessor prev) {
- GridHadoopJobId jobId = taskInfo.jobId();
- if (status.state() == FAILED || status.state() == CRASHED)
- // Fail the whole job.
- transform(jobId, new RemoveReducerProcessor(prev, taskInfo.taskNumber(), status.failCause()));
- else
- transform(jobId, new RemoveReducerProcessor(prev, taskInfo.taskNumber()));
- }
-
- /**
- * @param taskInfo Task info.
- * @param status Task status.
- * @param prev Previous closure.
- */
- private void onCombineFinished(GridHadoopTaskInfo taskInfo, GridHadoopTaskStatus status,
- final StackedProcessor prev) {
- final GridHadoopJobId jobId = taskInfo.jobId();
-
- if (status.state() == FAILED || status.state() == CRASHED)
- // Fail the whole job.
- transform(jobId, new RemoveMappersProcessor(prev, currMappers, status.failCause()));
- else {
- ctx.shuffle().flush(jobId).listenAsync(new CIX1<IgniteInternalFuture<?>>() {
- @Override public void applyx(IgniteInternalFuture<?> f) {
- Throwable err = null;
-
- if (f != null) {
- try {
- f.get();
- }
- catch (IgniteCheckedException e) {
- err = e;
- }
- }
-
- transform(jobId, new RemoveMappersProcessor(prev, currMappers, err));
- }
- });
- }
- }
-
- /**
- * @return {@code True} if job was cancelled by this (first) call.
- */
- public boolean onCancel() {
- if (!cancelled && !aborted) {
- cancelled = true;
-
- return true;
- }
-
- return false;
- }
-
- /**
- * @return {@code True} if job was aborted this (first) call.
- */
- public boolean onAborted() {
- if (!aborted) {
- aborted = true;
-
- return true;
- }
-
- return false;
- }
- }
-
- /**
- * Update job phase transform closure.
- */
- private static class UpdatePhaseProcessor extends StackedProcessor {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Phase to update. */
- private final GridHadoopJobPhase phase;
-
- /**
- * @param prev Previous closure.
- * @param phase Phase to update.
- */
- private UpdatePhaseProcessor(@Nullable StackedProcessor prev, GridHadoopJobPhase phase) {
- super(prev);
-
- this.phase = phase;
- }
-
- /** {@inheritDoc} */
- @Override protected void update(GridHadoopJobMetadata meta, GridHadoopJobMetadata cp) {
- cp.phase(phase);
- }
- }
-
- /**
- * Remove mapper transform closure.
- */
- private static class RemoveMappersProcessor extends StackedProcessor {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Mapper split to remove. */
- private final Collection<GridHadoopInputSplit> splits;
-
- /** Error. */
- private final Throwable err;
-
- /**
- * @param prev Previous closure.
- * @param split Mapper split to remove.
- * @param err Error.
- */
- private RemoveMappersProcessor(@Nullable StackedProcessor prev, GridHadoopInputSplit split, Throwable err) {
- this(prev, Collections.singletonList(split), err);
- }
-
- /**
- * @param prev Previous closure.
- * @param splits Mapper splits to remove.
- * @param err Error.
- */
- private RemoveMappersProcessor(@Nullable StackedProcessor prev, Collection<GridHadoopInputSplit> splits,
- Throwable err) {
- super(prev);
-
- this.splits = splits;
- this.err = err;
- }
-
- /** {@inheritDoc} */
- @Override protected void update(GridHadoopJobMetadata meta, GridHadoopJobMetadata cp) {
- Map<GridHadoopInputSplit, Integer> splitsCp = new HashMap<>(cp.pendingSplits());
-
- for (GridHadoopInputSplit s : splits)
- splitsCp.remove(s);
-
- cp.pendingSplits(splitsCp);
-
- if (cp.phase() != PHASE_CANCELLING && err != null)
- cp.failCause(err);
-
- if (err != null)
- cp.phase(PHASE_CANCELLING);
-
- if (splitsCp.isEmpty()) {
- if (cp.phase() != PHASE_CANCELLING)
- cp.phase(PHASE_REDUCE);
- }
- }
- }
-
- /**
- * Remove reducer transform closure.
- */
- private static class RemoveReducerProcessor extends StackedProcessor {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Mapper split to remove. */
- private final int rdc;
-
- /** Error. */
- private Throwable err;
-
- /**
- * @param prev Previous closure.
- * @param rdc Reducer to remove.
- */
- private RemoveReducerProcessor(@Nullable StackedProcessor prev, int rdc) {
- super(prev);
-
- this.rdc = rdc;
- }
-
- /**
- * @param prev Previous closure.
- * @param rdc Reducer to remove.
- * @param err Error.
- */
- private RemoveReducerProcessor(@Nullable StackedProcessor prev, int rdc, Throwable err) {
- super(prev);
-
- this.rdc = rdc;
- this.err = err;
- }
-
- /** {@inheritDoc} */
- @Override protected void update(GridHadoopJobMetadata meta, GridHadoopJobMetadata cp) {
- Collection<Integer> rdcCp = new HashSet<>(cp.pendingReducers());
-
- rdcCp.remove(rdc);
-
- cp.pendingReducers(rdcCp);
-
- if (err != null) {
- cp.phase(PHASE_CANCELLING);
- cp.failCause(err);
- }
- }
- }
-
- /**
- * Initialize reducers.
- */
- private static class InitializeReducersProcessor extends StackedProcessor {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Reducers. */
- private final Collection<Integer> rdc;
-
- /** Process descriptor for reducers. */
- private final GridHadoopProcessDescriptor desc;
-
- /**
- * @param prev Previous closure.
- * @param rdc Reducers to initialize.
- * @param desc External process descriptor.
- */
- private InitializeReducersProcessor(@Nullable StackedProcessor prev,
- Collection<Integer> rdc,
- GridHadoopProcessDescriptor desc) {
- super(prev);
-
- assert !F.isEmpty(rdc);
- assert desc != null;
-
- this.rdc = rdc;
- this.desc = desc;
- }
-
- /** {@inheritDoc} */
- @Override protected void update(GridHadoopJobMetadata meta, GridHadoopJobMetadata cp) {
- Map<Integer, GridHadoopProcessDescriptor> oldMap = meta.reducersAddresses();
-
- Map<Integer, GridHadoopProcessDescriptor> rdcMap = oldMap == null ?
- new HashMap<Integer, GridHadoopProcessDescriptor>() : new HashMap<>(oldMap);
-
- for (Integer r : rdc)
- rdcMap.put(r, desc);
-
- cp.reducersAddresses(rdcMap);
- }
- }
-
- /**
- * Remove reducer transform closure.
- */
- private static class CancelJobProcessor extends StackedProcessor {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Mapper split to remove. */
- private final Collection<GridHadoopInputSplit> splits;
-
- /** Reducers to remove. */
- private final Collection<Integer> rdc;
-
- /** Error. */
- private final Throwable err;
-
- /**
- * @param prev Previous closure.
- * @param err Fail cause.
- */
- private CancelJobProcessor(@Nullable StackedProcessor prev, Throwable err) {
- this(prev, err, null, null);
- }
-
- /**
- * @param prev Previous closure.
- * @param splits Splits to remove.
- * @param rdc Reducers to remove.
- */
- private CancelJobProcessor(@Nullable StackedProcessor prev,
- Collection<GridHadoopInputSplit> splits,
- Collection<Integer> rdc) {
- this(prev, null, splits, rdc);
- }
-
- /**
- * @param prev Previous closure.
- * @param err Error.
- * @param splits Splits to remove.
- * @param rdc Reducers to remove.
- */
- private CancelJobProcessor(@Nullable StackedProcessor prev,
- Throwable err,
- Collection<GridHadoopInputSplit> splits,
- Collection<Integer> rdc) {
- super(prev);
-
- this.splits = splits;
- this.rdc = rdc;
- this.err = err;
- }
-
- /** {@inheritDoc} */
- @Override protected void update(GridHadoopJobMetadata meta, GridHadoopJobMetadata cp) {
- assert meta.phase() == PHASE_CANCELLING || err != null: "Invalid phase for cancel: " + meta;
-
- Collection<Integer> rdcCp = new HashSet<>(cp.pendingReducers());
-
- if (rdc != null)
- rdcCp.removeAll(rdc);
-
- cp.pendingReducers(rdcCp);
-
- Map<GridHadoopInputSplit, Integer> splitsCp = new HashMap<>(cp.pendingSplits());
-
- if (splits != null) {
- for (GridHadoopInputSplit s : splits)
- splitsCp.remove(s);
- }
-
- cp.pendingSplits(splitsCp);
-
- cp.phase(PHASE_CANCELLING);
-
- if (err != null)
- cp.failCause(err);
- }
- }
-
- /**
- * Increment counter values closure.
- */
- private static class IncrementCountersProcessor extends StackedProcessor {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private final GridHadoopCounters counters;
-
- /**
- * @param prev Previous closure.
- * @param counters Task counters to add into job counters.
- */
- private IncrementCountersProcessor(@Nullable StackedProcessor prev, GridHadoopCounters counters) {
- super(prev);
-
- assert counters != null;
-
- this.counters = counters;
- }
-
- /** {@inheritDoc} */
- @Override protected void update(GridHadoopJobMetadata meta, GridHadoopJobMetadata cp) {
- GridHadoopCounters cntrs = new GridHadoopCountersImpl(cp.counters());
-
- cntrs.merge(counters);
-
- cp.counters(cntrs);
- }
- }
-
- /**
- * Abstract stacked closure.
- */
- private abstract static class StackedProcessor implements
- EntryProcessor<GridHadoopJobId, GridHadoopJobMetadata, Void>, Serializable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private final StackedProcessor prev;
-
- /**
- * @param prev Previous closure.
- */
- private StackedProcessor(@Nullable StackedProcessor prev) {
- this.prev = prev;
- }
-
- /** {@inheritDoc} */
- @Override public Void process(MutableEntry<GridHadoopJobId, GridHadoopJobMetadata> e, Object... args) {
- GridHadoopJobMetadata val = apply(e.getValue());
-
- if (val != null)
- e.setValue(val);
- else
- e.remove();;
-
- return null;
- }
-
- /**
- * @param meta Old value.
- * @return New value.
- */
- private GridHadoopJobMetadata apply(GridHadoopJobMetadata meta) {
- if (meta == null)
- return null;
-
- GridHadoopJobMetadata cp = prev != null ? prev.apply(meta) : new GridHadoopJobMetadata(meta);
-
- update(meta, cp);
-
- return cp;
- }
-
- /**
- * Update given job metadata object.
- *
- * @param meta Initial job metadata.
- * @param cp Copy.
- */
- protected abstract void update(GridHadoopJobMetadata meta, GridHadoopJobMetadata cp);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobMetadata.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobMetadata.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobMetadata.java
new file mode 100644
index 0000000..3f574e9
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobMetadata.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.jobtracker;
+
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
+import org.apache.ignite.internal.util.tostring.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobPhase.*;
+
+/**
+ * Hadoop job metadata. Internal object used for distributed job state tracking.
+ */
+public class HadoopJobMetadata implements Externalizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Job ID. */
+ private HadoopJobId jobId;
+
+ /** Job info. */
+ private HadoopJobInfo jobInfo;
+
+ /** Node submitted job. */
+ private UUID submitNodeId;
+
+ /** Map-reduce plan. */
+ private HadoopMapReducePlan mrPlan;
+
+ /** Pending splits for which mapper should be executed. */
+ private Map<HadoopInputSplit, Integer> pendingSplits;
+
+ /** Pending reducers. */
+ private Collection<Integer> pendingReducers;
+
+ /** Reducers addresses. */
+ @GridToStringInclude
+ private Map<Integer, HadoopProcessDescriptor> reducersAddrs;
+
+ /** Job phase. */
+ private HadoopJobPhase phase = PHASE_SETUP;
+
+ /** Fail cause. */
+ @GridToStringExclude
+ private Throwable failCause;
+
+ /** Version. */
+ private long ver;
+
+ /** Job counters */
+ private HadoopCounters counters = new HadoopCountersImpl();
+
+ /**
+ * Empty constructor required by {@link Externalizable}.
+ */
+ public HadoopJobMetadata() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param submitNodeId Submit node ID.
+ * @param jobId Job ID.
+ * @param jobInfo Job info.
+ */
+ public HadoopJobMetadata(UUID submitNodeId, HadoopJobId jobId, HadoopJobInfo jobInfo) {
+ this.jobId = jobId;
+ this.jobInfo = jobInfo;
+ this.submitNodeId = submitNodeId;
+ }
+
+ /**
+ * Copy constructor.
+ *
+ * @param src Metadata to copy.
+ */
+ public HadoopJobMetadata(HadoopJobMetadata src) {
+ // Make sure to preserve alphabetic order.
+ counters = src.counters;
+ failCause = src.failCause;
+ jobId = src.jobId;
+ jobInfo = src.jobInfo;
+ mrPlan = src.mrPlan;
+ pendingSplits = src.pendingSplits;
+ pendingReducers = src.pendingReducers;
+ phase = src.phase;
+ reducersAddrs = src.reducersAddrs;
+ submitNodeId = src.submitNodeId;
+ ver = src.ver + 1;
+ }
+
+ /**
+ * @return Submit node ID.
+ */
+ public UUID submitNodeId() {
+ return submitNodeId;
+ }
+
+ /**
+ * @param phase Job phase.
+ */
+ public void phase(HadoopJobPhase phase) {
+ this.phase = phase;
+ }
+
+ /**
+ * @return Job phase.
+ */
+ public HadoopJobPhase phase() {
+ return phase;
+ }
+
+ /**
+ * Gets reducers addresses for external execution.
+ *
+ * @return Reducers addresses.
+ */
+ public Map<Integer, HadoopProcessDescriptor> reducersAddresses() {
+ return reducersAddrs;
+ }
+
+ /**
+ * Sets reducers addresses for external execution.
+ *
+ * @param reducersAddrs Map of addresses.
+ */
+ public void reducersAddresses(Map<Integer, HadoopProcessDescriptor> reducersAddrs) {
+ this.reducersAddrs = reducersAddrs;
+ }
+
+ /**
+ * Sets collection of pending splits.
+ *
+ * @param pendingSplits Collection of pending splits.
+ */
+ public void pendingSplits(Map<HadoopInputSplit, Integer> pendingSplits) {
+ this.pendingSplits = pendingSplits;
+ }
+
+ /**
+ * Gets collection of pending splits.
+ *
+ * @return Collection of pending splits.
+ */
+ public Map<HadoopInputSplit, Integer> pendingSplits() {
+ return pendingSplits;
+ }
+
+ /**
+ * Sets collection of pending reducers.
+ *
+ * @param pendingReducers Collection of pending reducers.
+ */
+ public void pendingReducers(Collection<Integer> pendingReducers) {
+ this.pendingReducers = pendingReducers;
+ }
+
+ /**
+ * Gets collection of pending reducers.
+ *
+ * @return Collection of pending reducers.
+ */
+ public Collection<Integer> pendingReducers() {
+ return pendingReducers;
+ }
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /**
+ * @param mrPlan Map-reduce plan.
+ */
+ public void mapReducePlan(HadoopMapReducePlan mrPlan) {
+ assert this.mrPlan == null : "Map-reduce plan can only be initialized once.";
+
+ this.mrPlan = mrPlan;
+ }
+
+ /**
+ * @return Map-reduce plan.
+ */
+ public HadoopMapReducePlan mapReducePlan() {
+ return mrPlan;
+ }
+
+ /**
+ * @return Job info.
+ */
+ public HadoopJobInfo jobInfo() {
+ return jobInfo;
+ }
+
+ /**
+ * Returns job counters.
+ *
+ * @return Collection of counters.
+ */
+ public HadoopCounters counters() {
+ return counters;
+ }
+
+ /**
+ * Sets counters.
+ *
+ * @param counters Collection of counters.
+ */
+ public void counters(HadoopCounters counters) {
+ this.counters = counters;
+ }
+
+ /**
+ * @param failCause Fail cause.
+ */
+ public void failCause(Throwable failCause) {
+ assert failCause != null;
+
+ if (this.failCause == null) // Keep the first error.
+ this.failCause = failCause;
+ }
+
+ /**
+ * @return Fail cause.
+ */
+ public Throwable failCause() {
+ return failCause;
+ }
+
+ /**
+ * @return Version.
+ */
+ public long version() {
+ return ver;
+ }
+
+ /**
+ * @param split Split.
+ * @return Task number.
+ */
+ public int taskNumber(HadoopInputSplit split) {
+ return pendingSplits.get(split);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ U.writeUuid(out, submitNodeId);
+ out.writeObject(jobId);
+ out.writeObject(jobInfo);
+ out.writeObject(mrPlan);
+ out.writeObject(pendingSplits);
+ out.writeObject(pendingReducers);
+ out.writeObject(phase);
+ out.writeObject(failCause);
+ out.writeLong(ver);
+ out.writeObject(reducersAddrs);
+ out.writeObject(counters);
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ submitNodeId = U.readUuid(in);
+ jobId = (HadoopJobId)in.readObject();
+ jobInfo = (HadoopJobInfo)in.readObject();
+ mrPlan = (HadoopMapReducePlan)in.readObject();
+ pendingSplits = (Map<HadoopInputSplit,Integer>)in.readObject();
+ pendingReducers = (Collection<Integer>)in.readObject();
+ phase = (HadoopJobPhase)in.readObject();
+ failCause = (Throwable)in.readObject();
+ ver = in.readLong();
+ reducersAddrs = (Map<Integer, HadoopProcessDescriptor>)in.readObject();
+ counters = (HadoopCounters)in.readObject();
+ }
+
+ /** {@inheritDoc} */
+ public String toString() {
+ return S.toString(HadoopJobMetadata.class, this, "pendingMaps", pendingSplits.size(),
+ "pendingReduces", pendingReducers.size(), "failCause", failCause == null ? null :
+ failCause.getClass().getName());
+ }
+}
[07/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopAbstractSelfTest.java
deleted file mode 100644
index 8319255..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopAbstractSelfTest.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.conf.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem;
-import org.apache.ignite.internal.processors.hadoop.fs.*;
-import org.apache.ignite.spi.communication.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
-import org.apache.ignite.testframework.junits.common.*;
-
-import java.io.*;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.cache.CacheMode.*;
-import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*;
-
-/**
- * Abstract class for Hadoop tests.
- */
-public abstract class GridHadoopAbstractSelfTest extends GridCommonAbstractTest {
- /** */
- private static TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** REST port. */
- protected static final int REST_PORT = 11212;
-
- /** IGFS name. */
- protected static final String igfsName = null;
-
- /** IGFS name. */
- protected static final String igfsMetaCacheName = "meta";
-
- /** IGFS name. */
- protected static final String igfsDataCacheName = "data";
-
- /** IGFS block size. */
- protected static final int igfsBlockSize = 1024;
-
- /** IGFS block group size. */
- protected static final int igfsBlockGroupSize = 8;
-
- /** Initial REST port. */
- private int restPort = REST_PORT;
-
- /** Initial classpath. */
- private static String initCp;
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- // Add surefire classpath to regular classpath.
- initCp = System.getProperty("java.class.path");
-
- String surefireCp = System.getProperty("surefire.test.class.path");
-
- if (surefireCp != null)
- System.setProperty("java.class.path", initCp + File.pathSeparatorChar + surefireCp);
-
- super.beforeTestsStarted();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- super.afterTestsStopped();
-
- // Restore classpath.
- System.setProperty("java.class.path", initCp);
-
- initCp = null;
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setHadoopConfiguration(hadoopConfiguration(gridName));
-
- TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
-
- commSpi.setSharedMemoryPort(-1);
-
- cfg.setCommunicationSpi(commSpi);
-
- TcpDiscoverySpi discoSpi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
-
- discoSpi.setIpFinder(IP_FINDER);
-
- if (igfsEnabled()) {
- cfg.setCacheConfiguration(metaCacheConfiguration(), dataCacheConfiguration());
-
- cfg.setIgfsConfiguration(igfsConfiguration());
- }
-
- if (restEnabled()) {
- ConnectorConfiguration clnCfg = new ConnectorConfiguration();
-
- clnCfg.setPort(restPort++);
-
- cfg.setConnectorConfiguration(clnCfg);
- }
-
- cfg.setLocalHost("127.0.0.1");
- cfg.setPeerClassLoadingEnabled(false);
-
- return cfg;
- }
-
- /**
- * @param gridName Grid name.
- * @return Hadoop configuration.
- */
- public GridHadoopConfiguration hadoopConfiguration(String gridName) {
- GridHadoopConfiguration cfg = new GridHadoopConfiguration();
-
- cfg.setMaxParallelTasks(3);
-
- return cfg;
- }
-
- /**
- * @return IGFS configuration.
- */
- public IgfsConfiguration igfsConfiguration() {
- IgfsConfiguration cfg = new IgfsConfiguration();
-
- cfg.setName(igfsName);
- cfg.setBlockSize(igfsBlockSize);
- cfg.setDataCacheName(igfsDataCacheName);
- cfg.setMetaCacheName(igfsMetaCacheName);
- cfg.setFragmentizerEnabled(false);
-
- return cfg;
- }
-
- /**
- * @return IGFS meta cache configuration.
- */
- public CacheConfiguration metaCacheConfiguration() {
- CacheConfiguration cfg = new CacheConfiguration();
-
- cfg.setName(igfsMetaCacheName);
- cfg.setCacheMode(REPLICATED);
- cfg.setAtomicityMode(TRANSACTIONAL);
- cfg.setWriteSynchronizationMode(FULL_SYNC);
-
- return cfg;
- }
-
- /**
- * @return IGFS data cache configuration.
- */
- private CacheConfiguration dataCacheConfiguration() {
- CacheConfiguration cfg = new CacheConfiguration();
-
- cfg.setName(igfsDataCacheName);
- cfg.setCacheMode(PARTITIONED);
- cfg.setAtomicityMode(TRANSACTIONAL);
- cfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(igfsBlockGroupSize));
- cfg.setWriteSynchronizationMode(FULL_SYNC);
-
- return cfg;
- }
-
- /**
- * @return {@code True} if IGFS is enabled on Hadoop nodes.
- */
- protected boolean igfsEnabled() {
- return false;
- }
-
- /**
- * @return {@code True} if REST is enabled on Hadoop nodes.
- */
- protected boolean restEnabled() {
- return false;
- }
-
- /**
- * @return Number of nodes to start.
- */
- protected int gridCount() {
- return 3;
- }
-
- /**
- * @param cfg Config.
- */
- protected void setupFileSystems(Configuration cfg) {
- cfg.set("fs.defaultFS", igfsScheme());
- cfg.set("fs.igfs.impl", org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem.class.getName());
- cfg.set("fs.AbstractFileSystem.igfs.impl", IgfsHadoopFileSystem.
- class.getName());
-
- GridHadoopFileSystemsUtils.setupFileSystems(cfg);
- }
-
- /**
- * @return IGFS scheme for test.
- */
- protected String igfsScheme() {
- return "igfs://:" + getTestGridName(0) + "@/";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopAbstractWordCountTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopAbstractWordCountTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopAbstractWordCountTest.java
deleted file mode 100644
index ebbc0a6..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopAbstractWordCountTest.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import com.google.common.base.*;
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.processors.igfs.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Abstract class for tests based on WordCount test job.
- */
-public abstract class GridHadoopAbstractWordCountTest extends GridHadoopAbstractSelfTest {
- /** Input path. */
- protected static final String PATH_INPUT = "/input";
-
- /** Output path. */
- protected static final String PATH_OUTPUT = "/output";
-
- /** IGFS instance. */
- protected IgfsEx igfs;
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
- Configuration cfg = new Configuration();
-
- setupFileSystems(cfg);
-
- // Init cache by correct LocalFileSystem implementation
- FileSystem.getLocal(cfg);
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- igfs = (IgfsEx)startGrids(gridCount()).fileSystem(igfsName);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids(true);
- }
-
- /** {@inheritDoc} */
- @Override protected boolean igfsEnabled() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override protected int gridCount() {
- return 1;
- }
-
- /**
- * Generates test file.
- *
- * @param path File name.
- * @param wordCounts Words and counts.
- * @throws Exception If failed.
- */
- protected void generateTestFile(String path, Object... wordCounts) throws Exception {
- List<String> wordsArr = new ArrayList<>();
-
- //Generating
- for (int i = 0; i < wordCounts.length; i += 2) {
- String word = (String) wordCounts[i];
- int cnt = (Integer) wordCounts[i + 1];
-
- while (cnt-- > 0)
- wordsArr.add(word);
- }
-
- //Shuffling
- for (int i = 0; i < wordsArr.size(); i++) {
- int j = (int)(Math.random() * wordsArr.size());
-
- Collections.swap(wordsArr, i, j);
- }
-
- //Input file preparing
- PrintWriter testInputFileWriter = new PrintWriter(igfs.create(new IgfsPath(path), true));
-
- int j = 0;
-
- while (j < wordsArr.size()) {
- int i = 5 + (int)(Math.random() * 5);
-
- List<String> subList = wordsArr.subList(j, Math.min(j + i, wordsArr.size()));
- j += i;
-
- testInputFileWriter.println(Joiner.on(' ').join(subList));
- }
-
- testInputFileWriter.close();
- }
-
- /**
- * Reads whole text file into String.
- *
- * @param fileName Name of the file to read.
- * @return Content of the file as String value.
- * @throws Exception If could not read the file.
- */
- protected String readAndSortFile(String fileName) throws Exception {
- BufferedReader reader = new BufferedReader(new InputStreamReader(igfs.open(new IgfsPath(fileName))));
-
- List<String> list = new ArrayList<>();
-
- String line;
-
- while ((line = reader.readLine()) != null)
- list.add(line);
-
- Collections.sort(list);
-
- return Joiner.on('\n').join(list) + "\n";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopClassLoaderTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopClassLoaderTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopClassLoaderTest.java
deleted file mode 100644
index 767be7c..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopClassLoaderTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import junit.framework.*;
-import org.apache.hadoop.mapreduce.*;
-
-/**
- *
- */
-public class GridHadoopClassLoaderTest extends TestCase {
- /** */
- GridHadoopClassLoader ldr = new GridHadoopClassLoader(null);
-
- /**
- * @throws Exception If failed.
- */
- public void testClassLoading() throws Exception {
- assertNotSame(Test1.class, ldr.loadClass(Test1.class.getName()));
- assertNotSame(Test2.class, ldr.loadClass(Test2.class.getName()));
- assertSame(Test3.class, ldr.loadClass(Test3.class.getName()));
- }
-
-// public void testDependencySearch() {
-// assertTrue(ldr.hasExternalDependencies(Test1.class.getName(), new HashSet<String>()));
-// assertTrue(ldr.hasExternalDependencies(Test2.class.getName(), new HashSet<String>()));
-// }
-
- /**
- *
- */
- private static class Test1 {
- /** */
- Test2 t2;
-
- /** */
- Job[][] jobs = new Job[4][4];
- }
-
- /**
- *
- */
- private static abstract class Test2 {
- /** */
- abstract Test1 t1();
- }
-
- /**
- *
- */
- private static class Test3 {
- // No-op.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCommandLineTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCommandLineTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCommandLineTest.java
deleted file mode 100644
index 80cd226..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCommandLineTest.java
+++ /dev/null
@@ -1,440 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import com.google.common.base.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.processors.hadoop.counter.*;
-import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.testframework.junits.common.*;
-import org.jdk8.backport.*;
-
-import java.io.*;
-import java.nio.file.*;
-import java.util.*;
-
-/**
- * Test of integration with Hadoop client via command line interface.
- */
-public class GridHadoopCommandLineTest extends GridCommonAbstractTest {
- /** IGFS instance. */
- private IgfsEx igfs;
-
- /** */
- private static final String igfsName = "igfs";
-
- /** */
- private static File testWorkDir;
-
- /** */
- private static String hadoopHome;
-
- /** */
- private static String hiveHome;
-
- /** */
- private static File examplesJar;
-
- /**
- *
- * @param path File name.
- * @param wordCounts Words and counts.
- * @throws Exception If failed.
- */
- private void generateTestFile(File path, Object... wordCounts) throws Exception {
- List<String> wordsArr = new ArrayList<>();
-
- //Generating
- for (int i = 0; i < wordCounts.length; i += 2) {
- String word = (String) wordCounts[i];
- int cnt = (Integer) wordCounts[i + 1];
-
- while (cnt-- > 0)
- wordsArr.add(word);
- }
-
- //Shuffling
- for (int i = 0; i < wordsArr.size(); i++) {
- int j = (int)(Math.random() * wordsArr.size());
-
- Collections.swap(wordsArr, i, j);
- }
-
- //Writing file
- try (PrintWriter writer = new PrintWriter(path)) {
- int j = 0;
-
- while (j < wordsArr.size()) {
- int i = 5 + (int)(Math.random() * 5);
-
- List<String> subList = wordsArr.subList(j, Math.min(j + i, wordsArr.size()));
- j += i;
-
- writer.println(Joiner.on(' ').join(subList));
- }
-
- writer.flush();
- }
- }
-
- /**
- * Generates two data files to join its with Hive.
- *
- * @throws FileNotFoundException If failed.
- */
- private void generateHiveTestFiles() throws FileNotFoundException {
- try (PrintWriter writerA = new PrintWriter(new File(testWorkDir, "data-a"));
- PrintWriter writerB = new PrintWriter(new File(testWorkDir, "data-b"))) {
- char sep = '\t';
-
- int idB = 0;
- int idA = 0;
- int v = 1000;
-
- for (int i = 0; i < 1000; i++) {
- writerA.print(idA++);
- writerA.print(sep);
- writerA.println(idB);
-
- writerB.print(idB++);
- writerB.print(sep);
- writerB.println(v += 2);
-
- writerB.print(idB++);
- writerB.print(sep);
- writerB.println(v += 2);
- }
-
- writerA.flush();
- writerB.flush();
- }
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
- hiveHome = IgniteSystemProperties.getString("HIVE_HOME");
-
- assertFalse("HIVE_HOME hasn't been set.", F.isEmpty(hiveHome));
-
- hadoopHome = IgniteSystemProperties.getString("HADOOP_HOME");
-
- assertFalse("HADOOP_HOME hasn't been set.", F.isEmpty(hadoopHome));
-
- String mapredHome = hadoopHome + "/share/hadoop/mapreduce";
-
- File[] fileList = new File(mapredHome).listFiles(new FileFilter() {
- @Override public boolean accept(File pathname) {
- return pathname.getName().startsWith("hadoop-mapreduce-examples-") &&
- pathname.getName().endsWith(".jar");
- }
- });
-
- assertEquals("Invalid hadoop distribution.", 1, fileList.length);
-
- examplesJar = fileList[0];
-
- testWorkDir = Files.createTempDirectory("hadoop-cli-test").toFile();
-
- U.copy(U.resolveIgnitePath("docs/core-site.ignite.xml"), new File(testWorkDir, "core-site.xml"), false);
-
- File srcFile = U.resolveIgnitePath("docs/mapred-site.ignite.xml");
- File dstFile = new File(testWorkDir, "mapred-site.xml");
-
- try (BufferedReader in = new BufferedReader(new FileReader(srcFile));
- PrintWriter out = new PrintWriter(dstFile)) {
- String line;
-
- while ((line = in.readLine()) != null) {
- if (line.startsWith("</configuration>"))
- out.println(
- " <property>\n" +
- " <name>" + GridHadoopUtils.JOB_COUNTER_WRITER_PROPERTY + "</name>\n" +
- " <value>" + GridHadoopFSCounterWriter.class.getName() + "</value>\n" +
- " </property>\n");
-
- out.println(line);
- }
-
- out.flush();
- }
-
- generateTestFile(new File(testWorkDir, "test-data"), "red", 100, "green", 200, "blue", 150, "yellow", 50);
-
- generateHiveTestFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- super.afterTestsStopped();
-
- U.delete(testWorkDir);
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- igfs = (IgfsEx) Ignition.start("config/hadoop/default-config.xml").fileSystem(igfsName);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids(true);
- }
-
- /**
- * Creates the process build with appropriate environment to run Hadoop CLI.
- *
- * @return Process builder.
- */
- private ProcessBuilder createProcessBuilder() {
- String sep = ":";
-
- String ggClsPath = GridHadoopJob.class.getProtectionDomain().getCodeSource().getLocation().getPath() + sep +
- GridHadoopJobTracker.class.getProtectionDomain().getCodeSource().getLocation().getPath() + sep +
- ConcurrentHashMap8.class.getProtectionDomain().getCodeSource().getLocation().getPath();
-
- ProcessBuilder res = new ProcessBuilder();
-
- res.environment().put("HADOOP_HOME", hadoopHome);
- res.environment().put("HADOOP_CLASSPATH", ggClsPath);
- res.environment().put("HADOOP_CONF_DIR", testWorkDir.getAbsolutePath());
-
- res.redirectErrorStream(true);
-
- return res;
- }
-
- /**
- * Waits for process exit and prints the its output.
- *
- * @param proc Process.
- * @return Exit code.
- * @throws Exception If failed.
- */
- private int watchProcess(Process proc) throws Exception {
- BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream()));
-
- String line;
-
- while ((line = reader.readLine()) != null)
- log().info(line);
-
- return proc.waitFor();
- }
-
- /**
- * Executes Hadoop command line tool.
- *
- * @param args Arguments for Hadoop command line tool.
- * @return Process exit code.
- * @throws Exception If failed.
- */
- private int executeHadoopCmd(String... args) throws Exception {
- ProcessBuilder procBuilder = createProcessBuilder();
-
- List<String> cmd = new ArrayList<>();
-
- cmd.add(hadoopHome + "/bin/hadoop");
- cmd.addAll(Arrays.asList(args));
-
- procBuilder.command(cmd);
-
- log().info("Execute: " + procBuilder.command());
-
- return watchProcess(procBuilder.start());
- }
-
- /**
- * Executes Hive query.
- *
- * @param qry Query.
- * @return Process exit code.
- * @throws Exception If failed.
- */
- private int executeHiveQuery(String qry) throws Exception {
- ProcessBuilder procBuilder = createProcessBuilder();
-
- List<String> cmd = new ArrayList<>();
-
- procBuilder.command(cmd);
-
- cmd.add(hiveHome + "/bin/hive");
-
- cmd.add("--hiveconf");
- cmd.add("hive.rpc.query.plan=true");
-
- cmd.add("--hiveconf");
- cmd.add("javax.jdo.option.ConnectionURL=jdbc:derby:" + testWorkDir.getAbsolutePath() + "/metastore_db;" +
- "databaseName=metastore_db;create=true");
-
- cmd.add("-e");
- cmd.add(qry);
-
- procBuilder.command(cmd);
-
- log().info("Execute: " + procBuilder.command());
-
- return watchProcess(procBuilder.start());
- }
-
- /**
- * Tests Hadoop command line integration.
- */
- public void testHadoopCommandLine() throws Exception {
- assertEquals(0, executeHadoopCmd("fs", "-ls", "/"));
-
- assertEquals(0, executeHadoopCmd("fs", "-mkdir", "/input"));
-
- assertEquals(0, executeHadoopCmd("fs", "-put", new File(testWorkDir, "test-data").getAbsolutePath(), "/input"));
-
- assertTrue(igfs.exists(new IgfsPath("/input/test-data")));
-
- assertEquals(0, executeHadoopCmd("jar", examplesJar.getAbsolutePath(), "wordcount", "/input", "/output"));
-
- IgfsPath path = new IgfsPath("/user/" + System.getProperty("user.name") + "/");
-
- assertTrue(igfs.exists(path));
-
- IgfsPath jobStatPath = null;
-
- for (IgfsPath jobPath : igfs.listPaths(path)) {
- assertNull(jobStatPath);
-
- jobStatPath = jobPath;
- }
-
- File locStatFile = new File(testWorkDir, "performance");
-
- assertEquals(0, executeHadoopCmd("fs", "-get", jobStatPath.toString() + "/performance", locStatFile.toString()));
-
- long evtCnt = GridHadoopTestUtils.simpleCheckJobStatFile(new BufferedReader(new FileReader(locStatFile)));
-
- assertTrue(evtCnt >= 22); //It's the minimum amount of events for job with combiner.
-
- assertTrue(igfs.exists(new IgfsPath("/output")));
-
- BufferedReader in = new BufferedReader(new InputStreamReader(igfs.open(new IgfsPath("/output/part-r-00000"))));
-
- List<String> res = new ArrayList<>();
-
- String line;
-
- while ((line = in.readLine()) != null)
- res.add(line);
-
- Collections.sort(res);
-
- assertEquals("[blue\t150, green\t200, red\t100, yellow\t50]", res.toString());
- }
-
- /**
- * Runs query check result.
- *
- * @param expRes Expected result.
- * @param qry Query.
- * @throws Exception If failed.
- */
- private void checkQuery(String expRes, String qry) throws Exception {
- assertEquals(0, executeHiveQuery("drop table if exists result"));
-
- assertEquals(0, executeHiveQuery(
- "create table result " +
- "row format delimited fields terminated by ' ' " +
- "stored as textfile " +
- "location '/result' as " + qry
- ));
-
- IgfsInputStreamAdapter in = igfs.open(new IgfsPath("/result/000000_0"));
-
- byte[] buf = new byte[(int) in.length()];
-
- in.read(buf);
-
- assertEquals(expRes, new String(buf));
- }
-
- /**
- * Tests Hive integration.
- */
- public void testHiveCommandLine() throws Exception {
- assertEquals(0, executeHiveQuery(
- "create table table_a (" +
- "id_a int," +
- "id_b int" +
- ") " +
- "row format delimited fields terminated by '\\t'" +
- "stored as textfile " +
- "location '/table-a'"
- ));
-
- assertEquals(0, executeHadoopCmd("fs", "-put", new File(testWorkDir, "data-a").getAbsolutePath(), "/table-a"));
-
- assertEquals(0, executeHiveQuery(
- "create table table_b (" +
- "id_b int," +
- "rndv int" +
- ") " +
- "row format delimited fields terminated by '\\t'" +
- "stored as textfile " +
- "location '/table-b'"
- ));
-
- assertEquals(0, executeHadoopCmd("fs", "-put", new File(testWorkDir, "data-b").getAbsolutePath(), "/table-b"));
-
- checkQuery(
- "0 0\n" +
- "1 2\n" +
- "2 4\n" +
- "3 6\n" +
- "4 8\n" +
- "5 10\n" +
- "6 12\n" +
- "7 14\n" +
- "8 16\n" +
- "9 18\n",
- "select * from table_a order by id_a limit 10"
- );
-
- checkQuery("2000\n", "select count(id_b) from table_b");
-
- checkQuery(
- "250 500 2002\n" +
- "251 502 2006\n" +
- "252 504 2010\n" +
- "253 506 2014\n" +
- "254 508 2018\n" +
- "255 510 2022\n" +
- "256 512 2026\n" +
- "257 514 2030\n" +
- "258 516 2034\n" +
- "259 518 2038\n",
- "select a.id_a, a.id_b, b.rndv" +
- " from table_a a" +
- " inner join table_b b on a.id_b = b.id_b" +
- " where b.rndv > 2000" +
- " order by a.id_a limit 10"
- );
-
- checkQuery("1000\n", "select count(b.id_b) from table_a a inner join table_b b on a.id_b = b.id_b");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopDefaultMapReducePlannerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopDefaultMapReducePlannerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopDefaultMapReducePlannerSelfTest.java
deleted file mode 100644
index b1b0275..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopDefaultMapReducePlannerSelfTest.java
+++ /dev/null
@@ -1,1005 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.cache.*;
-import org.apache.ignite.cluster.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.igfs.mapreduce.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.cluster.*;
-import org.apache.ignite.internal.processors.cache.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.processors.hadoop.planner.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.lang.*;
-import org.apache.ignite.testframework.*;
-import org.jetbrains.annotations.*;
-
-import java.net.*;
-import java.util.*;
-
-/**
- *
- */
-public class GridHadoopDefaultMapReducePlannerSelfTest extends GridHadoopAbstractSelfTest {
- /** */
- private static final UUID ID_1 = new UUID(0, 1);
-
- /** */
- private static final UUID ID_2 = new UUID(0, 2);
-
- /** */
- private static final UUID ID_3 = new UUID(0, 3);
-
- /** */
- private static final String HOST_1 = "host1";
-
- /** */
- private static final String HOST_2 = "host2";
-
- /** */
- private static final String HOST_3 = "host3";
-
- /** */
- private static final String INVALID_HOST_1 = "invalid_host1";
-
- /** */
- private static final String INVALID_HOST_2 = "invalid_host2";
-
- /** */
- private static final String INVALID_HOST_3 = "invalid_host3";
-
- /** Mocked Grid. */
- private static final MockIgnite GRID = new MockIgnite();
-
- /** Mocked IGFS. */
- private static final IgniteFs IGFS = new MockIgfs();
-
- /** Planner. */
- private static final GridHadoopMapReducePlanner PLANNER = new GridHadoopDefaultMapReducePlanner();
-
- /** Block locations. */
- private static final Map<Block, Collection<IgfsBlockLocation>> BLOCK_MAP = new HashMap<>();
-
- /** Proxy map. */
- private static final Map<URI, Boolean> PROXY_MAP = new HashMap<>();
-
- /** Last created plan. */
- private static final ThreadLocal<GridHadoopMapReducePlan> PLAN = new ThreadLocal<>();
-
- /**
- *
- */
- static {
- GridTestUtils.setFieldValue(PLANNER, "ignite", GRID);
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- GridTestUtils.setFieldValue(PLANNER, "log", log());
-
- BLOCK_MAP.clear();
- PROXY_MAP.clear();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- public void testIgfsOneBlockPerNode() throws IgniteCheckedException {
- GridHadoopFileBlock split1 = split(true, "/file1", 0, 100, HOST_1);
- GridHadoopFileBlock split2 = split(true, "/file2", 0, 100, HOST_2);
- GridHadoopFileBlock split3 = split(true, "/file3", 0, 100, HOST_3);
-
- mapIgfsBlock(split1.file(), 0, 100, location(0, 100, ID_1));
- mapIgfsBlock(split2.file(), 0, 100, location(0, 100, ID_2));
- mapIgfsBlock(split3.file(), 0, 100, location(0, 100, ID_3));
-
- plan(1, split1);
- assert ensureMappers(ID_1, split1);
- assert ensureReducers(ID_1, 1);
- assert ensureEmpty(ID_2);
- assert ensureEmpty(ID_3);
-
- plan(2, split1);
- assert ensureMappers(ID_1, split1);
- assert ensureReducers(ID_1, 2);
- assert ensureEmpty(ID_2);
- assert ensureEmpty(ID_3);
-
- plan(1, split1, split2);
- assert ensureMappers(ID_1, split1);
- assert ensureMappers(ID_2, split2);
- assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 0) || ensureReducers(ID_1, 0) && ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
-
- plan(2, split1, split2);
- assert ensureMappers(ID_1, split1);
- assert ensureMappers(ID_2, split2);
- assert ensureReducers(ID_1, 1);
- assert ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
-
- plan(3, split1, split2);
- assert ensureMappers(ID_1, split1);
- assert ensureMappers(ID_2, split2);
- assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) || ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
-
- plan(3, split1, split2, split3);
- assert ensureMappers(ID_1, split1);
- assert ensureMappers(ID_2, split2);
- assert ensureMappers(ID_3, split3);
- assert ensureReducers(ID_1, 1);
- assert ensureReducers(ID_2, 1);
- assert ensureReducers(ID_3, 1);
-
- plan(5, split1, split2, split3);
- assert ensureMappers(ID_1, split1);
- assert ensureMappers(ID_2, split2);
- assert ensureMappers(ID_3, split3);
- assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 2) ||
- ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1) && ensureReducers(ID_3, 2) ||
- ensureReducers(ID_1, 2) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 1);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- public void testNonIgfsOneBlockPerNode() throws IgniteCheckedException {
- GridHadoopFileBlock split1 = split(false, "/file1", 0, 100, HOST_1);
- GridHadoopFileBlock split2 = split(false, "/file2", 0, 100, HOST_2);
- GridHadoopFileBlock split3 = split(false, "/file3", 0, 100, HOST_3);
-
- plan(1, split1);
- assert ensureMappers(ID_1, split1);
- assert ensureReducers(ID_1, 1);
- assert ensureEmpty(ID_2);
- assert ensureEmpty(ID_3);
-
- plan(2, split1);
- assert ensureMappers(ID_1, split1);
- assert ensureReducers(ID_1, 2);
- assert ensureEmpty(ID_2);
- assert ensureEmpty(ID_3);
-
- plan(1, split1, split2);
- assert ensureMappers(ID_1, split1);
- assert ensureMappers(ID_2, split2);
- assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 0) || ensureReducers(ID_1, 0) && ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
-
- plan(2, split1, split2);
- assert ensureMappers(ID_1, split1);
- assert ensureMappers(ID_2, split2);
- assert ensureReducers(ID_1, 1);
- assert ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
-
- plan(3, split1, split2);
- assert ensureMappers(ID_1, split1);
- assert ensureMappers(ID_2, split2);
- assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) || ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
-
- plan(3, split1, split2, split3);
- assert ensureMappers(ID_1, split1);
- assert ensureMappers(ID_2, split2);
- assert ensureMappers(ID_3, split3);
- assert ensureReducers(ID_1, 1);
- assert ensureReducers(ID_2, 1);
- assert ensureReducers(ID_3, 1);
-
- plan(5, split1, split2, split3);
- assert ensureMappers(ID_1, split1);
- assert ensureMappers(ID_2, split2);
- assert ensureMappers(ID_3, split3);
- assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 2) ||
- ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1) && ensureReducers(ID_3, 2) ||
- ensureReducers(ID_1, 2) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 1);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- public void testIgfsSeveralBlocksPerNode() throws IgniteCheckedException {
- GridHadoopFileBlock split1 = split(true, "/file1", 0, 100, HOST_1, HOST_2);
- GridHadoopFileBlock split2 = split(true, "/file2", 0, 100, HOST_1, HOST_2);
- GridHadoopFileBlock split3 = split(true, "/file3", 0, 100, HOST_1, HOST_3);
-
- mapIgfsBlock(split1.file(), 0, 100, location(0, 100, ID_1, ID_2));
- mapIgfsBlock(split2.file(), 0, 100, location(0, 100, ID_1, ID_2));
- mapIgfsBlock(split3.file(), 0, 100, location(0, 100, ID_1, ID_3));
-
- plan(1, split1);
- assert ensureMappers(ID_1, split1) && ensureReducers(ID_1, 1) && ensureEmpty(ID_2) ||
- ensureEmpty(ID_1) && ensureMappers(ID_2, split1) && ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
-
- plan(2, split1);
- assert ensureMappers(ID_1, split1) && ensureReducers(ID_1, 2) && ensureEmpty(ID_2) ||
- ensureEmpty(ID_1) && ensureMappers(ID_2, split1) && ensureReducers(ID_2, 2);
- assert ensureEmpty(ID_3);
-
- plan(1, split1, split2);
- assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) ||
- ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1);
- assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 0) || ensureReducers(ID_1, 0) && ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
-
- plan(2, split1, split2);
- assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) ||
- ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1);
- assert ensureReducers(ID_1, 1);
- assert ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
-
- plan(3, split1, split2, split3);
- assert ensureReducers(ID_1, 1);
- assert ensureReducers(ID_2, 1);
- assert ensureReducers(ID_3, 1);
-
- plan(5, split1, split2, split3);
- assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 2) ||
- ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1) && ensureReducers(ID_3, 2) ||
- ensureReducers(ID_1, 2) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 1);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- public void testNonIgfsSeveralBlocksPerNode() throws IgniteCheckedException {
- GridHadoopFileBlock split1 = split(false, "/file1", 0, 100, HOST_1, HOST_2);
- GridHadoopFileBlock split2 = split(false, "/file2", 0, 100, HOST_1, HOST_2);
- GridHadoopFileBlock split3 = split(false, "/file3", 0, 100, HOST_1, HOST_3);
-
- plan(1, split1);
- assert ensureMappers(ID_1, split1) && ensureReducers(ID_1, 1) && ensureEmpty(ID_2) ||
- ensureEmpty(ID_1) && ensureMappers(ID_2, split1) && ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
-
- plan(2, split1);
- assert ensureMappers(ID_1, split1) && ensureReducers(ID_1, 2) && ensureEmpty(ID_2) ||
- ensureEmpty(ID_1) && ensureMappers(ID_2, split1) && ensureReducers(ID_2, 2);
- assert ensureEmpty(ID_3);
-
- plan(1, split1, split2);
- assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) ||
- ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1);
- assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 0) || ensureReducers(ID_1, 0) && ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
-
- plan(2, split1, split2);
- assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) ||
- ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1);
- assert ensureReducers(ID_1, 1);
- assert ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
-
- plan(3, split1, split2, split3);
- assert ensureReducers(ID_1, 1);
- assert ensureReducers(ID_2, 1);
- assert ensureReducers(ID_3, 1);
-
- plan(5, split1, split2, split3);
- assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 2) ||
- ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1) && ensureReducers(ID_3, 2) ||
- ensureReducers(ID_1, 2) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 1);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- public void testIgfsSeveralComplexBlocksPerNode() throws IgniteCheckedException {
- GridHadoopFileBlock split1 = split(true, "/file1", 0, 100, HOST_1, HOST_2, HOST_3);
- GridHadoopFileBlock split2 = split(true, "/file2", 0, 100, HOST_1, HOST_2, HOST_3);
-
- mapIgfsBlock(split1.file(), 0, 100, location(0, 50, ID_1, ID_2), location(51, 100, ID_1, ID_3));
- mapIgfsBlock(split2.file(), 0, 100, location(0, 50, ID_1, ID_2), location(51, 100, ID_2, ID_3));
-
- plan(1, split1);
- assert ensureMappers(ID_1, split1);
- assert ensureReducers(ID_1, 1);
- assert ensureEmpty(ID_2);
- assert ensureEmpty(ID_3);
-
- plan(1, split2);
- assert ensureMappers(ID_2, split2);
- assert ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_1);
- assert ensureEmpty(ID_3);
-
- plan(1, split1, split2);
- assert ensureMappers(ID_1, split1);
- assert ensureMappers(ID_2, split2);
- assert ensureReducers(ID_1, 0) && ensureReducers(ID_2, 1) || ensureReducers(ID_1, 1) && ensureReducers(ID_2, 0);
- assert ensureEmpty(ID_3);
-
- plan(2, split1, split2);
- assert ensureMappers(ID_1, split1);
- assert ensureMappers(ID_2, split2);
- assert ensureReducers(ID_1, 1);
- assert ensureReducers(ID_2, 1);
- assert ensureEmpty(ID_3);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- public void testNonIgfsOrphans() throws IgniteCheckedException {
- GridHadoopFileBlock split1 = split(false, "/file1", 0, 100, INVALID_HOST_1, INVALID_HOST_2);
- GridHadoopFileBlock split2 = split(false, "/file2", 0, 100, INVALID_HOST_1, INVALID_HOST_3);
- GridHadoopFileBlock split3 = split(false, "/file3", 0, 100, INVALID_HOST_2, INVALID_HOST_3);
-
- plan(1, split1);
- assert ensureMappers(ID_1, split1) && ensureReducers(ID_1, 1) && ensureEmpty(ID_2) && ensureEmpty(ID_3) ||
- ensureEmpty(ID_1) && ensureMappers(ID_2, split1) && ensureReducers(ID_2, 1) && ensureEmpty(ID_3) ||
- ensureEmpty(ID_1) && ensureEmpty(ID_2) && ensureMappers(ID_3, split1) && ensureReducers(ID_3, 1);
-
- plan(2, split1);
- assert ensureMappers(ID_1, split1) && ensureReducers(ID_1, 2) && ensureEmpty(ID_2) && ensureEmpty(ID_3) ||
- ensureEmpty(ID_1) && ensureMappers(ID_2, split1) && ensureReducers(ID_2, 2) && ensureEmpty(ID_3) ||
- ensureEmpty(ID_1) && ensureEmpty(ID_2) && ensureMappers(ID_3, split1) && ensureReducers(ID_3, 2);
-
- plan(1, split1, split2, split3);
- assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) && ensureMappers(ID_3, split3) ||
- ensureMappers(ID_1, split1) && ensureMappers(ID_2, split3) && ensureMappers(ID_3, split2) ||
- ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1) && ensureMappers(ID_3, split3) ||
- ensureMappers(ID_1, split2) && ensureMappers(ID_2, split3) && ensureMappers(ID_3, split1) ||
- ensureMappers(ID_1, split3) && ensureMappers(ID_2, split1) && ensureMappers(ID_3, split2) ||
- ensureMappers(ID_1, split3) && ensureMappers(ID_2, split2) && ensureMappers(ID_3, split1);
- assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 0) && ensureReducers(ID_3, 0) ||
- ensureReducers(ID_1, 0) && ensureReducers(ID_2, 1) && ensureReducers(ID_3, 0) ||
- ensureReducers(ID_1, 0) && ensureReducers(ID_2, 0) && ensureReducers(ID_3, 1);
-
- plan(3, split1, split2, split3);
- assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) && ensureMappers(ID_3, split3) ||
- ensureMappers(ID_1, split1) && ensureMappers(ID_2, split3) && ensureMappers(ID_3, split2) ||
- ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1) && ensureMappers(ID_3, split3) ||
- ensureMappers(ID_1, split2) && ensureMappers(ID_2, split3) && ensureMappers(ID_3, split1) ||
- ensureMappers(ID_1, split3) && ensureMappers(ID_2, split1) && ensureMappers(ID_3, split2) ||
- ensureMappers(ID_1, split3) && ensureMappers(ID_2, split2) && ensureMappers(ID_3, split1);
- assert ensureReducers(ID_1, 1);
- assert ensureReducers(ID_2, 1);
- assert ensureReducers(ID_3, 1);
-
- plan(5, split1, split2, split3);
- assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) && ensureMappers(ID_3, split3) ||
- ensureMappers(ID_1, split1) && ensureMappers(ID_2, split3) && ensureMappers(ID_3, split2) ||
- ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1) && ensureMappers(ID_3, split3) ||
- ensureMappers(ID_1, split2) && ensureMappers(ID_2, split3) && ensureMappers(ID_3, split1) ||
- ensureMappers(ID_1, split3) && ensureMappers(ID_2, split1) && ensureMappers(ID_3, split2) ||
- ensureMappers(ID_1, split3) && ensureMappers(ID_2, split2) && ensureMappers(ID_3, split1);
- assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 2) ||
- ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1) && ensureReducers(ID_3, 2) ||
- ensureReducers(ID_1, 2) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 1);
- }
-
- /**
- * Create plan.
- *
- * @param reducers Reducers count.
- * @param splits Splits.
- * @return Plan.
- * @throws IgniteCheckedException If failed.
- */
- private static GridHadoopMapReducePlan plan(int reducers, GridHadoopInputSplit... splits) throws IgniteCheckedException {
- assert reducers > 0;
- assert splits != null && splits.length > 0;
-
- Collection<GridHadoopInputSplit> splitList = new ArrayList<>(splits.length);
-
- Collections.addAll(splitList, splits);
-
- Collection<ClusterNode> top = new ArrayList<>();
-
- GridTestNode node1 = new GridTestNode(ID_1);
- GridTestNode node2 = new GridTestNode(ID_2);
- GridTestNode node3 = new GridTestNode(ID_3);
-
- node1.setHostName(HOST_1);
- node2.setHostName(HOST_2);
- node3.setHostName(HOST_3);
-
- top.add(node1);
- top.add(node2);
- top.add(node3);
-
- GridHadoopMapReducePlan plan = PLANNER.preparePlan(new MockJob(reducers, splitList), top, null);
-
- PLAN.set(plan);
-
- return plan;
- }
-
- /**
- * Ensure that node contains the given mappers.
- *
- * @param nodeId Node ID.
- * @param expSplits Expected splits.
- * @return {@code True} if this assumption is valid.
- */
- private static boolean ensureMappers(UUID nodeId, GridHadoopInputSplit... expSplits) {
- Collection<GridHadoopInputSplit> expSplitsCol = new ArrayList<>();
-
- Collections.addAll(expSplitsCol, expSplits);
-
- Collection<GridHadoopInputSplit> splits = PLAN.get().mappers(nodeId);
-
- return F.eq(expSplitsCol, splits);
- }
-
- /**
- * Ensure that node contains the given amount of reducers.
- *
- * @param nodeId Node ID.
- * @param reducers Reducers.
- * @return {@code True} if this assumption is valid.
- */
- private static boolean ensureReducers(UUID nodeId, int reducers) {
- int[] reducersArr = PLAN.get().reducers(nodeId);
-
- return reducers == 0 ? F.isEmpty(reducersArr) : (reducersArr != null && reducersArr.length == reducers);
- }
-
- /**
- * Ensure that no mappers and reducers is located on this node.
- *
- * @param nodeId Node ID.
- * @return {@code True} if this assumption is valid.
- */
- private static boolean ensureEmpty(UUID nodeId) {
- return F.isEmpty(PLAN.get().mappers(nodeId)) && F.isEmpty(PLAN.get().reducers(nodeId));
- }
-
- /**
- * Create split.
- *
- * @param igfs IGFS flag.
- * @param file File.
- * @param start Start.
- * @param len Length.
- * @param hosts Hosts.
- * @return Split.
- */
- private static GridHadoopFileBlock split(boolean igfs, String file, long start, long len, String... hosts) {
- URI uri = URI.create((igfs ? "igfs://igfs@" : "hdfs://") + file);
-
- return new GridHadoopFileBlock(hosts, uri, start, len);
- }
-
- /**
- * Create block location.
- *
- * @param start Start.
- * @param len Length.
- * @param nodeIds Node IDs.
- * @return Block location.
- */
- private static IgfsBlockLocation location(long start, long len, UUID... nodeIds) {
- assert nodeIds != null && nodeIds.length > 0;
-
- Collection<ClusterNode> nodes = new ArrayList<>(nodeIds.length);
-
- for (UUID id : nodeIds)
- nodes.add(new GridTestNode(id));
-
- return new IgfsBlockLocationImpl(start, len, nodes);
- }
-
- /**
- * Map IGFS block to nodes.
- *
- * @param file File.
- * @param start Start.
- * @param len Length.
- * @param locations Locations.
- */
- private static void mapIgfsBlock(URI file, long start, long len, IgfsBlockLocation... locations) {
- assert locations != null && locations.length > 0;
-
- IgfsPath path = new IgfsPath(file);
-
- Block block = new Block(path, start, len);
-
- Collection<IgfsBlockLocation> locationsList = new ArrayList<>();
-
- Collections.addAll(locationsList, locations);
-
- BLOCK_MAP.put(block, locationsList);
- }
-
- /**
- * Block.
- */
- private static class Block {
- /** */
- private final IgfsPath path;
-
- /** */
- private final long start;
-
- /** */
- private final long len;
-
- /**
- * Constructor.
- *
- * @param path Path.
- * @param start Start.
- * @param len Length.
- */
- private Block(IgfsPath path, long start, long len) {
- this.path = path;
- this.start = start;
- this.len = len;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("RedundantIfStatement")
- @Override public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof Block)) return false;
-
- Block block = (Block) o;
-
- if (len != block.len)
- return false;
-
- if (start != block.start)
- return false;
-
- if (!path.equals(block.path))
- return false;
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = path.hashCode();
-
- res = 31 * res + (int) (start ^ (start >>> 32));
- res = 31 * res + (int) (len ^ (len >>> 32));
-
- return res;
- }
- }
-
- /**
- * Mocked job.
- */
- private static class MockJob implements GridHadoopJob {
- /** Reducers count. */
- private final int reducers;
-
- /** */
- private Collection<GridHadoopInputSplit> splitList;
-
- /**
- * Constructor.
- *
- * @param reducers Reducers count.
- * @param splitList Splits.
- */
- private MockJob(int reducers, Collection<GridHadoopInputSplit> splitList) {
- this.reducers = reducers;
- this.splitList = splitList;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopJobId id() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopJobInfo info() {
- return new GridHadoopDefaultJobInfo() {
- @Override public int reducers() {
- return reducers;
- }
- };
- }
-
- /** {@inheritDoc} */
- @Override public Collection<GridHadoopInputSplit> input() throws IgniteCheckedException {
- return splitList;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopTaskContext getTaskContext(GridHadoopTaskInfo info) throws IgniteCheckedException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void initialize(boolean external, UUID nodeId) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void dispose(boolean external) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void prepareTaskEnvironment(GridHadoopTaskInfo info) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void cleanupTaskEnvironment(GridHadoopTaskInfo info) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void cleanupStagingDirectory() {
- // No-op.
- }
- }
-
- /**
- * Mocked IGFS.
- */
- private static class MockIgfs implements IgfsEx {
- /** {@inheritDoc} */
- @Override public boolean isProxy(URI path) {
- return PROXY_MAP.containsKey(path) && PROXY_MAP.get(path);
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) {
- return BLOCK_MAP.get(new Block(path, start, len));
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len,
- long maxLen) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void stop() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public IgfsContext context() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsPaths proxyPaths() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsInputStreamAdapter open(IgfsPath path, int bufSize, int seqReadsBeforePrefetch) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsInputStreamAdapter open(IgfsPath path) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsInputStreamAdapter open(IgfsPath path, int bufSize) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsStatus globalSpace() throws IgniteCheckedException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void globalSampling(@Nullable Boolean val) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public Boolean globalSampling() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsLocalMetrics localMetrics() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public long groupBlockSize() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<?> awaitDeletesAsync() throws IgniteCheckedException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public String clientLogDirectory() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void clientLogDirectory(String logDir) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public boolean evictExclude(IgfsPath path, boolean primary) {
- return false;
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public String name() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsConfiguration configuration() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public boolean exists(IgfsPath path) {
- return false;
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public IgfsFile info(IgfsPath path) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsPathSummary summary(IgfsPath path) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public IgfsFile update(IgfsPath path, Map<String, String> props) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void rename(IgfsPath src, IgfsPath dest) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public boolean delete(IgfsPath path, boolean recursive) {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public void mkdirs(IgfsPath path) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsPath> listPaths(IgfsPath path) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsFile> listFiles(IgfsPath path) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public long usedSpaceSize() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public Map<String, String> properties() {
- return Collections.emptyMap();
- }
-
- /** {@inheritDoc} */
- @Override public IgfsOutputStream create(IgfsPath path, boolean overwrite) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
- long blockSize, @Nullable Map<String, String> props) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite,
- @Nullable IgniteUuid affKey, int replication, long blockSize, @Nullable Map<String, String> props) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsOutputStream append(IgfsPath path, boolean create) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsOutputStream append(IgfsPath path, int bufSize, boolean create,
- @Nullable Map<String, String> props) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void setTimes(IgfsPath path, long accessTime, long modificationTime) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public IgfsMetrics metrics() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void resetMetrics() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public long size(IgfsPath path) {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public void format() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
- Collection<IgfsPath> paths, @Nullable T arg) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
- Collection<IgfsPath> paths, boolean skipNonExistentFiles, long maxRangeLen, @Nullable T arg) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
- @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, @Nullable T arg) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
- @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles,
- long maxRangeLen, @Nullable T arg) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgniteUuid nextAffinityKey() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgniteFs withAsync() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public boolean isAsync() {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public <R> IgniteFuture<R> future() {
- return null;
- }
- }
-
- /**
- * Mocked Grid.
- */
- @SuppressWarnings("ExternalizableWithoutPublicNoArgConstructor")
- private static class MockIgnite extends IgniteSpringBean implements IgniteEx {
- /** {@inheritDoc} */
- @Override public IgniteClusterEx cluster() {
- return (IgniteClusterEx)super.cluster();
- }
-
- /** {@inheritDoc} */
- @Override public IgniteFs igfsx(String name) {
- assert F.eq("igfs", name);
-
- return IGFS;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoop hadoop() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public String name() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public <K extends GridCacheUtilityKey, V> GridCacheProjectionEx<K, V> utilityCache(Class<K> keyCls,
- Class<V> valCls) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public <K, V> GridCache<K, V> cachex(@Nullable String name) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public <K, V> GridCache<K, V> cachex() {
- return null;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public Collection<GridCache<?, ?>> cachesx(@Nullable IgnitePredicate<? super GridCache<?, ?>>... p) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public boolean eventUserRecordable(int type) {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public boolean allEventsUserRecordable(int[] types) {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public Collection<String> compatibleVersions() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public boolean isJmxRemoteEnabled() {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public boolean isRestartEnabled() {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public ClusterNode localNode() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public String latestVersion() {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopFileSystemsTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopFileSystemsTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopFileSystemsTest.java
deleted file mode 100644
index 18e5c03..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopFileSystemsTest.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.ignite.internal.processors.hadoop.fs.*;
-import org.apache.ignite.testframework.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-
-/**
- * Test file systems for the working directory multi-threading support.
- */
-public class GridHadoopFileSystemsTest extends GridHadoopAbstractSelfTest {
- private static final int THREAD_COUNT = 3;
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- startGrids(gridCount());
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids(true);
- }
-
- /** {@inheritDoc} */
- @Override protected boolean igfsEnabled() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override protected int gridCount() {
- return 1;
- }
-
-
- /**
- * Test the file system with specified URI for the multi-thread working directory support.
- *
- * @param uri Base URI of the file system (scheme and authority).
- * @throws Exception If fails.
- */
- private void testFileSystem(final URI uri) throws Exception {
- final Configuration cfg = new Configuration();
-
- setupFileSystems(cfg);
-
- cfg.set(GridHadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP,
- new Path(new Path(uri), "user/" + System.getProperty("user.name")).toString());
-
- final CountDownLatch changeUserPhase = new CountDownLatch(THREAD_COUNT);
- final CountDownLatch changeDirPhase = new CountDownLatch(THREAD_COUNT);
- final CountDownLatch changeAbsDirPhase = new CountDownLatch(THREAD_COUNT);
- final CountDownLatch finishPhase = new CountDownLatch(THREAD_COUNT);
-
- final Path[] newUserInitWorkDir = new Path[THREAD_COUNT];
- final Path[] newWorkDir = new Path[THREAD_COUNT];
- final Path[] newAbsWorkDir = new Path[THREAD_COUNT];
- final Path[] newInstanceWorkDir = new Path[THREAD_COUNT];
-
- final AtomicInteger threadNum = new AtomicInteger(0);
-
- GridTestUtils.runMultiThreadedAsync(new Runnable() {
- @Override public void run() {
- try {
- int curThreadNum = threadNum.getAndIncrement();
-
- FileSystem fs = FileSystem.get(uri, cfg);
-
- GridHadoopFileSystemsUtils.setUser(fs, "user" + curThreadNum);
-
- if ("file".equals(uri.getScheme()))
- FileSystem.get(uri, cfg).setWorkingDirectory(new Path("file:///user/user" + curThreadNum));
-
- changeUserPhase.countDown();
- changeUserPhase.await();
-
- newUserInitWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory();
-
- FileSystem.get(uri, cfg).setWorkingDirectory(new Path("folder" + curThreadNum));
-
- changeDirPhase.countDown();
- changeDirPhase.await();
-
- newWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory();
-
- FileSystem.get(uri, cfg).setWorkingDirectory(new Path("/folder" + curThreadNum));
-
- changeAbsDirPhase.countDown();
- changeAbsDirPhase.await();
-
- newAbsWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory();
-
- newInstanceWorkDir[curThreadNum] = FileSystem.newInstance(uri, cfg).getWorkingDirectory();
-
- finishPhase.countDown();
- }
- catch (InterruptedException | IOException e) {
- error("Failed to execute test thread.", e);
-
- fail();
- }
- }
- }, THREAD_COUNT, "filesystems-test");
-
- finishPhase.await();
-
- for (int i = 0; i < THREAD_COUNT; i ++) {
- cfg.set(MRJobConfig.USER_NAME, "user" + i);
-
- Path workDir = new Path(new Path(uri), "user/user" + i);
-
- cfg.set(GridHadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP, workDir.toString());
-
- assertEquals(workDir, FileSystem.newInstance(uri, cfg).getWorkingDirectory());
-
- assertEquals(workDir, newUserInitWorkDir[i]);
-
- assertEquals(new Path(new Path(uri), "user/user" + i + "/folder" + i), newWorkDir[i]);
-
- assertEquals(new Path("/folder" + i), newAbsWorkDir[i]);
-
- assertEquals(new Path(new Path(uri), "user/" + System.getProperty("user.name")), newInstanceWorkDir[i]);
- }
-
- System.out.println(System.getProperty("user.dir"));
- }
-
- /**
- * Test IGFS multi-thread working directory.
- *
- * @throws Exception If fails.
- */
- public void testIgfs() throws Exception {
- testFileSystem(URI.create(igfsScheme()));
- }
-
- /**
- * Test HDFS multi-thread working directory.
- *
- * @throws Exception If fails.
- */
- public void testHdfs() throws Exception {
- testFileSystem(URI.create("hdfs://localhost/"));
- }
-
- /**
- * Test LocalFS multi-thread working directory.
- *
- * @throws Exception If fails.
- */
- public void testLocal() throws Exception {
- testFileSystem(URI.create("file:///"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopGroupingTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopGroupingTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopGroupingTest.java
deleted file mode 100644
index 49099fc..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopGroupingTest.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.util.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Grouping test.
- */
-public class GridHadoopGroupingTest extends GridHadoopAbstractSelfTest {
- /** */
- private static final String PATH_OUTPUT = "/test-out";
-
- /** */
- private static final GridConcurrentHashSet<UUID> vals = GridHadoopSharedMap.map(GridHadoopGroupingTest.class)
- .put("vals", new GridConcurrentHashSet<UUID>());
-
- /** {@inheritDoc} */
- @Override protected int gridCount() {
- return 3;
- }
-
- /** {@inheritDoc} */
- protected boolean igfsEnabled() {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- startGrids(gridCount());
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids(true);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopConfiguration hadoopConfiguration(String gridName) {
- GridHadoopConfiguration cfg = super.hadoopConfiguration(gridName);
-
- cfg.setExternalExecution(false);
-
- return cfg;
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testGroupingReducer() throws Exception {
- doTestGrouping(false);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testGroupingCombiner() throws Exception {
- doTestGrouping(true);
- }
-
- /**
- * @param combiner With combiner.
- * @throws Exception If failed.
- */
- public void doTestGrouping(boolean combiner) throws Exception {
- vals.clear();
-
- Job job = Job.getInstance();
-
- job.setInputFormatClass(InFormat.class);
- job.setOutputFormatClass(OutFormat.class);
-
- job.setOutputKeyClass(YearTemperature.class);
- job.setOutputValueClass(Text.class);
-
- job.setMapperClass(Mapper.class);
-
- if (combiner) {
- job.setCombinerClass(MyReducer.class);
- job.setNumReduceTasks(0);
- job.setCombinerKeyGroupingComparatorClass(YearComparator.class);
- }
- else {
- job.setReducerClass(MyReducer.class);
- job.setNumReduceTasks(4);
- job.setGroupingComparatorClass(YearComparator.class);
- }
-
- grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 2),
- createJobInfo(job.getConfiguration())).get(30000);
-
- assertTrue(vals.isEmpty());
- }
-
- public static class MyReducer extends Reducer<YearTemperature, Text, Text, Object> {
- /** */
- int lastYear;
-
- @Override protected void reduce(YearTemperature key, Iterable<Text> vals0, Context context)
- throws IOException, InterruptedException {
- X.println("___ : " + context.getTaskAttemptID() + " --> " + key);
-
- Set<UUID> ids = new HashSet<>();
-
- for (Text val : vals0)
- assertTrue(ids.add(UUID.fromString(val.toString())));
-
- for (Text val : vals0)
- assertTrue(ids.remove(UUID.fromString(val.toString())));
-
- assertTrue(ids.isEmpty());
-
- assertTrue(key.year > lastYear);
-
- lastYear = key.year;
-
- for (Text val : vals0)
- assertTrue(vals.remove(UUID.fromString(val.toString())));
- }
- }
-
- public static class YearComparator implements RawComparator<YearTemperature> { // Grouping comparator.
- /** {@inheritDoc} */
- @Override public int compare(YearTemperature o1, YearTemperature o2) {
- return Integer.compare(o1.year, o2.year);
- }
-
- /** {@inheritDoc} */
- @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- throw new IllegalStateException();
- }
- }
-
- public static class YearTemperature implements WritableComparable<YearTemperature>, Cloneable {
- /** */
- private int year;
-
- /** */
- private int temperature;
-
- /** {@inheritDoc} */
- @Override public void write(DataOutput out) throws IOException {
- out.writeInt(year);
- out.writeInt(temperature);
- }
-
- /** {@inheritDoc} */
- @Override public void readFields(DataInput in) throws IOException {
- year = in.readInt();
- temperature = in.readInt();
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- throw new IllegalStateException();
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() { // To be partitioned by year.
- return year;
- }
-
- /** {@inheritDoc} */
- @Override public int compareTo(YearTemperature o) {
- int res = Integer.compare(year, o.year);
-
- if (res != 0)
- return res;
-
- // Sort comparator by year and temperature, to find max for year.
- return Integer.compare(o.temperature, temperature);
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(YearTemperature.class, this);
- }
- }
-
- public static class InFormat extends InputFormat<YearTemperature, Text> {
- /** {@inheritDoc} */
- @Override public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
- ArrayList<InputSplit> list = new ArrayList<>();
-
- for (int i = 0; i < 10; i++)
- list.add(new GridHadoopSortingTest.FakeSplit(20));
-
- return list;
- }
-
- /** {@inheritDoc} */
- @Override public RecordReader<YearTemperature, Text> createRecordReader(final InputSplit split,
- TaskAttemptContext context) throws IOException, InterruptedException {
- return new RecordReader<YearTemperature, Text>() {
- /** */
- int cnt;
-
- /** */
- Random rnd = new GridRandom();
-
- /** */
- YearTemperature key = new YearTemperature();
-
- /** */
- Text val = new Text();
-
- @Override public void initialize(InputSplit split, TaskAttemptContext context) {
- // No-op.
- }
-
- @Override public boolean nextKeyValue() throws IOException, InterruptedException {
- return cnt++ < split.getLength();
- }
-
- @Override public YearTemperature getCurrentKey() {
- key.year = 1990 + rnd.nextInt(10);
- key.temperature = 10 + rnd.nextInt(20);
-
- return key;
- }
-
- @Override public Text getCurrentValue() {
- UUID id = UUID.randomUUID();
-
- assertTrue(vals.add(id));
-
- val.set(id.toString());
-
- return val;
- }
-
- @Override public float getProgress() {
- return 0;
- }
-
- @Override public void close() {
- // No-op.
- }
- };
- }
- }
-
- /**
- *
- */
- public static class OutFormat extends OutputFormat {
- /** {@inheritDoc} */
- @Override public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
- return null;
- }
- }
-}
[29/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java
new file mode 100644
index 0000000..54e87dd
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.hadoop.fs.permission.*;
+import org.apache.ignite.*;
+
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
+
+/**
+ * Hadoop file system properties.
+ */
+public class HadoopIgfsProperties {
+ /** Username. */
+ private String usrName;
+
+ /** Group name. */
+ private String grpName;
+
+ /** Permissions. */
+ private FsPermission perm;
+
+ /**
+ * Constructor.
+ *
+ * @param props Properties.
+ * @throws IgniteException In case of error.
+ */
+ public HadoopIgfsProperties(Map<String, String> props) throws IgniteException {
+ usrName = props.get(PROP_USER_NAME);
+ grpName = props.get(PROP_GROUP_NAME);
+
+ String permStr = props.get(PROP_PERMISSION);
+
+ if (permStr != null) {
+ try {
+ perm = new FsPermission((short)Integer.parseInt(permStr, 8));
+ }
+ catch (NumberFormatException ignore) {
+ throw new IgniteException("Permissions cannot be parsed: " + permStr);
+ }
+ }
+ }
+
+ /**
+ * Get user name.
+ *
+ * @return User name.
+ */
+ public String userName() {
+ return usrName;
+ }
+
+ /**
+ * Get group name.
+ *
+ * @return Group name.
+ */
+ public String groupName() {
+ return grpName;
+ }
+
+ /**
+ * Get permission.
+ *
+ * @return Permission.
+ */
+ public FsPermission permission() {
+ return perm;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProxyInputStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProxyInputStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProxyInputStream.java
new file mode 100644
index 0000000..4530e64
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProxyInputStream.java
@@ -0,0 +1,335 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.internal.igfs.common.*;
+
+import java.io.*;
+
+/**
+ * Secondary Hadoop file system input stream wrapper.
+ */
+public class HadoopIgfsProxyInputStream extends InputStream implements Seekable, PositionedReadable {
+ /** Actual input stream to the secondary file system. */
+ private final FSDataInputStream is;
+
+ /** Client logger. */
+ private final IgfsLogger clientLog;
+
+ /** Log stream ID. */
+ private final long logStreamId;
+
+ /** Read time. */
+ private long readTime;
+
+ /** User time. */
+ private long userTime;
+
+ /** Last timestamp. */
+ private long lastTs;
+
+ /** Amount of read bytes. */
+ private long total;
+
+ /** Closed flag. */
+ private boolean closed;
+
+ /**
+ * Constructor.
+ *
+ * @param is Actual input stream to the secondary file system.
+ * @param clientLog Client log.
+ */
+ public HadoopIgfsProxyInputStream(FSDataInputStream is, IgfsLogger clientLog, long logStreamId) {
+ assert is != null;
+ assert clientLog != null;
+
+ this.is = is;
+ this.clientLog = clientLog;
+ this.logStreamId = logStreamId;
+
+ lastTs = System.nanoTime();
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int read(byte[] b) throws IOException {
+ readStart();
+
+ int res;
+
+ try {
+ res = is.read(b);
+ }
+ finally {
+ readEnd();
+ }
+
+ if (res != -1)
+ total += res;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int read(byte[] b, int off, int len) throws IOException {
+ readStart();
+
+ int res;
+
+ try {
+ res = super.read(b, off, len);
+ }
+ finally {
+ readEnd();
+ }
+
+ if (res != -1)
+ total += res;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized long skip(long n) throws IOException {
+ readStart();
+
+ long res;
+
+ try {
+ res = is.skip(n);
+ }
+ finally {
+ readEnd();
+ }
+
+ if (clientLog.isLogEnabled())
+ clientLog.logSkip(logStreamId, res);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int available() throws IOException {
+ readStart();
+
+ try {
+ return is.available();
+ }
+ finally {
+ readEnd();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void close() throws IOException {
+ if (!closed) {
+ closed = true;
+
+ readStart();
+
+ try {
+ is.close();
+ }
+ finally {
+ readEnd();
+ }
+
+ if (clientLog.isLogEnabled())
+ clientLog.logCloseIn(logStreamId, userTime, readTime, total);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void mark(int readLimit) {
+ readStart();
+
+ try {
+ is.mark(readLimit);
+ }
+ finally {
+ readEnd();
+ }
+
+ if (clientLog.isLogEnabled())
+ clientLog.logMark(logStreamId, readLimit);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void reset() throws IOException {
+ readStart();
+
+ try {
+ is.reset();
+ }
+ finally {
+ readEnd();
+ }
+
+ if (clientLog.isLogEnabled())
+ clientLog.logReset(logStreamId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized boolean markSupported() {
+ readStart();
+
+ try {
+ return is.markSupported();
+ }
+ finally {
+ readEnd();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int read() throws IOException {
+ readStart();
+
+ int res;
+
+ try {
+ res = is.read();
+ }
+ finally {
+ readEnd();
+ }
+
+ if (res != -1)
+ total++;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int read(long pos, byte[] buf, int off, int len) throws IOException {
+ readStart();
+
+ int res;
+
+ try {
+ res = is.read(pos, buf, off, len);
+ }
+ finally {
+ readEnd();
+ }
+
+ if (res != -1)
+ total += res;
+
+ if (clientLog.isLogEnabled())
+ clientLog.logRandomRead(logStreamId, pos, res);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void readFully(long pos, byte[] buf, int off, int len) throws IOException {
+ readStart();
+
+ try {
+ is.readFully(pos, buf, off, len);
+ }
+ finally {
+ readEnd();
+ }
+
+ total += len;
+
+ if (clientLog.isLogEnabled())
+ clientLog.logRandomRead(logStreamId, pos, len);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void readFully(long pos, byte[] buf) throws IOException {
+ readStart();
+
+ try {
+ is.readFully(pos, buf);
+ }
+ finally {
+ readEnd();
+ }
+
+ total += buf.length;
+
+ if (clientLog.isLogEnabled())
+ clientLog.logRandomRead(logStreamId, pos, buf.length);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void seek(long pos) throws IOException {
+ readStart();
+
+ try {
+ is.seek(pos);
+ }
+ finally {
+ readEnd();
+ }
+
+ if (clientLog.isLogEnabled())
+ clientLog.logSeek(logStreamId, pos);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized long getPos() throws IOException {
+ readStart();
+
+ try {
+ return is.getPos();
+ }
+ finally {
+ readEnd();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized boolean seekToNewSource(long targetPos) throws IOException {
+ readStart();
+
+ try {
+ return is.seekToNewSource(targetPos);
+ }
+ finally {
+ readEnd();
+ }
+ }
+
+ /**
+ * Read start.
+ */
+ private void readStart() {
+ long now = System.nanoTime();
+
+ userTime += now - lastTs;
+
+ lastTs = now;
+ }
+
+ /**
+ * Read end.
+ */
+ private void readEnd() {
+ long now = System.nanoTime();
+
+ readTime += now - lastTs;
+
+ lastTs = now;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProxyOutputStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProxyOutputStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProxyOutputStream.java
new file mode 100644
index 0000000..9ab552e
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProxyOutputStream.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.internal.igfs.common.*;
+
+import java.io.*;
+
+/**
+ * Secondary Hadoop file system output stream wrapper.
+ */
+public class HadoopIgfsProxyOutputStream extends OutputStream {
+ /** Actual output stream. */
+ private FSDataOutputStream os;
+
+ /** Client logger. */
+ private final IgfsLogger clientLog;
+
+ /** Log stream ID. */
+ private final long logStreamId;
+
+ /** Read time. */
+ private long writeTime;
+
+ /** User time. */
+ private long userTime;
+
+ /** Last timestamp. */
+ private long lastTs;
+
+ /** Amount of written bytes. */
+ private long total;
+
+ /** Closed flag. */
+ private boolean closed;
+
+ /**
+ * Constructor.
+ *
+ * @param os Actual output stream.
+ * @param clientLog Client logger.
+ * @param logStreamId Log stream ID.
+ */
+ public HadoopIgfsProxyOutputStream(FSDataOutputStream os, IgfsLogger clientLog, long logStreamId) {
+ assert os != null;
+ assert clientLog != null;
+
+ this.os = os;
+ this.clientLog = clientLog;
+ this.logStreamId = logStreamId;
+
+ lastTs = System.nanoTime();
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void write(int b) throws IOException {
+ writeStart();
+
+ try {
+ os.write(b);
+ }
+ finally {
+ writeEnd();
+ }
+
+ total++;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void write(byte[] b) throws IOException {
+ writeStart();
+
+ try {
+ os.write(b);
+ }
+ finally {
+ writeEnd();
+ }
+
+ total += b.length;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void write(byte[] b, int off, int len) throws IOException {
+ writeStart();
+
+ try {
+ os.write(b, off, len);
+ }
+ finally {
+ writeEnd();
+ }
+
+ total += len;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void flush() throws IOException {
+ writeStart();
+
+ try {
+ os.flush();
+ }
+ finally {
+ writeEnd();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void close() throws IOException {
+ if (!closed) {
+ closed = true;
+
+ writeStart();
+
+ try {
+ os.close();
+ }
+ finally {
+ writeEnd();
+ }
+
+ if (clientLog.isLogEnabled())
+ clientLog.logCloseOut(logStreamId, userTime, writeTime, total);
+ }
+ }
+
+ /**
+ * Read start.
+ */
+ private void writeStart() {
+ long now = System.nanoTime();
+
+ userTime += now - lastTs;
+
+ lastTs = now;
+ }
+
+ /**
+ * Read end.
+ */
+ private void writeEnd() {
+ long now = System.nanoTime();
+
+ writeTime += now - lastTs;
+
+ lastTs = now;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java
new file mode 100644
index 0000000..e921fd3
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.ignite.igfs.secondary.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+
+/**
+ * Secondary file system input stream wrapper which actually opens input stream only in case it is explicitly
+ * requested.
+ * <p>
+ * The class is expected to be used only from synchronized context and therefore is not tread-safe.
+ */
+public class HadoopIgfsSecondaryFileSystemPositionedReadable implements IgfsSecondaryFileSystemPositionedReadable {
+ /** Secondary file system. */
+ private final FileSystem fs;
+
+ /** Path to the file to open. */
+ private final Path path;
+
+ /** Buffer size. */
+ private final int bufSize;
+
+ /** Actual input stream. */
+ private FSDataInputStream in;
+
+ /** Cached error occurred during output stream open. */
+ private IOException err;
+
+ /** Flag indicating that the stream was already opened. */
+ private boolean opened;
+
+ /**
+ * Constructor.
+ *
+ * @param fs Secondary file system.
+ * @param path Path to the file to open.
+ * @param bufSize Buffer size.
+ */
+ public HadoopIgfsSecondaryFileSystemPositionedReadable(FileSystem fs, Path path, int bufSize) {
+ assert fs != null;
+ assert path != null;
+
+ this.fs = fs;
+ this.path = path;
+ this.bufSize = bufSize;
+ }
+
+ /** Get input stream. */
+ private PositionedReadable in() throws IOException {
+ if (opened) {
+ if (err != null)
+ throw err;
+ }
+ else {
+ opened = true;
+
+ try {
+ in = fs.open(path, bufSize);
+
+ if (in == null)
+ throw new IOException("Failed to open input stream (file system returned null): " + path);
+ }
+ catch (IOException e) {
+ err = e;
+
+ throw err;
+ }
+ }
+
+ return in;
+ }
+
+ /**
+ * Close wrapped input stream in case it was previously opened.
+ */
+ @Override public void close() {
+ U.closeQuiet(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(long pos, byte[] buf, int off, int len) throws IOException {
+ return in().read(pos, buf, off, len);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsStreamDelegate.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsStreamDelegate.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsStreamDelegate.java
new file mode 100644
index 0000000..54f7377
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsStreamDelegate.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+/**
+ * IGFS Hadoop stream descriptor.
+ */
+public class HadoopIgfsStreamDelegate {
+ /** RPC handler. */
+ private final HadoopIgfsEx hadoop;
+
+ /** Target. */
+ private final Object target;
+
+ /** Optional stream length. */
+ private final long len;
+
+ /**
+ * Constructor.
+ *
+ * @param target Target.
+ */
+ public HadoopIgfsStreamDelegate(HadoopIgfsEx hadoop, Object target) {
+ this(hadoop, target, -1);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param target Target.
+ * @param len Optional length.
+ */
+ public HadoopIgfsStreamDelegate(HadoopIgfsEx hadoop, Object target, long len) {
+ assert hadoop != null;
+ assert target != null;
+
+ this.hadoop = hadoop;
+ this.target = target;
+ this.len = len;
+ }
+
+ /**
+ * @return RPC handler.
+ */
+ public HadoopIgfsEx hadoop() {
+ return hadoop;
+ }
+
+ /**
+ * @return Stream target.
+ */
+ @SuppressWarnings("unchecked")
+ public <T> T target() {
+ return (T) target;
+ }
+
+ /**
+ * @return Length.
+ */
+ public long length() {
+ return len;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return System.identityHashCode(target);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ return obj != null && obj instanceof HadoopIgfsStreamDelegate &&
+ target == ((HadoopIgfsStreamDelegate)obj).target;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopIgfsStreamDelegate.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsStreamEventListener.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsStreamEventListener.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsStreamEventListener.java
new file mode 100644
index 0000000..6b3fa82
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsStreamEventListener.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.ignite.*;
+
+/**
+ * IGFS input stream event listener.
+ */
+public interface HadoopIgfsStreamEventListener {
+ /**
+ * Callback invoked when the stream is being closed.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public void onClose() throws IgniteCheckedException;
+
+ /**
+ * Callback invoked when remote error occurs.
+ *
+ * @param errMsg Error message.
+ */
+ public void onError(String errMsg);
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java
new file mode 100644
index 0000000..e30a4ec
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * Utility constants and methods for IGFS Hadoop file system.
+ */
+public class HadoopIgfsUtils {
+ /** Parameter name for endpoint no embed mode flag. */
+ public static final String PARAM_IGFS_ENDPOINT_NO_EMBED = "fs.igfs.%s.endpoint.no_embed";
+
+ /** Parameter name for endpoint no shared memory flag. */
+ public static final String PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM = "fs.igfs.%s.endpoint.no_local_shmem";
+
+ /** Parameter name for endpoint no local TCP flag. */
+ public static final String PARAM_IGFS_ENDPOINT_NO_LOCAL_TCP = "fs.igfs.%s.endpoint.no_local_tcp";
+
+ /**
+ * Get string parameter.
+ *
+ * @param cfg Configuration.
+ * @param name Parameter name.
+ * @param authority Authority.
+ * @param dflt Default value.
+ * @return String value.
+ */
+ public static String parameter(Configuration cfg, String name, String authority, String dflt) {
+ return cfg.get(String.format(name, authority != null ? authority : ""), dflt);
+ }
+
+ /**
+ * Get integer parameter.
+ *
+ * @param cfg Configuration.
+ * @param name Parameter name.
+ * @param authority Authority.
+ * @param dflt Default value.
+ * @return Integer value.
+ * @throws IOException In case of parse exception.
+ */
+ public static int parameter(Configuration cfg, String name, String authority, int dflt) throws IOException {
+ String name0 = String.format(name, authority != null ? authority : "");
+
+ try {
+ return cfg.getInt(name0, dflt);
+ }
+ catch (NumberFormatException ignore) {
+ throw new IOException("Failed to parse parameter value to integer: " + name0);
+ }
+ }
+
+ /**
+ * Get boolean parameter.
+ *
+ * @param cfg Configuration.
+ * @param name Parameter name.
+ * @param authority Authority.
+ * @param dflt Default value.
+ * @return Boolean value.
+ */
+ public static boolean parameter(Configuration cfg, String name, String authority, boolean dflt) {
+ return cfg.getBoolean(String.format(name, authority != null ? authority : ""), dflt);
+ }
+
+ /**
+ * Cast Ignite exception to appropriate IO exception.
+ *
+ * @param e Exception to cast.
+ * @return Casted exception.
+ */
+ public static IOException cast(IgniteCheckedException e) {
+ return cast(e, null);
+ }
+
+ /**
+ * Cast Ignite exception to appropriate IO exception.
+ *
+ * @param e Exception to cast.
+ * @param path Path for exceptions.
+ * @return Casted exception.
+ */
+ @SuppressWarnings("unchecked")
+ public static IOException cast(IgniteCheckedException e, @Nullable String path) {
+ assert e != null;
+
+ // First check for any nested IOException; if exists - re-throw it.
+ if (e.hasCause(IOException.class))
+ return e.getCause(IOException.class);
+ else if (e.hasCause(IgfsFileNotFoundException.class))
+ return new FileNotFoundException(path); // TODO: Or PathNotFoundException?
+ else if (e.hasCause(IgfsParentNotDirectoryException.class))
+ return new ParentNotDirectoryException(path);
+ else if (path != null && e.hasCause(IgfsDirectoryNotEmptyException.class))
+ return new PathIsNotEmptyDirectoryException(path);
+ else if (path != null && e.hasCause(IgfsPathAlreadyExistsException.class))
+ return new PathExistsException(path);
+ else
+ return new IOException(e);
+ }
+
+ /**
+ * Constructor.
+ */
+ private HadoopIgfsUtils() {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsWrapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsWrapper.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsWrapper.java
new file mode 100644
index 0000000..1dada21
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsWrapper.java
@@ -0,0 +1,511 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.commons.logging.*;
+import org.apache.hadoop.conf.*;
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint.*;
+import static org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils.*;
+
+/**
+ * Wrapper for IGFS server.
+ */
+public class HadoopIgfsWrapper implements HadoopIgfs {
+ /** Delegate. */
+ private final AtomicReference<Delegate> delegateRef = new AtomicReference<>();
+
+ /** Authority. */
+ private final String authority;
+
+ /** Connection string. */
+ private final HadoopIgfsEndpoint endpoint;
+
+ /** Log directory. */
+ private final String logDir;
+
+ /** Configuration. */
+ private final Configuration conf;
+
+ /** Logger. */
+ private final Log log;
+
+ /**
+ * Constructor.
+ *
+ * @param authority Authority (connection string).
+ * @param logDir Log directory for server.
+ * @param conf Configuration.
+ * @param log Current logger.
+ */
+ public HadoopIgfsWrapper(String authority, String logDir, Configuration conf, Log log) throws IOException {
+ try {
+ this.authority = authority;
+ this.endpoint = new HadoopIgfsEndpoint(authority);
+ this.logDir = logDir;
+ this.conf = conf;
+ this.log = log;
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException("Failed to parse endpoint: " + authority, e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsHandshakeResponse handshake(String logDir) throws IOException {
+ return withReconnectHandling(new FileSystemClosure<IgfsHandshakeResponse>() {
+ @Override public IgfsHandshakeResponse apply(HadoopIgfsEx hadoop,
+ IgfsHandshakeResponse hndResp) {
+ return hndResp;
+ }
+ });
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close(boolean force) {
+ Delegate delegate = delegateRef.get();
+
+ if (delegate != null && delegateRef.compareAndSet(delegate, null))
+ delegate.close(force);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsFile info(final IgfsPath path) throws IOException {
+ return withReconnectHandling(new FileSystemClosure<IgfsFile>() {
+ @Override public IgfsFile apply(HadoopIgfsEx hadoop, IgfsHandshakeResponse hndResp)
+ throws IgniteCheckedException, IOException {
+ return hadoop.info(path);
+ }
+ }, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsFile update(final IgfsPath path, final Map<String, String> props) throws IOException {
+ return withReconnectHandling(new FileSystemClosure<IgfsFile>() {
+ @Override public IgfsFile apply(HadoopIgfsEx hadoop, IgfsHandshakeResponse hndResp)
+ throws IgniteCheckedException, IOException {
+ return hadoop.update(path, props);
+ }
+ }, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Boolean setTimes(final IgfsPath path, final long accessTime, final long modificationTime)
+ throws IOException {
+ return withReconnectHandling(new FileSystemClosure<Boolean>() {
+ @Override public Boolean apply(HadoopIgfsEx hadoop, IgfsHandshakeResponse hndResp)
+ throws IgniteCheckedException, IOException {
+ return hadoop.setTimes(path, accessTime, modificationTime);
+ }
+ }, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Boolean rename(final IgfsPath src, final IgfsPath dest) throws IOException {
+ return withReconnectHandling(new FileSystemClosure<Boolean>() {
+ @Override public Boolean apply(HadoopIgfsEx hadoop, IgfsHandshakeResponse hndResp)
+ throws IgniteCheckedException, IOException {
+ return hadoop.rename(src, dest);
+ }
+ }, src);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Boolean delete(final IgfsPath path, final boolean recursive) throws IOException {
+ return withReconnectHandling(new FileSystemClosure<Boolean>() {
+ @Override public Boolean apply(HadoopIgfsEx hadoop, IgfsHandshakeResponse hndResp)
+ throws IgniteCheckedException, IOException {
+ return hadoop.delete(path, recursive);
+ }
+ }, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsBlockLocation> affinity(final IgfsPath path, final long start,
+ final long len) throws IOException {
+ return withReconnectHandling(new FileSystemClosure<Collection<IgfsBlockLocation>>() {
+ @Override public Collection<IgfsBlockLocation> apply(HadoopIgfsEx hadoop,
+ IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
+ return hadoop.affinity(path, start, len);
+ }
+ }, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsPathSummary contentSummary(final IgfsPath path) throws IOException {
+ return withReconnectHandling(new FileSystemClosure<IgfsPathSummary>() {
+ @Override public IgfsPathSummary apply(HadoopIgfsEx hadoop, IgfsHandshakeResponse hndResp)
+ throws IgniteCheckedException, IOException {
+ return hadoop.contentSummary(path);
+ }
+ }, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Boolean mkdirs(final IgfsPath path, final Map<String, String> props) throws IOException {
+ return withReconnectHandling(new FileSystemClosure<Boolean>() {
+ @Override public Boolean apply(HadoopIgfsEx hadoop, IgfsHandshakeResponse hndResp)
+ throws IgniteCheckedException, IOException {
+ return hadoop.mkdirs(path, props);
+ }
+ }, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsFile> listFiles(final IgfsPath path) throws IOException {
+ return withReconnectHandling(new FileSystemClosure<Collection<IgfsFile>>() {
+ @Override public Collection<IgfsFile> apply(HadoopIgfsEx hadoop,
+ IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
+ return hadoop.listFiles(path);
+ }
+ }, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsPath> listPaths(final IgfsPath path) throws IOException {
+ return withReconnectHandling(new FileSystemClosure<Collection<IgfsPath>>() {
+ @Override public Collection<IgfsPath> apply(HadoopIgfsEx hadoop,
+ IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
+ return hadoop.listPaths(path);
+ }
+ }, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsStatus fsStatus() throws IOException {
+ return withReconnectHandling(new FileSystemClosure<IgfsStatus>() {
+ @Override public IgfsStatus apply(HadoopIgfsEx hadoop, IgfsHandshakeResponse hndResp)
+ throws IgniteCheckedException, IOException {
+ return hadoop.fsStatus();
+ }
+ });
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopIgfsStreamDelegate open(final IgfsPath path) throws IOException {
+ return withReconnectHandling(new FileSystemClosure<HadoopIgfsStreamDelegate>() {
+ @Override public HadoopIgfsStreamDelegate apply(HadoopIgfsEx hadoop,
+ IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
+ return hadoop.open(path);
+ }
+ }, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopIgfsStreamDelegate open(final IgfsPath path, final int seqReadsBeforePrefetch)
+ throws IOException {
+ return withReconnectHandling(new FileSystemClosure<HadoopIgfsStreamDelegate>() {
+ @Override public HadoopIgfsStreamDelegate apply(HadoopIgfsEx hadoop,
+ IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
+ return hadoop.open(path, seqReadsBeforePrefetch);
+ }
+ }, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopIgfsStreamDelegate create(final IgfsPath path, final boolean overwrite,
+ final boolean colocate, final int replication, final long blockSize, @Nullable final Map<String, String> props)
+ throws IOException {
+ return withReconnectHandling(new FileSystemClosure<HadoopIgfsStreamDelegate>() {
+ @Override public HadoopIgfsStreamDelegate apply(HadoopIgfsEx hadoop,
+ IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
+ return hadoop.create(path, overwrite, colocate, replication, blockSize, props);
+ }
+ }, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopIgfsStreamDelegate append(final IgfsPath path, final boolean create,
+ @Nullable final Map<String, String> props) throws IOException {
+ return withReconnectHandling(new FileSystemClosure<HadoopIgfsStreamDelegate>() {
+ @Override public HadoopIgfsStreamDelegate apply(HadoopIgfsEx hadoop,
+ IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
+ return hadoop.append(path, create, props);
+ }
+ }, path);
+ }
+
+ /**
+ * Execute closure which is not path-specific.
+ *
+ * @param clo Closure.
+ * @return Result.
+ * @throws IOException If failed.
+ */
+ private <T> T withReconnectHandling(FileSystemClosure<T> clo) throws IOException {
+ return withReconnectHandling(clo, null);
+ }
+
+ /**
+ * Execute closure.
+ *
+ * @param clo Closure.
+ * @param path Path for exceptions.
+ * @return Result.
+ * @throws IOException If failed.
+ */
+ private <T> T withReconnectHandling(final FileSystemClosure<T> clo, @Nullable IgfsPath path)
+ throws IOException {
+ Exception err = null;
+
+ for (int i = 0; i < 2; i++) {
+ Delegate curDelegate = null;
+
+ boolean close = false;
+ boolean force = false;
+
+ try {
+ curDelegate = delegate();
+
+ assert curDelegate != null;
+
+ close = curDelegate.doomed;
+
+ return clo.apply(curDelegate.hadoop, curDelegate.hndResp);
+ }
+ catch (HadoopIgfsCommunicationException e) {
+ if (curDelegate != null && !curDelegate.doomed) {
+ // Try getting rid fo faulty delegate ASAP.
+ delegateRef.compareAndSet(curDelegate, null);
+
+ close = true;
+ force = true;
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Failed to send message to a server: " + e);
+
+ err = e;
+ }
+ catch (IgniteCheckedException e) {
+ throw HadoopIgfsUtils.cast(e, path != null ? path.toString() : null);
+ }
+ finally {
+ if (close) {
+ assert curDelegate != null;
+
+ curDelegate.close(force);
+ }
+ }
+ }
+
+ throw new IOException("Failed to communicate with IGFS.", err);
+ }
+
+ /**
+ * Get delegate creating it if needed.
+ *
+ * @return Delegate.
+ */
+ private Delegate delegate() throws HadoopIgfsCommunicationException {
+ Exception err = null;
+
+ // 1. If delegate is set, return it immediately.
+ Delegate curDelegate = delegateRef.get();
+
+ if (curDelegate != null)
+ return curDelegate;
+
+ // 2. Guess that we are in the same VM.
+ if (!parameter(conf, PARAM_IGFS_ENDPOINT_NO_EMBED, authority, false)) {
+ IgfsEx igfs = null;
+
+ if (endpoint.grid() == null) {
+ try {
+ Ignite ignite = G.ignite();
+
+ igfs = (IgfsEx)ignite.fileSystem(endpoint.igfs());
+ }
+ catch (Exception e) {
+ err = e;
+ }
+ }
+ else {
+ for (Ignite ignite : G.allGrids()) {
+ try {
+ igfs = (IgfsEx)ignite.fileSystem(endpoint.igfs());
+
+ break;
+ }
+ catch (Exception e) {
+ err = e;
+ }
+ }
+ }
+
+ if (igfs != null) {
+ HadoopIgfsEx hadoop = null;
+
+ try {
+ hadoop = new HadoopIgfsInProc(igfs, log);
+
+ curDelegate = new Delegate(hadoop, hadoop.handshake(logDir));
+ }
+ catch (IOException | IgniteCheckedException e) {
+ if (e instanceof HadoopIgfsCommunicationException)
+ hadoop.close(true);
+
+ if (log.isDebugEnabled())
+ log.debug("Failed to connect to in-proc IGFS, fallback to IPC mode.", e);
+
+ err = e;
+ }
+ }
+ }
+
+ // 3. Try connecting using shmem.
+ if (!parameter(conf, PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority, false)) {
+ if (curDelegate == null && !U.isWindows()) {
+ HadoopIgfsEx hadoop = null;
+
+ try {
+ hadoop = new HadoopIgfsOutProc(endpoint.port(), endpoint.grid(), endpoint.igfs(), log);
+
+ curDelegate = new Delegate(hadoop, hadoop.handshake(logDir));
+ }
+ catch (IOException | IgniteCheckedException e) {
+ if (e instanceof HadoopIgfsCommunicationException)
+ hadoop.close(true);
+
+ if (log.isDebugEnabled())
+ log.debug("Failed to connect to out-proc local IGFS using shmem.", e);
+
+ err = e;
+ }
+ }
+ }
+
+ // 4. Try local TCP connection.
+ boolean skipLocTcp = parameter(conf, PARAM_IGFS_ENDPOINT_NO_LOCAL_TCP, authority, false);
+
+ if (!skipLocTcp) {
+ if (curDelegate == null) {
+ HadoopIgfsEx hadoop = null;
+
+ try {
+ hadoop = new HadoopIgfsOutProc(LOCALHOST, endpoint.port(), endpoint.grid(), endpoint.igfs(),
+ log);
+
+ curDelegate = new Delegate(hadoop, hadoop.handshake(logDir));
+ }
+ catch (IOException | IgniteCheckedException e) {
+ if (e instanceof HadoopIgfsCommunicationException)
+ hadoop.close(true);
+
+ if (log.isDebugEnabled())
+ log.debug("Failed to connect to out-proc local IGFS using TCP.", e);
+
+ err = e;
+ }
+ }
+ }
+
+ // 5. Try remote TCP connection.
+ if (curDelegate == null && (skipLocTcp || !F.eq(LOCALHOST, endpoint.host()))) {
+ HadoopIgfsEx hadoop = null;
+
+ try {
+ hadoop = new HadoopIgfsOutProc(endpoint.host(), endpoint.port(), endpoint.grid(), endpoint.igfs(), log);
+
+ curDelegate = new Delegate(hadoop, hadoop.handshake(logDir));
+ }
+ catch (IOException | IgniteCheckedException e) {
+ if (e instanceof HadoopIgfsCommunicationException)
+ hadoop.close(true);
+
+ if (log.isDebugEnabled())
+ log.debug("Failed to connect to out-proc remote IGFS using TCP.", e);
+
+ err = e;
+ }
+ }
+
+ if (curDelegate != null) {
+ if (!delegateRef.compareAndSet(null, curDelegate))
+ curDelegate.doomed = true;
+
+ return curDelegate;
+ }
+ else
+ throw new HadoopIgfsCommunicationException("Failed to connect to IGFS: " + endpoint, err);
+ }
+
+ /**
+ * File system operation closure.
+ */
+ private static interface FileSystemClosure<T> {
+ /**
+ * Call closure body.
+ *
+ * @param hadoop RPC handler.
+ * @param hndResp Handshake response.
+ * @return Result.
+ * @throws IgniteCheckedException If failed.
+ * @throws IOException If failed.
+ */
+ public T apply(HadoopIgfsEx hadoop, IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException;
+ }
+
+ /**
+ * Delegate.
+ */
+ private static class Delegate {
+ /** RPC handler. */
+ private final HadoopIgfsEx hadoop;
+
+ /** Handshake request. */
+ private final IgfsHandshakeResponse hndResp;
+
+ /** Close guard. */
+ private final AtomicBoolean closeGuard = new AtomicBoolean();
+
+ /** Whether this delegate must be closed at the end of the next invocation. */
+ private boolean doomed;
+
+ /**
+ * Constructor.
+ *
+ * @param hadoop Hadoop.
+ * @param hndResp Handshake response.
+ */
+ private Delegate(HadoopIgfsEx hadoop, IgfsHandshakeResponse hndResp) {
+ this.hadoop = hadoop;
+ this.hndResp = hndResp;
+ }
+
+ /**
+ * Close underlying RPC handler.
+ *
+ * @param force Force flag.
+ */
+ private void close(boolean force) {
+ if (closeGuard.compareAndSet(false, true))
+ hadoop.close(force);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobMetadata.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobMetadata.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobMetadata.java
deleted file mode 100644
index b124312..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobMetadata.java
+++ /dev/null
@@ -1,305 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.jobtracker;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.counter.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
-import org.apache.ignite.internal.util.tostring.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.util.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopJobPhase.*;
-
-/**
- * Hadoop job metadata. Internal object used for distributed job state tracking.
- */
-public class GridHadoopJobMetadata implements Externalizable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Job ID. */
- private GridHadoopJobId jobId;
-
- /** Job info. */
- private GridHadoopJobInfo jobInfo;
-
- /** Node submitted job. */
- private UUID submitNodeId;
-
- /** Map-reduce plan. */
- private GridHadoopMapReducePlan mrPlan;
-
- /** Pending splits for which mapper should be executed. */
- private Map<GridHadoopInputSplit, Integer> pendingSplits;
-
- /** Pending reducers. */
- private Collection<Integer> pendingReducers;
-
- /** Reducers addresses. */
- @GridToStringInclude
- private Map<Integer, GridHadoopProcessDescriptor> reducersAddrs;
-
- /** Job phase. */
- private GridHadoopJobPhase phase = PHASE_SETUP;
-
- /** Fail cause. */
- @GridToStringExclude
- private Throwable failCause;
-
- /** Version. */
- private long ver;
-
- /** Job counters */
- private GridHadoopCounters counters = new GridHadoopCountersImpl();
-
- /**
- * Empty constructor required by {@link Externalizable}.
- */
- public GridHadoopJobMetadata() {
- // No-op.
- }
-
- /**
- * Constructor.
- *
- * @param submitNodeId Submit node ID.
- * @param jobId Job ID.
- * @param jobInfo Job info.
- */
- public GridHadoopJobMetadata(UUID submitNodeId, GridHadoopJobId jobId, GridHadoopJobInfo jobInfo) {
- this.jobId = jobId;
- this.jobInfo = jobInfo;
- this.submitNodeId = submitNodeId;
- }
-
- /**
- * Copy constructor.
- *
- * @param src Metadata to copy.
- */
- public GridHadoopJobMetadata(GridHadoopJobMetadata src) {
- // Make sure to preserve alphabetic order.
- counters = src.counters;
- failCause = src.failCause;
- jobId = src.jobId;
- jobInfo = src.jobInfo;
- mrPlan = src.mrPlan;
- pendingSplits = src.pendingSplits;
- pendingReducers = src.pendingReducers;
- phase = src.phase;
- reducersAddrs = src.reducersAddrs;
- submitNodeId = src.submitNodeId;
- ver = src.ver + 1;
- }
-
- /**
- * @return Submit node ID.
- */
- public UUID submitNodeId() {
- return submitNodeId;
- }
-
- /**
- * @param phase Job phase.
- */
- public void phase(GridHadoopJobPhase phase) {
- this.phase = phase;
- }
-
- /**
- * @return Job phase.
- */
- public GridHadoopJobPhase phase() {
- return phase;
- }
-
- /**
- * Gets reducers addresses for external execution.
- *
- * @return Reducers addresses.
- */
- public Map<Integer, GridHadoopProcessDescriptor> reducersAddresses() {
- return reducersAddrs;
- }
-
- /**
- * Sets reducers addresses for external execution.
- *
- * @param reducersAddrs Map of addresses.
- */
- public void reducersAddresses(Map<Integer, GridHadoopProcessDescriptor> reducersAddrs) {
- this.reducersAddrs = reducersAddrs;
- }
-
- /**
- * Sets collection of pending splits.
- *
- * @param pendingSplits Collection of pending splits.
- */
- public void pendingSplits(Map<GridHadoopInputSplit, Integer> pendingSplits) {
- this.pendingSplits = pendingSplits;
- }
-
- /**
- * Gets collection of pending splits.
- *
- * @return Collection of pending splits.
- */
- public Map<GridHadoopInputSplit, Integer> pendingSplits() {
- return pendingSplits;
- }
-
- /**
- * Sets collection of pending reducers.
- *
- * @param pendingReducers Collection of pending reducers.
- */
- public void pendingReducers(Collection<Integer> pendingReducers) {
- this.pendingReducers = pendingReducers;
- }
-
- /**
- * Gets collection of pending reducers.
- *
- * @return Collection of pending reducers.
- */
- public Collection<Integer> pendingReducers() {
- return pendingReducers;
- }
-
- /**
- * @return Job ID.
- */
- public GridHadoopJobId jobId() {
- return jobId;
- }
-
- /**
- * @param mrPlan Map-reduce plan.
- */
- public void mapReducePlan(GridHadoopMapReducePlan mrPlan) {
- assert this.mrPlan == null : "Map-reduce plan can only be initialized once.";
-
- this.mrPlan = mrPlan;
- }
-
- /**
- * @return Map-reduce plan.
- */
- public GridHadoopMapReducePlan mapReducePlan() {
- return mrPlan;
- }
-
- /**
- * @return Job info.
- */
- public GridHadoopJobInfo jobInfo() {
- return jobInfo;
- }
-
- /**
- * Returns job counters.
- *
- * @return Collection of counters.
- */
- public GridHadoopCounters counters() {
- return counters;
- }
-
- /**
- * Sets counters.
- *
- * @param counters Collection of counters.
- */
- public void counters(GridHadoopCounters counters) {
- this.counters = counters;
- }
-
- /**
- * @param failCause Fail cause.
- */
- public void failCause(Throwable failCause) {
- assert failCause != null;
-
- if (this.failCause == null) // Keep the first error.
- this.failCause = failCause;
- }
-
- /**
- * @return Fail cause.
- */
- public Throwable failCause() {
- return failCause;
- }
-
- /**
- * @return Version.
- */
- public long version() {
- return ver;
- }
-
- /**
- * @param split Split.
- * @return Task number.
- */
- public int taskNumber(GridHadoopInputSplit split) {
- return pendingSplits.get(split);
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- U.writeUuid(out, submitNodeId);
- out.writeObject(jobId);
- out.writeObject(jobInfo);
- out.writeObject(mrPlan);
- out.writeObject(pendingSplits);
- out.writeObject(pendingReducers);
- out.writeObject(phase);
- out.writeObject(failCause);
- out.writeLong(ver);
- out.writeObject(reducersAddrs);
- out.writeObject(counters);
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- submitNodeId = U.readUuid(in);
- jobId = (GridHadoopJobId)in.readObject();
- jobInfo = (GridHadoopJobInfo)in.readObject();
- mrPlan = (GridHadoopMapReducePlan)in.readObject();
- pendingSplits = (Map<GridHadoopInputSplit,Integer>)in.readObject();
- pendingReducers = (Collection<Integer>)in.readObject();
- phase = (GridHadoopJobPhase)in.readObject();
- failCause = (Throwable)in.readObject();
- ver = in.readLong();
- reducersAddrs = (Map<Integer, GridHadoopProcessDescriptor>)in.readObject();
- counters = (GridHadoopCounters)in.readObject();
- }
-
- /** {@inheritDoc} */
- public String toString() {
- return S.toString(GridHadoopJobMetadata.class, this, "pendingMaps", pendingSplits.size(),
- "pendingReduces", pendingReducers.size(), "failCause", failCause == null ? null :
- failCause.getClass().getName());
- }
-}
[16/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopExternalSplit.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopExternalSplit.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopExternalSplit.java
new file mode 100644
index 0000000..496a710
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopExternalSplit.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.ignite.internal.processors.hadoop.*;
+
+import java.io.*;
+
+/**
+ * Split serialized in external file.
+ */
+public class HadoopExternalSplit extends HadoopInputSplit {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private long off;
+
+ /**
+ * For {@link Externalizable}.
+ */
+ public HadoopExternalSplit() {
+ // No-op.
+ }
+
+ /**
+ * @param hosts Hosts.
+ * @param off Offset of this split in external file.
+ */
+ public HadoopExternalSplit(String[] hosts, long off) {
+ assert off >= 0 : off;
+ assert hosts != null;
+
+ this.hosts = hosts;
+ this.off = off;
+ }
+
+ /**
+ * @return Offset of this input split in external file.
+ */
+ public long offset() {
+ return off;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeLong(off);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ off = in.readLong();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ HadoopExternalSplit that = (HadoopExternalSplit) o;
+
+ return off == that.off;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return (int)(off ^ (off >>> 32));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java
new file mode 100644
index 0000000..081182d
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.classification.*;
+import org.apache.hadoop.conf.*;
+
+/**
+ * A fake helper to load the native hadoop code i.e. libhadoop.so.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class HadoopNativeCodeLoader {
+ /**
+ * Check if native-hadoop code is loaded for this platform.
+ *
+ * @return <code>true</code> if native-hadoop is loaded,
+ * else <code>false</code>
+ */
+ public static boolean isNativeCodeLoaded() {
+ return false;
+ }
+
+ /**
+ * Returns true only if this build was compiled with support for snappy.
+ */
+ public static boolean buildSupportsSnappy() {
+ return false;
+ }
+
+ /**
+ * @return Library name.
+ */
+ public static String getLibraryName() {
+ throw new IllegalStateException();
+ }
+
+ /**
+ * Return if native hadoop libraries, if present, can be used for this job.
+ * @param conf configuration
+ *
+ * @return <code>true</code> if native hadoop libraries, if present, can be
+ * used for this job; <code>false</code> otherwise.
+ */
+ public boolean getLoadNativeLibraries(Configuration conf) {
+ return false;
+ }
+
+ /**
+ * Set if native hadoop libraries, if present, can be used for this job.
+ *
+ * @param conf configuration
+ * @param loadNativeLibraries can native hadoop libraries be loaded
+ */
+ public void setLoadNativeLibraries(Configuration conf, boolean loadNativeLibraries) {
+ // No-op.
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopSerializationWrapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopSerializationWrapper.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopSerializationWrapper.java
new file mode 100644
index 0000000..bb9cb68
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopSerializationWrapper.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.io.serializer.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * The wrapper around external serializer.
+ */
+public class HadoopSerializationWrapper<T> implements HadoopSerialization {
+ /** External serializer - writer. */
+ private final Serializer<T> serializer;
+
+ /** External serializer - reader. */
+ private final Deserializer<T> deserializer;
+
+ /** Data output for current write operation. */
+ private OutputStream currOut;
+
+ /** Data input for current read operation. */
+ private InputStream currIn;
+
+ /** Wrapper around current output to provide OutputStream interface. */
+ private final OutputStream outStream = new OutputStream() {
+ /** {@inheritDoc} */
+ @Override public void write(int b) throws IOException {
+ currOut.write(b);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(byte[] b, int off, int len) throws IOException {
+ currOut.write(b, off, len);
+ }
+ };
+
+ /** Wrapper around current input to provide InputStream interface. */
+ private final InputStream inStream = new InputStream() {
+ /** {@inheritDoc} */
+ @Override public int read() throws IOException {
+ return currIn.read();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(byte[] b, int off, int len) throws IOException {
+ return currIn.read(b, off, len);
+ }
+ };
+
+ /**
+ * @param serialization External serializer to wrap.
+ * @param cls The class to serialize.
+ */
+ public HadoopSerializationWrapper(Serialization<T> serialization, Class<T> cls) throws IgniteCheckedException {
+ assert cls != null;
+
+ serializer = serialization.getSerializer(cls);
+ deserializer = serialization.getDeserializer(cls);
+
+ try {
+ serializer.open(outStream);
+ deserializer.open(inStream);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(DataOutput out, Object obj) throws IgniteCheckedException {
+ assert out != null;
+ assert obj != null;
+
+ try {
+ currOut = (OutputStream)out;
+
+ serializer.serialize((T)obj);
+
+ currOut = null;
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object read(DataInput in, @Nullable Object obj) throws IgniteCheckedException {
+ assert in != null;
+
+ try {
+ currIn = (InputStream)in;
+
+ T res = deserializer.deserialize((T) obj);
+
+ currIn = null;
+
+ return res;
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ try {
+ serializer.close();
+ deserializer.close();
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopShutdownHookManager.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopShutdownHookManager.java
new file mode 100644
index 0000000..454f90a
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopShutdownHookManager.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import java.util.*;
+import java.util.concurrent.atomic.*;
+
+/**
+ * Fake manager for shutdown hooks.
+ */
+public class HadoopShutdownHookManager {
+ /** */
+ private static final HadoopShutdownHookManager MGR = new HadoopShutdownHookManager();
+
+ /**
+ * Return <code>ShutdownHookManager</code> singleton.
+ *
+ * @return <code>ShutdownHookManager</code> singleton.
+ */
+ public static HadoopShutdownHookManager get() {
+ return MGR;
+ }
+
+ /** */
+ private Set<Runnable> hooks = Collections.synchronizedSet(new HashSet<Runnable>());
+
+ /** */
+ private AtomicBoolean shutdownInProgress = new AtomicBoolean(false);
+
+ /**
+ * Singleton.
+ */
+ private HadoopShutdownHookManager() {
+ // No-op.
+ }
+
+ /**
+ * Adds a shutdownHook with a priority, the higher the priority
+ * the earlier will run. ShutdownHooks with same priority run
+ * in a non-deterministic order.
+ *
+ * @param shutdownHook shutdownHook <code>Runnable</code>
+ * @param priority priority of the shutdownHook.
+ */
+ public void addShutdownHook(Runnable shutdownHook, int priority) {
+ if (shutdownHook == null)
+ throw new IllegalArgumentException("shutdownHook cannot be NULL");
+
+ hooks.add(shutdownHook);
+ }
+
+ /**
+ * Removes a shutdownHook.
+ *
+ * @param shutdownHook shutdownHook to remove.
+ * @return TRUE if the shutdownHook was registered and removed,
+ * FALSE otherwise.
+ */
+ public boolean removeShutdownHook(Runnable shutdownHook) {
+ return hooks.remove(shutdownHook);
+ }
+
+ /**
+ * Indicates if a shutdownHook is registered or not.
+ *
+ * @param shutdownHook shutdownHook to check if registered.
+ * @return TRUE/FALSE depending if the shutdownHook is is registered.
+ */
+ public boolean hasShutdownHook(Runnable shutdownHook) {
+ return hooks.contains(shutdownHook);
+ }
+
+ /**
+ * Indicates if shutdown is in progress or not.
+ *
+ * @return TRUE if the shutdown is in progress, otherwise FALSE.
+ */
+ public boolean isShutdownInProgress() {
+ return shutdownInProgress.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopSplitWrapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopSplitWrapper.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopSplitWrapper.java
new file mode 100644
index 0000000..bc7ded3
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopSplitWrapper.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+
+/**
+ * The wrapper for native hadoop input splits.
+ *
+ * Warning!! This class must not depend on any Hadoop classes directly or indirectly.
+ */
+public class HadoopSplitWrapper extends HadoopInputSplit {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Native hadoop input split. */
+ private byte[] bytes;
+
+ /** */
+ private String clsName;
+
+ /** Internal ID */
+ private int id;
+
+ /**
+ * Creates new split wrapper.
+ */
+ public HadoopSplitWrapper() {
+ // No-op.
+ }
+
+ /**
+ * Creates new split wrapper.
+ *
+ * @param id Split ID.
+ * @param clsName Class name.
+ * @param bytes Serialized class.
+ * @param hosts Hosts where split is located.
+ */
+ public HadoopSplitWrapper(int id, String clsName, byte[] bytes, String[] hosts) {
+ assert hosts != null;
+ assert clsName != null;
+ assert bytes != null;
+
+ this.hosts = hosts;
+ this.id = id;
+
+ this.clsName = clsName;
+ this.bytes = bytes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeInt(id);
+
+ out.writeUTF(clsName);
+ U.writeByteArray(out, bytes);
+ }
+
+ /**
+ * @return Class name.
+ */
+ public String className() {
+ return clsName;
+ }
+
+ /**
+ * @return Class bytes.
+ */
+ public byte[] bytes() {
+ return bytes;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ id = in.readInt();
+
+ clsName = in.readUTF();
+ bytes = U.readByteArray(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ HadoopSplitWrapper that = (HadoopSplitWrapper)o;
+
+ return id == that.id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return id;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2CleanupTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2CleanupTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2CleanupTask.java
new file mode 100644
index 0000000..534033b
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2CleanupTask.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.*;
+
+import java.io.*;
+
+/**
+ * Hadoop cleanup task (commits or aborts job).
+ */
+public class HadoopV2CleanupTask extends HadoopV2Task {
+ /** Abort flag. */
+ private final boolean abort;
+
+ /**
+ * @param taskInfo Task info.
+ * @param abort Abort flag.
+ */
+ public HadoopV2CleanupTask(HadoopTaskInfo taskInfo, boolean abort) {
+ super(taskInfo);
+
+ this.abort = abort;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("ConstantConditions")
+ @Override public void run0(HadoopV2TaskContext taskCtx) throws IgniteCheckedException {
+ JobContextImpl jobCtx = taskCtx.jobContext();
+
+ try {
+ OutputFormat outputFormat = getOutputFormat(jobCtx);
+
+ OutputCommitter committer = outputFormat.getOutputCommitter(hadoopContext());
+
+ if (committer != null) {
+ if (abort)
+ committer.abortJob(jobCtx, JobStatus.State.FAILED);
+ else
+ committer.commitJob(jobCtx);
+ }
+ }
+ catch (ClassNotFoundException | IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteInterruptedCheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Context.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Context.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Context.java
new file mode 100644
index 0000000..3f8e2b6
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Context.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.*;
+import org.apache.hadoop.mapreduce.task.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Hadoop context implementation for v2 API. It provides IO operations for hadoop tasks.
+ */
+public class HadoopV2Context extends JobContextImpl implements MapContext, ReduceContext {
+ /** Input reader to overriding of HadoopTaskContext input. */
+ private RecordReader reader;
+
+ /** Output writer to overriding of HadoopTaskContext output. */
+ private RecordWriter writer;
+
+ /** Output is provided by executor environment. */
+ private final HadoopTaskOutput output;
+
+ /** Input is provided by executor environment. */
+ private final HadoopTaskInput input;
+
+ /** Unique identifier for a task attempt. */
+ private final TaskAttemptID taskAttemptID;
+
+ /** Indicates that this task is to be cancelled. */
+ private volatile boolean cancelled;
+
+ /** Input split. */
+ private InputSplit inputSplit;
+
+ /** */
+ private final HadoopTaskContext ctx;
+
+ /** */
+ private String status;
+
+ /**
+ * @param ctx Context for IO operations.
+ */
+ public HadoopV2Context(HadoopV2TaskContext ctx) {
+ super(ctx.jobConf(), ctx.jobContext().getJobID());
+
+ taskAttemptID = ctx.attemptId();
+
+ conf.set("mapreduce.job.id", taskAttemptID.getJobID().toString());
+ conf.set("mapreduce.task.id", taskAttemptID.getTaskID().toString());
+
+ output = ctx.output();
+ input = ctx.input();
+
+ this.ctx = ctx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public InputSplit getInputSplit() {
+ if (inputSplit == null) {
+ HadoopInputSplit split = ctx.taskInfo().inputSplit();
+
+ if (split == null)
+ return null;
+
+ if (split instanceof HadoopFileBlock) {
+ HadoopFileBlock fileBlock = (HadoopFileBlock)split;
+
+ inputSplit = new FileSplit(new Path(fileBlock.file()), fileBlock.start(), fileBlock.length(), null);
+ }
+ else if (split instanceof HadoopExternalSplit)
+ throw new UnsupportedOperationException(); // TODO
+ else if (split instanceof HadoopSplitWrapper)
+ inputSplit = (InputSplit) HadoopUtils.unwrapSplit((HadoopSplitWrapper) split);
+ else
+ throw new IllegalStateException();
+ }
+
+ return inputSplit;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean nextKeyValue() throws IOException, InterruptedException {
+ if (cancelled)
+ throw new HadoopTaskCancelledException("Task cancelled.");
+
+ return reader.nextKeyValue();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object getCurrentKey() throws IOException, InterruptedException {
+ if (reader != null)
+ return reader.getCurrentKey();
+
+ return input.key();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object getCurrentValue() throws IOException, InterruptedException {
+ return reader.getCurrentValue();
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public void write(Object key, Object val) throws IOException, InterruptedException {
+ if (cancelled)
+ throw new HadoopTaskCancelledException("Task cancelled.");
+
+ if (writer != null)
+ writer.write(key, val);
+ else {
+ try {
+ output.write(key, val);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputCommitter getOutputCommitter() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public TaskAttemptID getTaskAttemptID() {
+ return taskAttemptID;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setStatus(String msg) {
+ status = msg;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getStatus() {
+ return status;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getProgress() {
+ return 0.5f; // TODO
+ }
+
+ /** {@inheritDoc} */
+ @Override public Counter getCounter(Enum<?> cntrName) {
+ return getCounter(cntrName.getDeclaringClass().getName(), cntrName.name());
+ }
+
+ /** {@inheritDoc} */
+ @Override public Counter getCounter(String grpName, String cntrName) {
+ return new HadoopV2Counter(ctx.counter(grpName, cntrName, HadoopLongCounter.class));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void progress() {
+ // No-op.
+ }
+
+ /**
+ * Overrides default input data reader.
+ *
+ * @param reader New reader.
+ */
+ public void reader(RecordReader reader) {
+ this.reader = reader;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean nextKey() throws IOException, InterruptedException {
+ if (cancelled)
+ throw new HadoopTaskCancelledException("Task cancelled.");
+
+ return input.next();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Iterable getValues() throws IOException, InterruptedException {
+ return new Iterable() {
+ @Override public Iterator iterator() {
+ return input.values();
+ }
+ };
+ }
+
+ /**
+ * @return Overridden output data writer.
+ */
+ public RecordWriter writer() {
+ return writer;
+ }
+
+ /**
+ * Overrides default output data writer.
+ *
+ * @param writer New writer.
+ */
+ public void writer(RecordWriter writer) {
+ this.writer = writer;
+ }
+
+ /**
+ * Cancels the task by stop the IO.
+ */
+ public void cancel() {
+ cancelled = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Counter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Counter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Counter.java
new file mode 100644
index 0000000..96ede0d
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Counter.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.mapreduce.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+
+import java.io.*;
+
+/**
+ * Adapter from own counter implementation into Hadoop API Counter od version 2.0.
+ */
+public class HadoopV2Counter implements Counter {
+ /** Delegate. */
+ private final HadoopLongCounter cntr;
+
+ /**
+ * Creates new instance with given delegate.
+ *
+ * @param cntr Internal counter.
+ */
+ public HadoopV2Counter(HadoopLongCounter cntr) {
+ assert cntr != null : "counter must be non-null";
+
+ this.cntr = cntr;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setDisplayName(String displayName) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getName() {
+ return cntr.name();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getDisplayName() {
+ return getName();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getValue() {
+ return cntr.value();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setValue(long val) {
+ cntr.value(val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void increment(long incr) {
+ cntr.increment(incr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Counter getUnderlyingCounter() {
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(DataOutput out) throws IOException {
+ throw new UnsupportedOperationException("not implemented");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFields(DataInput in) throws IOException {
+ throw new UnsupportedOperationException("not implemented");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
new file mode 100644
index 0000000..f2f0cab
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.split.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.fs.*;
+import org.apache.ignite.internal.processors.hadoop.v1.*;
+import org.apache.ignite.internal.util.future.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jdk8.backport.*;
+
+import java.io.*;
+import java.lang.reflect.*;
+import java.util.*;
+import java.util.Queue;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Hadoop job implementation for v2 API.
+ */
+public class HadoopV2Job implements HadoopJob {
+ /** */
+ private final JobConf jobConf;
+
+ /** */
+ private final JobContextImpl jobCtx;
+
+ /** Hadoop job ID. */
+ private final HadoopJobId jobId;
+
+ /** Job info. */
+ protected HadoopJobInfo jobInfo;
+
+ /** */
+ private final JobID hadoopJobID;
+
+ /** */
+ private final HadoopV2JobResourceManager rsrcMgr;
+
+ /** */
+ private final ConcurrentMap<T2<HadoopTaskType, Integer>, GridFutureAdapter<HadoopTaskContext>> ctxs =
+ new ConcurrentHashMap8<>();
+
+ /** Pooling task context class and thus class loading environment. */
+ private final Queue<Class<?>> taskCtxClsPool = new ConcurrentLinkedQueue<>();
+
+ /** Local node ID */
+ private UUID locNodeId;
+
+ /** Serialized JobConf. */
+ private volatile byte[] jobConfData;
+
+ /**
+ * @param jobId Job ID.
+ * @param jobInfo Job info.
+ * @param log Logger.
+ */
+ public HadoopV2Job(HadoopJobId jobId, final HadoopDefaultJobInfo jobInfo, IgniteLogger log) {
+ assert jobId != null;
+ assert jobInfo != null;
+
+ this.jobId = jobId;
+ this.jobInfo = jobInfo;
+
+ hadoopJobID = new JobID(jobId.globalId().toString(), jobId.localId());
+
+ HadoopClassLoader clsLdr = (HadoopClassLoader)getClass().getClassLoader();
+
+ // Before create JobConf instance we should set new context class loader.
+ Thread.currentThread().setContextClassLoader(clsLdr);
+
+ jobConf = new JobConf();
+
+ HadoopFileSystemsUtils.setupFileSystems(jobConf);
+
+ Thread.currentThread().setContextClassLoader(null);
+
+ for (Map.Entry<String,String> e : jobInfo.properties().entrySet())
+ jobConf.set(e.getKey(), e.getValue());
+
+ jobCtx = new JobContextImpl(jobConf, hadoopJobID);
+
+ rsrcMgr = new HadoopV2JobResourceManager(jobId, jobCtx, log);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopJobId id() {
+ return jobId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopJobInfo info() {
+ return jobInfo;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<HadoopInputSplit> input() throws IgniteCheckedException {
+ Thread.currentThread().setContextClassLoader(jobConf.getClassLoader());
+
+ try {
+ String jobDirPath = jobConf.get(MRJobConfig.MAPREDUCE_JOB_DIR);
+
+ if (jobDirPath == null) { // Probably job was submitted not by hadoop client.
+ // Assume that we have needed classes and try to generate input splits ourself.
+ if (jobConf.getUseNewMapper())
+ return HadoopV2Splitter.splitJob(jobCtx);
+ else
+ return HadoopV1Splitter.splitJob(jobConf);
+ }
+
+ Path jobDir = new Path(jobDirPath);
+
+ try (FileSystem fs = FileSystem.get(jobDir.toUri(), jobConf)) {
+ JobSplit.TaskSplitMetaInfo[] metaInfos = SplitMetaInfoReader.readSplitMetaInfo(hadoopJobID, fs, jobConf,
+ jobDir);
+
+ if (F.isEmpty(metaInfos))
+ throw new IgniteCheckedException("No input splits found.");
+
+ Path splitsFile = JobSubmissionFiles.getJobSplitFile(jobDir);
+
+ try (FSDataInputStream in = fs.open(splitsFile)) {
+ Collection<HadoopInputSplit> res = new ArrayList<>(metaInfos.length);
+
+ for (JobSplit.TaskSplitMetaInfo metaInfo : metaInfos) {
+ long off = metaInfo.getStartOffset();
+
+ String[] hosts = metaInfo.getLocations();
+
+ in.seek(off);
+
+ String clsName = Text.readString(in);
+
+ HadoopFileBlock block = HadoopV1Splitter.readFileBlock(clsName, in, hosts);
+
+ if (block == null)
+ block = HadoopV2Splitter.readFileBlock(clsName, in, hosts);
+
+ res.add(block != null ? block : new HadoopExternalSplit(hosts, off));
+ }
+
+ return res;
+ }
+ }
+ catch (Throwable e) {
+ throw transformException(e);
+ }
+ }
+ finally {
+ Thread.currentThread().setContextClassLoader(null);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopTaskContext getTaskContext(HadoopTaskInfo info) throws IgniteCheckedException {
+ T2<HadoopTaskType, Integer> locTaskId = new T2<>(info.type(), info.taskNumber());
+
+ GridFutureAdapter<HadoopTaskContext> fut = ctxs.get(locTaskId);
+
+ if (fut != null)
+ return fut.get();
+
+ GridFutureAdapter<HadoopTaskContext> old = ctxs.putIfAbsent(locTaskId, fut = new GridFutureAdapter<>());
+
+ if (old != null)
+ return old.get();
+
+ Class<?> cls = taskCtxClsPool.poll();
+
+ try {
+ if (cls == null) {
+ // If there is no pooled class, then load new one.
+ HadoopClassLoader ldr = new HadoopClassLoader(rsrcMgr.classPath());
+
+ cls = ldr.loadClass(HadoopV2TaskContext.class.getName());
+ }
+
+ Constructor<?> ctr = cls.getConstructor(HadoopTaskInfo.class, HadoopJob.class,
+ HadoopJobId.class, UUID.class, DataInput.class);
+
+ if (jobConfData == null)
+ synchronized(jobConf) {
+ if (jobConfData == null) {
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+
+ jobConf.write(new DataOutputStream(buf));
+
+ jobConfData = buf.toByteArray();
+ }
+ }
+
+ HadoopTaskContext res = (HadoopTaskContext)ctr.newInstance(info, this, jobId, locNodeId,
+ new DataInputStream(new ByteArrayInputStream(jobConfData)));
+
+ fut.onDone(res);
+
+ return res;
+ }
+ catch (Throwable e) {
+ IgniteCheckedException te = transformException(e);
+
+ fut.onDone(te);
+
+ throw te;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initialize(boolean external, UUID locNodeId) throws IgniteCheckedException {
+ this.locNodeId = locNodeId;
+
+ Thread.currentThread().setContextClassLoader(jobConf.getClassLoader());
+
+ try {
+ rsrcMgr.prepareJobEnvironment(!external, jobLocalDir(locNodeId, jobId));
+ }
+ finally {
+ Thread.currentThread().setContextClassLoader(null);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void dispose(boolean external) throws IgniteCheckedException {
+ if (rsrcMgr != null && !external) {
+ File jobLocDir = jobLocalDir(locNodeId, jobId);
+
+ if (jobLocDir.exists())
+ U.delete(jobLocDir);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void prepareTaskEnvironment(HadoopTaskInfo info) throws IgniteCheckedException {
+ rsrcMgr.prepareTaskWorkDir(taskLocalDir(locNodeId, info));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cleanupTaskEnvironment(HadoopTaskInfo info) throws IgniteCheckedException {
+ HadoopTaskContext ctx = ctxs.remove(new T2<>(info.type(), info.taskNumber())).get();
+
+ taskCtxClsPool.offer(ctx.getClass());
+
+ File locDir = taskLocalDir(locNodeId, info);
+
+ if (locDir.exists())
+ U.delete(locDir);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cleanupStagingDirectory() {
+ if (rsrcMgr != null)
+ rsrcMgr.cleanupStagingDirectory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
new file mode 100644
index 0000000..6f6bfa1
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.util.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.fs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.net.*;
+import java.nio.file.*;
+import java.util.*;
+
+/**
+ * Provides all resources are needed to the job execution. Downloads the main jar, the configuration and additional
+ * files are needed to be placed on local files system.
+ */
+public class HadoopV2JobResourceManager {
+ /** Hadoop job context. */
+ private final JobContextImpl ctx;
+
+ /** Logger. */
+ private final IgniteLogger log;
+
+ /** Job ID. */
+ private final HadoopJobId jobId;
+
+ /** Class path list. */
+ private URL[] clsPath;
+
+ /** Set of local resources. */
+ private final Collection<File> rsrcSet = new HashSet<>();
+
+ /** Staging directory to delivery job jar and config to the work nodes. */
+ private Path stagingDir;
+
+ /**
+ * Creates new instance.
+ * @param jobId Job ID.
+ * @param ctx Hadoop job context.
+ * @param log Logger.
+ */
+ public HadoopV2JobResourceManager(HadoopJobId jobId, JobContextImpl ctx, IgniteLogger log) {
+ this.jobId = jobId;
+ this.ctx = ctx;
+ this.log = log.getLogger(HadoopV2JobResourceManager.class);
+ }
+
+ /**
+ * Set working directory in local file system.
+ *
+ * @param dir Working directory.
+ * @throws IOException If fails.
+ */
+ private void setLocalFSWorkingDirectory(File dir) throws IOException {
+ JobConf cfg = ctx.getJobConf();
+
+ Thread.currentThread().setContextClassLoader(cfg.getClassLoader());
+
+ try {
+ cfg.set(HadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP, dir.getAbsolutePath());
+
+ if(!cfg.getBoolean("fs.file.impl.disable.cache", false))
+ FileSystem.getLocal(cfg).setWorkingDirectory(new Path(dir.getAbsolutePath()));
+ }
+ finally {
+ Thread.currentThread().setContextClassLoader(null);
+ }
+ }
+
+ /**
+ * Prepare job resources. Resolve the classpath list and download it if needed.
+ *
+ * @param download {@code true} If need to download resources.
+ * @param jobLocDir Work directory for the job.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void prepareJobEnvironment(boolean download, File jobLocDir) throws IgniteCheckedException {
+ try {
+ if (jobLocDir.exists())
+ throw new IgniteCheckedException("Local job directory already exists: " + jobLocDir.getAbsolutePath());
+
+ JobConf cfg = ctx.getJobConf();
+
+ String mrDir = cfg.get("mapreduce.job.dir");
+
+ if (mrDir != null) {
+ stagingDir = new Path(new URI(mrDir));
+
+ if (download) {
+ FileSystem fs = FileSystem.get(stagingDir.toUri(), cfg);
+
+ if (!fs.exists(stagingDir))
+ throw new IgniteCheckedException("Failed to find map-reduce submission directory (does not exist): " +
+ stagingDir);
+
+ if (!FileUtil.copy(fs, stagingDir, jobLocDir, false, cfg))
+ throw new IgniteCheckedException("Failed to copy job submission directory contents to local file system " +
+ "[path=" + stagingDir + ", locDir=" + jobLocDir.getAbsolutePath() + ", jobId=" + jobId + ']');
+ }
+
+ File jarJobFile = new File(jobLocDir, "job.jar");
+
+ Collection<URL> clsPathUrls = new ArrayList<>();
+
+ clsPathUrls.add(jarJobFile.toURI().toURL());
+
+ rsrcSet.add(jarJobFile);
+ rsrcSet.add(new File(jobLocDir, "job.xml"));
+
+ processFiles(jobLocDir, ctx.getCacheFiles(), download, false, null, MRJobConfig.CACHE_LOCALFILES);
+ processFiles(jobLocDir, ctx.getCacheArchives(), download, true, null, MRJobConfig.CACHE_LOCALARCHIVES);
+ processFiles(jobLocDir, ctx.getFileClassPaths(), download, false, clsPathUrls, null);
+ processFiles(jobLocDir, ctx.getArchiveClassPaths(), download, true, clsPathUrls, null);
+
+ if (!clsPathUrls.isEmpty()) {
+ clsPath = new URL[clsPathUrls.size()];
+
+ clsPathUrls.toArray(clsPath);
+ }
+ }
+ else if (!jobLocDir.mkdirs())
+ throw new IgniteCheckedException("Failed to create local job directory: " + jobLocDir.getAbsolutePath());
+
+ setLocalFSWorkingDirectory(jobLocDir);
+ }
+ catch (URISyntaxException | IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /**
+ * Process list of resources.
+ *
+ * @param jobLocDir Job working directory.
+ * @param files Array of {@link java.net.URI} or {@link org.apache.hadoop.fs.Path} to process resources.
+ * @param download {@code true}, if need to download. Process class path only else.
+ * @param extract {@code true}, if need to extract archive.
+ * @param clsPathUrls Collection to add resource as classpath resource.
+ * @param rsrcNameProp Property for resource name array setting.
+ * @throws IOException If failed.
+ */
+ private void processFiles(File jobLocDir, @Nullable Object[] files, boolean download, boolean extract,
+ @Nullable Collection<URL> clsPathUrls, @Nullable String rsrcNameProp) throws IOException {
+ if (F.isEmptyOrNulls(files))
+ return;
+
+ Collection<String> res = new ArrayList<>();
+
+ for (Object pathObj : files) {
+ String locName = null;
+ Path srcPath;
+
+ if (pathObj instanceof URI) {
+ URI uri = (URI)pathObj;
+
+ locName = uri.getFragment();
+
+ srcPath = new Path(uri);
+ }
+ else
+ srcPath = (Path)pathObj;
+
+ if (locName == null)
+ locName = srcPath.getName();
+
+ File dstPath = new File(jobLocDir.getAbsolutePath(), locName);
+
+ res.add(locName);
+
+ rsrcSet.add(dstPath);
+
+ if (clsPathUrls != null)
+ clsPathUrls.add(dstPath.toURI().toURL());
+
+ if (!download)
+ continue;
+
+ JobConf cfg = ctx.getJobConf();
+
+ FileSystem dstFs = FileSystem.getLocal(cfg);
+
+ FileSystem srcFs = srcPath.getFileSystem(cfg);
+
+ if (extract) {
+ File archivesPath = new File(jobLocDir.getAbsolutePath(), ".cached-archives");
+
+ if (!archivesPath.exists() && !archivesPath.mkdir())
+ throw new IOException("Failed to create directory " +
+ "[path=" + archivesPath + ", jobId=" + jobId + ']');
+
+ File archiveFile = new File(archivesPath, locName);
+
+ FileUtil.copy(srcFs, srcPath, dstFs, new Path(archiveFile.toString()), false, cfg);
+
+ String archiveNameLC = archiveFile.getName().toLowerCase();
+
+ if (archiveNameLC.endsWith(".jar"))
+ RunJar.unJar(archiveFile, dstPath);
+ else if (archiveNameLC.endsWith(".zip"))
+ FileUtil.unZip(archiveFile, dstPath);
+ else if (archiveNameLC.endsWith(".tar.gz") ||
+ archiveNameLC.endsWith(".tgz") ||
+ archiveNameLC.endsWith(".tar"))
+ FileUtil.unTar(archiveFile, dstPath);
+ else
+ throw new IOException("Cannot unpack archive [path=" + srcPath + ", jobId=" + jobId + ']');
+ }
+ else
+ FileUtil.copy(srcFs, srcPath, dstFs, new Path(dstPath.toString()), false, cfg);
+ }
+
+ if (!res.isEmpty() && rsrcNameProp != null)
+ ctx.getJobConf().setStrings(rsrcNameProp, res.toArray(new String[res.size()]));
+ }
+
+ /**
+ * Prepares working directory for the task.
+ *
+ * <ul>
+ * <li>Creates working directory.</li>
+ * <li>Creates symbolic links to all job resources in working directory.</li>
+ * </ul>
+ *
+ * @param path Path to working directory of the task.
+ * @throws IgniteCheckedException If fails.
+ */
+ public void prepareTaskWorkDir(File path) throws IgniteCheckedException {
+ try {
+ if (path.exists())
+ throw new IOException("Task local directory already exists: " + path);
+
+ if (!path.mkdir())
+ throw new IOException("Failed to create directory: " + path);
+
+ for (File resource : rsrcSet) {
+ File symLink = new File(path, resource.getName());
+
+ try {
+ Files.createSymbolicLink(symLink.toPath(), resource.toPath());
+ }
+ catch (IOException e) {
+ String msg = "Unable to create symlink \"" + symLink + "\" to \"" + resource + "\".";
+
+ if (U.isWindows() && e instanceof FileSystemException)
+ msg += "\n\nAbility to create symbolic links is required!\n" +
+ "On Windows platform you have to grant permission 'Create symbolic links'\n" +
+ "to your user or run the Accelerator as Administrator.\n";
+
+ throw new IOException(msg, e);
+ }
+ }
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Unable to prepare local working directory for the task " +
+ "[jobId=" + jobId + ", path=" + path+ ']', e);
+ }
+ }
+
+ /**
+ * Cleans up job staging directory.
+ */
+ public void cleanupStagingDirectory() {
+ try {
+ if (stagingDir != null)
+ stagingDir.getFileSystem(ctx.getJobConf()).delete(stagingDir, true);
+ }
+ catch (Exception e) {
+ log.error("Failed to remove job staging directory [path=" + stagingDir + ", jobId=" + jobId + ']' , e);
+ }
+ }
+
+ /**
+ * Returns array of class path for current job.
+ *
+ * @return Class path collection.
+ */
+ @Nullable public URL[] classPath() {
+ return clsPath;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2MapTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2MapTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2MapTask.java
new file mode 100644
index 0000000..2bf4292
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2MapTask.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.map.*;
+import org.apache.hadoop.util.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.*;
+
+/**
+ * Hadoop map task implementation for v2 API.
+ */
+public class HadoopV2MapTask extends HadoopV2Task {
+ /**
+ * @param taskInfo Task info.
+ */
+ public HadoopV2MapTask(HadoopTaskInfo taskInfo) {
+ super(taskInfo);
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings({"ConstantConditions", "unchecked"})
+ @Override public void run0(HadoopV2TaskContext taskCtx) throws IgniteCheckedException {
+ HadoopInputSplit split = info().inputSplit();
+
+ InputSplit nativeSplit;
+
+ if (split instanceof HadoopFileBlock) {
+ HadoopFileBlock block = (HadoopFileBlock)split;
+
+ nativeSplit = new FileSplit(new Path(block.file().toString()), block.start(), block.length(), null);
+ }
+ else
+ nativeSplit = (InputSplit)taskCtx.getNativeSplit(split);
+
+ assert nativeSplit != null;
+
+ OutputFormat outputFormat = null;
+ Exception err = null;
+
+ JobContextImpl jobCtx = taskCtx.jobContext();
+
+ try {
+ InputFormat inFormat = ReflectionUtils.newInstance(jobCtx.getInputFormatClass(),
+ hadoopContext().getConfiguration());
+
+ RecordReader reader = inFormat.createRecordReader(nativeSplit, hadoopContext());
+
+ reader.initialize(nativeSplit, hadoopContext());
+
+ hadoopContext().reader(reader);
+
+ HadoopJobInfo jobInfo = taskCtx.job().info();
+
+ outputFormat = jobInfo.hasCombiner() || jobInfo.hasReducer() ? null : prepareWriter(jobCtx);
+
+ Mapper mapper = ReflectionUtils.newInstance(jobCtx.getMapperClass(), hadoopContext().getConfiguration());
+
+ try {
+ mapper.run(new WrappedMapper().getMapContext(hadoopContext()));
+ }
+ finally {
+ closeWriter();
+ }
+
+ commit(outputFormat);
+ }
+ catch (InterruptedException e) {
+ err = e;
+
+ Thread.currentThread().interrupt();
+
+ throw new IgniteInterruptedCheckedException(e);
+ }
+ catch (Exception e) {
+ err = e;
+
+ throw new IgniteCheckedException(e);
+ }
+ finally {
+ if (err != null)
+ abort(outputFormat);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Partitioner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Partitioner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Partitioner.java
new file mode 100644
index 0000000..36382d4
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Partitioner.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.util.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+
+/**
+ * Hadoop partitioner adapter for v2 API.
+ */
+public class HadoopV2Partitioner implements HadoopPartitioner {
+ /** Partitioner instance. */
+ private Partitioner<Object, Object> part;
+
+ /**
+ * @param cls Hadoop partitioner class.
+ * @param conf Job configuration.
+ */
+ public HadoopV2Partitioner(Class<? extends Partitioner<?, ?>> cls, Configuration conf) {
+ part = (Partitioner<Object, Object>) ReflectionUtils.newInstance(cls, conf);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int partition(Object key, Object val, int parts) {
+ return part.getPartition(key, val, parts);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2ReduceTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2ReduceTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2ReduceTask.java
new file mode 100644
index 0000000..250c41b
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2ReduceTask.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.reduce.*;
+import org.apache.hadoop.util.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.*;
+
+/**
+ * Hadoop reduce task implementation for v2 API.
+ */
+public class HadoopV2ReduceTask extends HadoopV2Task {
+ /** {@code True} if reduce, {@code false} if combine. */
+ private final boolean reduce;
+
+ /**
+ * Constructor.
+ *
+ * @param taskInfo Task info.
+ * @param reduce {@code True} if reduce, {@code false} if combine.
+ */
+ public HadoopV2ReduceTask(HadoopTaskInfo taskInfo, boolean reduce) {
+ super(taskInfo);
+
+ this.reduce = reduce;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings({"ConstantConditions", "unchecked"})
+ @Override public void run0(HadoopV2TaskContext taskCtx) throws IgniteCheckedException {
+ OutputFormat outputFormat = null;
+ Exception err = null;
+
+ JobContextImpl jobCtx = taskCtx.jobContext();
+
+ try {
+ outputFormat = reduce || !taskCtx.job().info().hasReducer() ? prepareWriter(jobCtx) : null;
+
+ Reducer reducer = ReflectionUtils.newInstance(reduce ? jobCtx.getReducerClass() : jobCtx.getCombinerClass(),
+ jobCtx.getConfiguration());
+
+ try {
+ reducer.run(new WrappedReducer().getReducerContext(hadoopContext()));
+ }
+ finally {
+ closeWriter();
+ }
+
+ commit(outputFormat);
+ }
+ catch (InterruptedException e) {
+ err = e;
+
+ Thread.currentThread().interrupt();
+
+ throw new IgniteInterruptedCheckedException(e);
+ }
+ catch (Exception e) {
+ err = e;
+
+ throw new IgniteCheckedException(e);
+ }
+ finally {
+ if (err != null)
+ abort(outputFormat);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2SetupTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2SetupTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2SetupTask.java
new file mode 100644
index 0000000..81587c1
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2SetupTask.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.*;
+
+import java.io.*;
+
+/**
+ * Hadoop setup task (prepares job).
+ */
+public class HadoopV2SetupTask extends HadoopV2Task {
+ /**
+ * Constructor.
+ *
+ * @param taskInfo task info.
+ */
+ public HadoopV2SetupTask(HadoopTaskInfo taskInfo) {
+ super(taskInfo);
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("ConstantConditions")
+ @Override protected void run0(HadoopV2TaskContext taskCtx) throws IgniteCheckedException {
+ try {
+ JobContextImpl jobCtx = taskCtx.jobContext();
+
+ OutputFormat outputFormat = getOutputFormat(jobCtx);
+
+ outputFormat.checkOutputSpecs(jobCtx);
+
+ OutputCommitter committer = outputFormat.getOutputCommitter(hadoopContext());
+
+ if (committer != null)
+ committer.setupJob(jobCtx);
+ }
+ catch (ClassNotFoundException | IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteInterruptedCheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Splitter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Splitter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Splitter.java
new file mode 100644
index 0000000..76a3329
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Splitter.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.*;
+import org.apache.hadoop.util.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Hadoop API v2 splitter.
+ */
+public class HadoopV2Splitter {
+ /** */
+ private static final String[] EMPTY_HOSTS = {};
+
+ /**
+ * @param ctx Job context.
+ * @return Collection of mapped splits.
+ * @throws IgniteCheckedException If mapping failed.
+ */
+ public static Collection<HadoopInputSplit> splitJob(JobContext ctx) throws IgniteCheckedException {
+ try {
+ InputFormat<?, ?> format = ReflectionUtils.newInstance(ctx.getInputFormatClass(), ctx.getConfiguration());
+
+ assert format != null;
+
+ List<InputSplit> splits = format.getSplits(ctx);
+
+ Collection<HadoopInputSplit> res = new ArrayList<>(splits.size());
+
+ int id = 0;
+
+ for (InputSplit nativeSplit : splits) {
+ if (nativeSplit instanceof FileSplit) {
+ FileSplit s = (FileSplit)nativeSplit;
+
+ res.add(new HadoopFileBlock(s.getLocations(), s.getPath().toUri(), s.getStart(), s.getLength()));
+ }
+ else
+ res.add(HadoopUtils.wrapSplit(id, nativeSplit, nativeSplit.getLocations()));
+
+ id++;
+ }
+
+ return res;
+ }
+ catch (IOException | ClassNotFoundException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteInterruptedCheckedException(e);
+ }
+ }
+
+ /**
+ * @param clsName Input split class name.
+ * @param in Input stream.
+ * @param hosts Optional hosts.
+ * @return File block or {@code null} if it is not a {@link FileSplit} instance.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static HadoopFileBlock readFileBlock(String clsName, DataInput in, @Nullable String[] hosts)
+ throws IgniteCheckedException {
+ if (!FileSplit.class.getName().equals(clsName))
+ return null;
+
+ FileSplit split = new FileSplit();
+
+ try {
+ split.readFields(in);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+
+ if (hosts == null)
+ hosts = EMPTY_HOSTS;
+
+ return new HadoopFileBlock(hosts, split.getPath().toUri(), split.getStart(), split.getLength());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Task.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Task.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Task.java
new file mode 100644
index 0000000..5ade3fb
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Task.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.util.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * Extended Hadoop v2 task.
+ */
+public abstract class HadoopV2Task extends HadoopTask {
+ /** Hadoop context. */
+ private HadoopV2Context hadoopCtx;
+
+ /**
+ * Constructor.
+ *
+ * @param taskInfo Task info.
+ */
+ protected HadoopV2Task(HadoopTaskInfo taskInfo) {
+ super(taskInfo);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ HadoopV2TaskContext ctx = (HadoopV2TaskContext)taskCtx;
+
+ hadoopCtx = new HadoopV2Context(ctx);
+
+ run0(ctx);
+ }
+
+ /**
+ * Internal task routine.
+ *
+ * @param taskCtx Task context.
+ * @throws IgniteCheckedException
+ */
+ protected abstract void run0(HadoopV2TaskContext taskCtx) throws IgniteCheckedException;
+
+ /**
+ * @return hadoop context.
+ */
+ protected HadoopV2Context hadoopContext() {
+ return hadoopCtx;
+ }
+
+ /**
+ * Create and configure an OutputFormat instance.
+ *
+ * @param jobCtx Job context.
+ * @return Instance of OutputFormat is specified in job configuration.
+ * @throws ClassNotFoundException If specified class not found.
+ */
+ protected OutputFormat getOutputFormat(JobContext jobCtx) throws ClassNotFoundException {
+ return ReflectionUtils.newInstance(jobCtx.getOutputFormatClass(), hadoopContext().getConfiguration());
+ }
+
+ /**
+ * Put write into Hadoop context and return associated output format instance.
+ *
+ * @param jobCtx Job context.
+ * @return Output format.
+ * @throws IgniteCheckedException In case of Grid exception.
+ * @throws InterruptedException In case of interrupt.
+ */
+ protected OutputFormat prepareWriter(JobContext jobCtx)
+ throws IgniteCheckedException, InterruptedException {
+ try {
+ OutputFormat outputFormat = getOutputFormat(jobCtx);
+
+ assert outputFormat != null;
+
+ OutputCommitter outCommitter = outputFormat.getOutputCommitter(hadoopCtx);
+
+ if (outCommitter != null)
+ outCommitter.setupTask(hadoopCtx);
+
+ RecordWriter writer = outputFormat.getRecordWriter(hadoopCtx);
+
+ hadoopCtx.writer(writer);
+
+ return outputFormat;
+ }
+ catch (IOException | ClassNotFoundException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /**
+ * Closes writer.
+ *
+ * @throws Exception If fails and logger hasn't been specified.
+ */
+ protected void closeWriter() throws Exception {
+ RecordWriter writer = hadoopCtx.writer();
+
+ if (writer != null)
+ writer.close(hadoopCtx);
+ }
+
+ /**
+ * Setup task.
+ *
+ * @param outputFormat Output format.
+ * @throws IOException In case of IO exception.
+ * @throws InterruptedException In case of interrupt.
+ */
+ protected void setup(@Nullable OutputFormat outputFormat) throws IOException, InterruptedException {
+ if (hadoopCtx.writer() != null) {
+ assert outputFormat != null;
+
+ outputFormat.getOutputCommitter(hadoopCtx).setupTask(hadoopCtx);
+ }
+ }
+
+ /**
+ * Commit task.
+ *
+ * @param outputFormat Output format.
+ * @throws IgniteCheckedException In case of Grid exception.
+ * @throws IOException In case of IO exception.
+ * @throws InterruptedException In case of interrupt.
+ */
+ protected void commit(@Nullable OutputFormat outputFormat) throws IgniteCheckedException, IOException, InterruptedException {
+ if (hadoopCtx.writer() != null) {
+ assert outputFormat != null;
+
+ OutputCommitter outputCommitter = outputFormat.getOutputCommitter(hadoopCtx);
+
+ if (outputCommitter.needsTaskCommit(hadoopCtx))
+ outputCommitter.commitTask(hadoopCtx);
+ }
+ }
+
+ /**
+ * Abort task.
+ *
+ * @param outputFormat Output format.
+ */
+ protected void abort(@Nullable OutputFormat outputFormat) {
+ if (hadoopCtx.writer() != null) {
+ assert outputFormat != null;
+
+ try {
+ outputFormat.getOutputCommitter(hadoopCtx).abortTask(hadoopCtx);
+ }
+ catch (IOException ignore) {
+ // Ignore.
+ }
+ catch (InterruptedException ignore) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancel() {
+ hadoopCtx.cancel();
+ }
+}
[18/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMessageListener.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMessageListener.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMessageListener.java
new file mode 100644
index 0000000..c21e494
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMessageListener.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
+
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
+
+/**
+ * Hadoop communication message listener.
+ */
+public interface HadoopMessageListener {
+ /**
+ * @param desc Process descriptor.
+ * @param msg Hadoop message.
+ */
+ public void onMessageReceived(HadoopProcessDescriptor desc, HadoopMessage msg);
+
+ /**
+ * Called when connection to remote process was lost.
+ *
+ * @param desc Process descriptor.
+ */
+ public void onConnectionLost(HadoopProcessDescriptor desc);
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopTcpNioCommunicationClient.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopTcpNioCommunicationClient.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopTcpNioCommunicationClient.java
new file mode 100644
index 0000000..c4d1c54
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopTcpNioCommunicationClient.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
+import org.apache.ignite.internal.util.nio.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+
+/**
+ * Grid client for NIO server.
+ */
+public class HadoopTcpNioCommunicationClient extends HadoopAbstractCommunicationClient {
+ /** Socket. */
+ private final GridNioSession ses;
+
+ /**
+ * Constructor for test purposes only.
+ */
+ public HadoopTcpNioCommunicationClient() {
+ ses = null;
+ }
+
+ /**
+ * @param ses Session.
+ */
+ public HadoopTcpNioCommunicationClient(GridNioSession ses) {
+ assert ses != null;
+
+ this.ses = ses;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean close() {
+ boolean res = super.close();
+
+ if (res)
+ ses.close();
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void forceClose() {
+ super.forceClose();
+
+ ses.close();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void sendMessage(HadoopProcessDescriptor desc, HadoopMessage msg)
+ throws IgniteCheckedException {
+ if (closed())
+ throw new IgniteCheckedException("Client was closed: " + this);
+
+ GridNioFuture<?> fut = ses.send(msg);
+
+ if (fut.isDone()) {
+ try {
+ fut.get();
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to send message [client=" + this + ']', e);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getIdleTime() {
+ long now = U.currentTimeMillis();
+
+ // Session can be used for receiving and sending.
+ return Math.min(Math.min(now - ses.lastReceiveTime(), now - ses.lastSendScheduleTime()),
+ now - ses.lastSendTime());
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopTcpNioCommunicationClient.class, this, super.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1CleanupTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1CleanupTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1CleanupTask.java
deleted file mode 100644
index 99ee9b77..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1CleanupTask.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v1;
-
-import org.apache.hadoop.mapred.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-
-import java.io.*;
-
-/**
- * Hadoop cleanup task implementation for v1 API.
- */
-public class GridHadoopV1CleanupTask extends GridHadoopV1Task {
- /** Abort flag. */
- private final boolean abort;
-
- /**
- * @param taskInfo Task info.
- * @param abort Abort flag.
- */
- public GridHadoopV1CleanupTask(GridHadoopTaskInfo taskInfo, boolean abort) {
- super(taskInfo);
-
- this.abort = abort;
- }
-
- /** {@inheritDoc} */
- @Override public void run(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- GridHadoopV2TaskContext ctx = (GridHadoopV2TaskContext)taskCtx;
-
- JobContext jobCtx = ctx.jobContext();
-
- try {
- OutputCommitter committer = jobCtx.getJobConf().getOutputCommitter();
-
- if (abort)
- committer.abortJob(jobCtx, JobStatus.State.FAILED);
- else
- committer.commitJob(jobCtx);
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Counter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Counter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Counter.java
deleted file mode 100644
index b986d3e..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Counter.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v1;
-
-import org.apache.hadoop.mapred.Counters;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.ignite.internal.processors.hadoop.counter.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-
-import java.io.*;
-
-import static org.apache.hadoop.mapreduce.util.CountersStrings.*;
-
-/**
- * Hadoop counter implementation for v1 API.
- */
-public class GridHadoopV1Counter extends Counters.Counter {
- /** Delegate. */
- private final GridHadoopLongCounter cntr;
-
- /**
- * Creates new instance.
- *
- * @param cntr Delegate counter.
- */
- public GridHadoopV1Counter(GridHadoopLongCounter cntr) {
- this.cntr = cntr;
- }
-
- /** {@inheritDoc} */
- @Override public void setDisplayName(String displayName) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public String getName() {
- return cntr.name();
- }
-
- /** {@inheritDoc} */
- @Override public String getDisplayName() {
- return getName();
- }
-
- /** {@inheritDoc} */
- @Override public long getValue() {
- return cntr.value();
- }
-
- /** {@inheritDoc} */
- @Override public void setValue(long val) {
- cntr.value(val);
- }
-
- /** {@inheritDoc} */
- @Override public void increment(long incr) {
- cntr.increment(incr);
- }
-
- /** {@inheritDoc} */
- @Override public void write(DataOutput out) throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-
- /** {@inheritDoc} */
- @Override public void readFields(DataInput in) throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-
- /** {@inheritDoc} */
- @Override public String makeEscapedCompactString() {
- return toEscapedCompactString(new GridHadoopV2Counter(cntr));
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("deprecation")
- @Override public boolean contentEquals(Counters.Counter cntr) {
- return getUnderlyingCounter().equals(cntr.getUnderlyingCounter());
- }
-
- /** {@inheritDoc} */
- @Override public long getCounter() {
- return cntr.value();
- }
-
- /** {@inheritDoc} */
- @Override public Counter getUnderlyingCounter() {
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1MapTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1MapTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1MapTask.java
deleted file mode 100644
index 878b61b..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1MapTask.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v1;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.util.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-
-/**
- * Hadoop map task implementation for v1 API.
- */
-public class GridHadoopV1MapTask extends GridHadoopV1Task {
- /** */
- private static final String[] EMPTY_HOSTS = new String[0];
-
- /**
- * Constructor.
- *
- * @param taskInfo
- */
- public GridHadoopV1MapTask(GridHadoopTaskInfo taskInfo) {
- super(taskInfo);
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public void run(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- GridHadoopJob job = taskCtx.job();
-
- GridHadoopV2TaskContext ctx = (GridHadoopV2TaskContext)taskCtx;
-
- JobConf jobConf = ctx.jobConf();
-
- InputFormat inFormat = jobConf.getInputFormat();
-
- GridHadoopInputSplit split = info().inputSplit();
-
- InputSplit nativeSplit;
-
- if (split instanceof GridHadoopFileBlock) {
- GridHadoopFileBlock block = (GridHadoopFileBlock)split;
-
- nativeSplit = new FileSplit(new Path(block.file().toString()), block.start(), block.length(), EMPTY_HOSTS);
- }
- else
- nativeSplit = (InputSplit)ctx.getNativeSplit(split);
-
- assert nativeSplit != null;
-
- Reporter reporter = new GridHadoopV1Reporter(taskCtx);
-
- GridHadoopV1OutputCollector collector = null;
-
- try {
- collector = collector(jobConf, ctx, !job.info().hasCombiner() && !job.info().hasReducer(),
- fileName(), ctx.attemptId());
-
- RecordReader reader = inFormat.getRecordReader(nativeSplit, jobConf, reporter);
-
- Mapper mapper = ReflectionUtils.newInstance(jobConf.getMapperClass(), jobConf);
-
- Object key = reader.createKey();
- Object val = reader.createValue();
-
- assert mapper != null;
-
- try {
- try {
- while (reader.next(key, val)) {
- if (isCancelled())
- throw new GridHadoopTaskCancelledException("Map task cancelled.");
-
- mapper.map(key, val, collector, reporter);
- }
- }
- finally {
- mapper.close();
- }
- }
- finally {
- collector.closeWriter();
- }
-
- collector.commit();
- }
- catch (Exception e) {
- if (collector != null)
- collector.abort();
-
- throw new IgniteCheckedException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1OutputCollector.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1OutputCollector.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1OutputCollector.java
deleted file mode 100644
index 2a38684..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1OutputCollector.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v1;
-
-import org.apache.hadoop.mapred.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * Hadoop output collector.
- */
-public class GridHadoopV1OutputCollector implements OutputCollector {
- /** Job configuration. */
- private final JobConf jobConf;
-
- /** Task context. */
- private final GridHadoopTaskContext taskCtx;
-
- /** Optional direct writer. */
- private final RecordWriter writer;
-
- /** Task attempt. */
- private final TaskAttemptID attempt;
-
- /**
- * @param jobConf Job configuration.
- * @param taskCtx Task context.
- * @param directWrite Direct write flag.
- * @param fileName File name.
- * @throws IOException In case of IO exception.
- */
- GridHadoopV1OutputCollector(JobConf jobConf, GridHadoopTaskContext taskCtx, boolean directWrite,
- @Nullable String fileName, TaskAttemptID attempt) throws IOException {
- this.jobConf = jobConf;
- this.taskCtx = taskCtx;
- this.attempt = attempt;
-
- if (directWrite) {
- jobConf.set("mapreduce.task.attempt.id", attempt.toString());
-
- OutputFormat outFormat = jobConf.getOutputFormat();
-
- writer = outFormat.getRecordWriter(null, jobConf, fileName, Reporter.NULL);
- }
- else
- writer = null;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public void collect(Object key, Object val) throws IOException {
- if (writer != null)
- writer.write(key, val);
- else {
- try {
- taskCtx.output().write(key, val);
- }
- catch (IgniteCheckedException e) {
- throw new IOException(e);
- }
- }
- }
-
- /**
- * Close writer.
- *
- * @throws IOException In case of IO exception.
- */
- public void closeWriter() throws IOException {
- if (writer != null)
- writer.close(Reporter.NULL);
- }
-
- /**
- * Setup task.
- *
- * @throws IOException If failed.
- */
- public void setup() throws IOException {
- if (writer != null)
- jobConf.getOutputCommitter().setupTask(new TaskAttemptContextImpl(jobConf, attempt));
- }
-
- /**
- * Commit task.
- *
- * @throws IOException In failed.
- */
- public void commit() throws IOException {
- if (writer != null) {
- OutputCommitter outputCommitter = jobConf.getOutputCommitter();
-
- TaskAttemptContext taskCtx = new TaskAttemptContextImpl(jobConf, attempt);
-
- if (outputCommitter.needsTaskCommit(taskCtx))
- outputCommitter.commitTask(taskCtx);
- }
- }
-
- /**
- * Abort task.
- */
- public void abort() {
- try {
- if (writer != null)
- jobConf.getOutputCommitter().abortTask(new TaskAttemptContextImpl(jobConf, attempt));
- }
- catch (IOException ignore) {
- // No-op.
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Partitioner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Partitioner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Partitioner.java
deleted file mode 100644
index 688ccef..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Partitioner.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v1;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.util.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-
-/**
- * Hadoop partitioner adapter for v1 API.
- */
-public class GridHadoopV1Partitioner implements GridHadoopPartitioner {
- /** Partitioner instance. */
- private Partitioner<Object, Object> part;
-
- /**
- * @param cls Hadoop partitioner class.
- * @param conf Job configuration.
- */
- public GridHadoopV1Partitioner(Class<? extends Partitioner> cls, Configuration conf) {
- part = (Partitioner<Object, Object>) ReflectionUtils.newInstance(cls, conf);
- }
-
- /** {@inheritDoc} */
- @Override public int partition(Object key, Object val, int parts) {
- return part.getPartition(key, val, parts);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1ReduceTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1ReduceTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1ReduceTask.java
deleted file mode 100644
index 7deea90..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1ReduceTask.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v1;
-
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.util.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-
-/**
- * Hadoop reduce task implementation for v1 API.
- */
-public class GridHadoopV1ReduceTask extends GridHadoopV1Task {
- /** {@code True} if reduce, {@code false} if combine. */
- private final boolean reduce;
-
- /**
- * Constructor.
- *
- * @param taskInfo Task info.
- * @param reduce {@code True} if reduce, {@code false} if combine.
- */
- public GridHadoopV1ReduceTask(GridHadoopTaskInfo taskInfo, boolean reduce) {
- super(taskInfo);
-
- this.reduce = reduce;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public void run(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- GridHadoopJob job = taskCtx.job();
-
- GridHadoopV2TaskContext ctx = (GridHadoopV2TaskContext)taskCtx;
-
- JobConf jobConf = ctx.jobConf();
-
- GridHadoopTaskInput input = taskCtx.input();
-
- GridHadoopV1OutputCollector collector = null;
-
- try {
- collector = collector(jobConf, ctx, reduce || !job.info().hasReducer(), fileName(), ctx.attemptId());
-
- Reducer reducer = ReflectionUtils.newInstance(reduce ? jobConf.getReducerClass() : jobConf.getCombinerClass(),
- jobConf);
-
- assert reducer != null;
-
- try {
- try {
- while (input.next()) {
- if (isCancelled())
- throw new GridHadoopTaskCancelledException("Reduce task cancelled.");
-
- reducer.reduce(input.key(), input.values(), collector, Reporter.NULL);
- }
- }
- finally {
- reducer.close();
- }
- }
- finally {
- collector.closeWriter();
- }
-
- collector.commit();
- }
- catch (Exception e) {
- if (collector != null)
- collector.abort();
-
- throw new IgniteCheckedException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Reporter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Reporter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Reporter.java
deleted file mode 100644
index 1abb2c0..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Reporter.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v1;
-
-import org.apache.hadoop.mapred.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.counter.*;
-
-/**
- * Hadoop reporter implementation for v1 API.
- */
-public class GridHadoopV1Reporter implements Reporter {
- /** Context. */
- private final GridHadoopTaskContext ctx;
-
- /**
- * Creates new instance.
- *
- * @param ctx Context.
- */
- public GridHadoopV1Reporter(GridHadoopTaskContext ctx) {
- this.ctx = ctx;
- }
-
- /** {@inheritDoc} */
- @Override public void setStatus(String status) {
- // TODO
- }
-
- /** {@inheritDoc} */
- @Override public Counters.Counter getCounter(Enum<?> name) {
- return getCounter(name.getDeclaringClass().getName(), name.name());
- }
-
- /** {@inheritDoc} */
- @Override public Counters.Counter getCounter(String grp, String name) {
- return new GridHadoopV1Counter(ctx.counter(grp, name, GridHadoopLongCounter.class));
- }
-
- /** {@inheritDoc} */
- @Override public void incrCounter(Enum<?> key, long amount) {
- getCounter(key).increment(amount);
- }
-
- /** {@inheritDoc} */
- @Override public void incrCounter(String grp, String cntr, long amount) {
- getCounter(grp, cntr).increment(amount);
- }
-
- /** {@inheritDoc} */
- @Override public InputSplit getInputSplit() throws UnsupportedOperationException {
- throw new UnsupportedOperationException("reporter has no input"); // TODO
- }
-
- /** {@inheritDoc} */
- @Override public float getProgress() {
- return 0.5f; // TODO
- }
-
- /** {@inheritDoc} */
- @Override public void progress() {
- // TODO
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1SetupTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1SetupTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1SetupTask.java
deleted file mode 100644
index c7dc3fd..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1SetupTask.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v1;
-
-import org.apache.hadoop.mapred.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-
-import java.io.*;
-
-/**
- * Hadoop setup task implementation for v1 API.
- */
-public class GridHadoopV1SetupTask extends GridHadoopV1Task {
- /**
- * Constructor.
- *
- * @param taskInfo Task info.
- */
- public GridHadoopV1SetupTask(GridHadoopTaskInfo taskInfo) {
- super(taskInfo);
- }
-
- /** {@inheritDoc} */
- @Override public void run(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- GridHadoopV2TaskContext ctx = (GridHadoopV2TaskContext)taskCtx;
-
- try {
- ctx.jobConf().getOutputFormat().checkOutputSpecs(null, ctx.jobConf());
-
- OutputCommitter committer = ctx.jobConf().getOutputCommitter();
-
- if (committer != null)
- committer.setupJob(ctx.jobContext());
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Splitter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Splitter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Splitter.java
deleted file mode 100644
index 257f4ea..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Splitter.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v1;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.mapred.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Hadoop API v1 splitter.
- */
-public class GridHadoopV1Splitter {
- /** */
- private static final String[] EMPTY_HOSTS = {};
-
- /**
- * @param jobConf Job configuration.
- * @return Collection of mapped splits.
- * @throws IgniteCheckedException If mapping failed.
- */
- public static Collection<GridHadoopInputSplit> splitJob(JobConf jobConf) throws IgniteCheckedException {
- try {
- InputFormat<?, ?> format = jobConf.getInputFormat();
-
- assert format != null;
-
- InputSplit[] splits = format.getSplits(jobConf, 0);
-
- Collection<GridHadoopInputSplit> res = new ArrayList<>(splits.length);
-
- for (int i = 0; i < splits.length; i++) {
- InputSplit nativeSplit = splits[i];
-
- if (nativeSplit instanceof FileSplit) {
- FileSplit s = (FileSplit)nativeSplit;
-
- res.add(new GridHadoopFileBlock(s.getLocations(), s.getPath().toUri(), s.getStart(), s.getLength()));
- }
- else
- res.add(GridHadoopUtils.wrapSplit(i, nativeSplit, nativeSplit.getLocations()));
- }
-
- return res;
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /**
- * @param clsName Input split class name.
- * @param in Input stream.
- * @param hosts Optional hosts.
- * @return File block or {@code null} if it is not a {@link FileSplit} instance.
- * @throws IgniteCheckedException If failed.
- */
- @Nullable public static GridHadoopFileBlock readFileBlock(String clsName, FSDataInputStream in,
- @Nullable String[] hosts) throws IgniteCheckedException {
- if (!FileSplit.class.getName().equals(clsName))
- return null;
-
- FileSplit split = U.newInstance(FileSplit.class);
-
- try {
- split.readFields(in);
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
-
- if (hosts == null)
- hosts = EMPTY_HOSTS;
-
- return new GridHadoopFileBlock(hosts, split.getPath().toUri(), split.getStart(), split.getLength());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Task.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Task.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Task.java
deleted file mode 100644
index 86a7264..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/GridHadoopV1Task.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v1;
-
-import org.apache.hadoop.mapred.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.text.*;
-
-/**
- * Extended Hadoop v1 task.
- */
-public abstract class GridHadoopV1Task extends GridHadoopTask {
- /** Indicates that this task is to be cancelled. */
- private volatile boolean cancelled;
-
- /**
- * Constructor.
- *
- * @param taskInfo Task info.
- */
- protected GridHadoopV1Task(GridHadoopTaskInfo taskInfo) {
- super(taskInfo);
- }
-
- /**
- * Gets file name for that task result.
- *
- * @return File name.
- */
- public String fileName() {
- NumberFormat numFormat = NumberFormat.getInstance();
-
- numFormat.setMinimumIntegerDigits(5);
- numFormat.setGroupingUsed(false);
-
- return "part-" + numFormat.format(info().taskNumber());
- }
-
- /**
- *
- * @param jobConf Job configuration.
- * @param taskCtx Task context.
- * @param directWrite Direct write flag.
- * @param fileName File name.
- * @param attempt Attempt of task.
- * @return Collector.
- * @throws IOException In case of IO exception.
- */
- protected GridHadoopV1OutputCollector collector(JobConf jobConf, GridHadoopV2TaskContext taskCtx,
- boolean directWrite, @Nullable String fileName, TaskAttemptID attempt) throws IOException {
- GridHadoopV1OutputCollector collector = new GridHadoopV1OutputCollector(jobConf, taskCtx, directWrite,
- fileName, attempt) {
- /** {@inheritDoc} */
- @Override public void collect(Object key, Object val) throws IOException {
- if (cancelled)
- throw new GridHadoopTaskCancelledException("Task cancelled.");
-
- super.collect(key, val);
- }
- };
-
- collector.setup();
-
- return collector;
- }
-
- /** {@inheritDoc} */
- @Override public void cancel() {
- cancelled = true;
- }
-
- /** Returns true if task is cancelled. */
- public boolean isCancelled() {
- return cancelled;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1CleanupTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1CleanupTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1CleanupTask.java
new file mode 100644
index 0000000..fa570ea
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1CleanupTask.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v1;
+
+import org.apache.hadoop.mapred.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+
+import java.io.*;
+
+/**
+ * Hadoop cleanup task implementation for v1 API.
+ */
+public class HadoopV1CleanupTask extends HadoopV1Task {
+ /** Abort flag. */
+ private final boolean abort;
+
+ /**
+ * @param taskInfo Task info.
+ * @param abort Abort flag.
+ */
+ public HadoopV1CleanupTask(HadoopTaskInfo taskInfo, boolean abort) {
+ super(taskInfo);
+
+ this.abort = abort;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ HadoopV2TaskContext ctx = (HadoopV2TaskContext)taskCtx;
+
+ JobContext jobCtx = ctx.jobContext();
+
+ try {
+ OutputCommitter committer = jobCtx.getJobConf().getOutputCommitter();
+
+ if (abort)
+ committer.abortJob(jobCtx, JobStatus.State.FAILED);
+ else
+ committer.commitJob(jobCtx);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Counter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Counter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Counter.java
new file mode 100644
index 0000000..609297b
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Counter.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v1;
+
+import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+
+import java.io.*;
+
+import static org.apache.hadoop.mapreduce.util.CountersStrings.*;
+
+/**
+ * Hadoop counter implementation for v1 API.
+ */
+public class HadoopV1Counter extends Counters.Counter {
+ /** Delegate. */
+ private final HadoopLongCounter cntr;
+
+ /**
+ * Creates new instance.
+ *
+ * @param cntr Delegate counter.
+ */
+ public HadoopV1Counter(HadoopLongCounter cntr) {
+ this.cntr = cntr;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setDisplayName(String displayName) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getName() {
+ return cntr.name();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getDisplayName() {
+ return getName();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getValue() {
+ return cntr.value();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setValue(long val) {
+ cntr.value(val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void increment(long incr) {
+ cntr.increment(incr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(DataOutput out) throws IOException {
+ throw new UnsupportedOperationException("not implemented");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFields(DataInput in) throws IOException {
+ throw new UnsupportedOperationException("not implemented");
+ }
+
+ /** {@inheritDoc} */
+ @Override public String makeEscapedCompactString() {
+ return toEscapedCompactString(new HadoopV2Counter(cntr));
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("deprecation")
+ @Override public boolean contentEquals(Counters.Counter cntr) {
+ return getUnderlyingCounter().equals(cntr.getUnderlyingCounter());
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getCounter() {
+ return cntr.value();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Counter getUnderlyingCounter() {
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1MapTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1MapTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1MapTask.java
new file mode 100644
index 0000000..ad7b058
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1MapTask.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v1;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.util.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+
+/**
+ * Hadoop map task implementation for v1 API.
+ */
+public class HadoopV1MapTask extends HadoopV1Task {
+ /** */
+ private static final String[] EMPTY_HOSTS = new String[0];
+
+ /**
+ * Constructor.
+ *
+ * @param taskInfo
+ */
+ public HadoopV1MapTask(HadoopTaskInfo taskInfo) {
+ super(taskInfo);
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ HadoopJob job = taskCtx.job();
+
+ HadoopV2TaskContext ctx = (HadoopV2TaskContext)taskCtx;
+
+ JobConf jobConf = ctx.jobConf();
+
+ InputFormat inFormat = jobConf.getInputFormat();
+
+ HadoopInputSplit split = info().inputSplit();
+
+ InputSplit nativeSplit;
+
+ if (split instanceof HadoopFileBlock) {
+ HadoopFileBlock block = (HadoopFileBlock)split;
+
+ nativeSplit = new FileSplit(new Path(block.file().toString()), block.start(), block.length(), EMPTY_HOSTS);
+ }
+ else
+ nativeSplit = (InputSplit)ctx.getNativeSplit(split);
+
+ assert nativeSplit != null;
+
+ Reporter reporter = new HadoopV1Reporter(taskCtx);
+
+ HadoopV1OutputCollector collector = null;
+
+ try {
+ collector = collector(jobConf, ctx, !job.info().hasCombiner() && !job.info().hasReducer(),
+ fileName(), ctx.attemptId());
+
+ RecordReader reader = inFormat.getRecordReader(nativeSplit, jobConf, reporter);
+
+ Mapper mapper = ReflectionUtils.newInstance(jobConf.getMapperClass(), jobConf);
+
+ Object key = reader.createKey();
+ Object val = reader.createValue();
+
+ assert mapper != null;
+
+ try {
+ try {
+ while (reader.next(key, val)) {
+ if (isCancelled())
+ throw new HadoopTaskCancelledException("Map task cancelled.");
+
+ mapper.map(key, val, collector, reporter);
+ }
+ }
+ finally {
+ mapper.close();
+ }
+ }
+ finally {
+ collector.closeWriter();
+ }
+
+ collector.commit();
+ }
+ catch (Exception e) {
+ if (collector != null)
+ collector.abort();
+
+ throw new IgniteCheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1OutputCollector.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1OutputCollector.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1OutputCollector.java
new file mode 100644
index 0000000..348274d
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1OutputCollector.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v1;
+
+import org.apache.hadoop.mapred.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * Hadoop output collector.
+ */
+public class HadoopV1OutputCollector implements OutputCollector {
+ /** Job configuration. */
+ private final JobConf jobConf;
+
+ /** Task context. */
+ private final HadoopTaskContext taskCtx;
+
+ /** Optional direct writer. */
+ private final RecordWriter writer;
+
+ /** Task attempt. */
+ private final TaskAttemptID attempt;
+
+ /**
+ * @param jobConf Job configuration.
+ * @param taskCtx Task context.
+ * @param directWrite Direct write flag.
+ * @param fileName File name.
+ * @throws IOException In case of IO exception.
+ */
+ HadoopV1OutputCollector(JobConf jobConf, HadoopTaskContext taskCtx, boolean directWrite,
+ @Nullable String fileName, TaskAttemptID attempt) throws IOException {
+ this.jobConf = jobConf;
+ this.taskCtx = taskCtx;
+ this.attempt = attempt;
+
+ if (directWrite) {
+ jobConf.set("mapreduce.task.attempt.id", attempt.toString());
+
+ OutputFormat outFormat = jobConf.getOutputFormat();
+
+ writer = outFormat.getRecordWriter(null, jobConf, fileName, Reporter.NULL);
+ }
+ else
+ writer = null;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public void collect(Object key, Object val) throws IOException {
+ if (writer != null)
+ writer.write(key, val);
+ else {
+ try {
+ taskCtx.output().write(key, val);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ /**
+ * Close writer.
+ *
+ * @throws IOException In case of IO exception.
+ */
+ public void closeWriter() throws IOException {
+ if (writer != null)
+ writer.close(Reporter.NULL);
+ }
+
+ /**
+ * Setup task.
+ *
+ * @throws IOException If failed.
+ */
+ public void setup() throws IOException {
+ if (writer != null)
+ jobConf.getOutputCommitter().setupTask(new TaskAttemptContextImpl(jobConf, attempt));
+ }
+
+ /**
+ * Commit task.
+ *
+ * @throws IOException In failed.
+ */
+ public void commit() throws IOException {
+ if (writer != null) {
+ OutputCommitter outputCommitter = jobConf.getOutputCommitter();
+
+ TaskAttemptContext taskCtx = new TaskAttemptContextImpl(jobConf, attempt);
+
+ if (outputCommitter.needsTaskCommit(taskCtx))
+ outputCommitter.commitTask(taskCtx);
+ }
+ }
+
+ /**
+ * Abort task.
+ */
+ public void abort() {
+ try {
+ if (writer != null)
+ jobConf.getOutputCommitter().abortTask(new TaskAttemptContextImpl(jobConf, attempt));
+ }
+ catch (IOException ignore) {
+ // No-op.
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Partitioner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Partitioner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Partitioner.java
new file mode 100644
index 0000000..e45f92b
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Partitioner.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v1;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.util.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+
+/**
+ * Hadoop partitioner adapter for v1 API.
+ */
+public class HadoopV1Partitioner implements HadoopPartitioner {
+ /** Partitioner instance. */
+ private Partitioner<Object, Object> part;
+
+ /**
+ * @param cls Hadoop partitioner class.
+ * @param conf Job configuration.
+ */
+ public HadoopV1Partitioner(Class<? extends Partitioner> cls, Configuration conf) {
+ part = (Partitioner<Object, Object>) ReflectionUtils.newInstance(cls, conf);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int partition(Object key, Object val, int parts) {
+ return part.getPartition(key, val, parts);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1ReduceTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1ReduceTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1ReduceTask.java
new file mode 100644
index 0000000..18ee09d
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1ReduceTask.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v1;
+
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.util.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+
+/**
+ * Hadoop reduce task implementation for v1 API.
+ */
+public class HadoopV1ReduceTask extends HadoopV1Task {
+ /** {@code True} if reduce, {@code false} if combine. */
+ private final boolean reduce;
+
+ /**
+ * Constructor.
+ *
+ * @param taskInfo Task info.
+ * @param reduce {@code True} if reduce, {@code false} if combine.
+ */
+ public HadoopV1ReduceTask(HadoopTaskInfo taskInfo, boolean reduce) {
+ super(taskInfo);
+
+ this.reduce = reduce;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ HadoopJob job = taskCtx.job();
+
+ HadoopV2TaskContext ctx = (HadoopV2TaskContext)taskCtx;
+
+ JobConf jobConf = ctx.jobConf();
+
+ HadoopTaskInput input = taskCtx.input();
+
+ HadoopV1OutputCollector collector = null;
+
+ try {
+ collector = collector(jobConf, ctx, reduce || !job.info().hasReducer(), fileName(), ctx.attemptId());
+
+ Reducer reducer = ReflectionUtils.newInstance(reduce ? jobConf.getReducerClass() : jobConf.getCombinerClass(),
+ jobConf);
+
+ assert reducer != null;
+
+ try {
+ try {
+ while (input.next()) {
+ if (isCancelled())
+ throw new HadoopTaskCancelledException("Reduce task cancelled.");
+
+ reducer.reduce(input.key(), input.values(), collector, Reporter.NULL);
+ }
+ }
+ finally {
+ reducer.close();
+ }
+ }
+ finally {
+ collector.closeWriter();
+ }
+
+ collector.commit();
+ }
+ catch (Exception e) {
+ if (collector != null)
+ collector.abort();
+
+ throw new IgniteCheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Reporter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Reporter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Reporter.java
new file mode 100644
index 0000000..d799373
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Reporter.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v1;
+
+import org.apache.hadoop.mapred.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+
+/**
+ * Hadoop reporter implementation for v1 API.
+ */
+public class HadoopV1Reporter implements Reporter {
+ /** Context. */
+ private final HadoopTaskContext ctx;
+
+ /**
+ * Creates new instance.
+ *
+ * @param ctx Context.
+ */
+ public HadoopV1Reporter(HadoopTaskContext ctx) {
+ this.ctx = ctx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setStatus(String status) {
+ // TODO
+ }
+
+ /** {@inheritDoc} */
+ @Override public Counters.Counter getCounter(Enum<?> name) {
+ return getCounter(name.getDeclaringClass().getName(), name.name());
+ }
+
+ /** {@inheritDoc} */
+ @Override public Counters.Counter getCounter(String grp, String name) {
+ return new HadoopV1Counter(ctx.counter(grp, name, HadoopLongCounter.class));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void incrCounter(Enum<?> key, long amount) {
+ getCounter(key).increment(amount);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void incrCounter(String grp, String cntr, long amount) {
+ getCounter(grp, cntr).increment(amount);
+ }
+
+ /** {@inheritDoc} */
+ @Override public InputSplit getInputSplit() throws UnsupportedOperationException {
+ throw new UnsupportedOperationException("reporter has no input"); // TODO
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getProgress() {
+ return 0.5f; // TODO
+ }
+
+ /** {@inheritDoc} */
+ @Override public void progress() {
+ // TODO
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1SetupTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1SetupTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1SetupTask.java
new file mode 100644
index 0000000..a758f1d
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1SetupTask.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v1;
+
+import org.apache.hadoop.mapred.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+
+import java.io.*;
+
+/**
+ * Hadoop setup task implementation for v1 API.
+ */
+public class HadoopV1SetupTask extends HadoopV1Task {
+ /**
+ * Constructor.
+ *
+ * @param taskInfo Task info.
+ */
+ public HadoopV1SetupTask(HadoopTaskInfo taskInfo) {
+ super(taskInfo);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ HadoopV2TaskContext ctx = (HadoopV2TaskContext)taskCtx;
+
+ try {
+ ctx.jobConf().getOutputFormat().checkOutputSpecs(null, ctx.jobConf());
+
+ OutputCommitter committer = ctx.jobConf().getOutputCommitter();
+
+ if (committer != null)
+ committer.setupJob(ctx.jobContext());
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Splitter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Splitter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Splitter.java
new file mode 100644
index 0000000..9eebbb8
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Splitter.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v1;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Hadoop API v1 splitter.
+ */
+public class HadoopV1Splitter {
+ /** */
+ private static final String[] EMPTY_HOSTS = {};
+
+ /**
+ * @param jobConf Job configuration.
+ * @return Collection of mapped splits.
+ * @throws IgniteCheckedException If mapping failed.
+ */
+ public static Collection<HadoopInputSplit> splitJob(JobConf jobConf) throws IgniteCheckedException {
+ try {
+ InputFormat<?, ?> format = jobConf.getInputFormat();
+
+ assert format != null;
+
+ InputSplit[] splits = format.getSplits(jobConf, 0);
+
+ Collection<HadoopInputSplit> res = new ArrayList<>(splits.length);
+
+ for (int i = 0; i < splits.length; i++) {
+ InputSplit nativeSplit = splits[i];
+
+ if (nativeSplit instanceof FileSplit) {
+ FileSplit s = (FileSplit)nativeSplit;
+
+ res.add(new HadoopFileBlock(s.getLocations(), s.getPath().toUri(), s.getStart(), s.getLength()));
+ }
+ else
+ res.add(HadoopUtils.wrapSplit(i, nativeSplit, nativeSplit.getLocations()));
+ }
+
+ return res;
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /**
+ * @param clsName Input split class name.
+ * @param in Input stream.
+ * @param hosts Optional hosts.
+ * @return File block or {@code null} if it is not a {@link FileSplit} instance.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable public static HadoopFileBlock readFileBlock(String clsName, FSDataInputStream in,
+ @Nullable String[] hosts) throws IgniteCheckedException {
+ if (!FileSplit.class.getName().equals(clsName))
+ return null;
+
+ FileSplit split = U.newInstance(FileSplit.class);
+
+ try {
+ split.readFields(in);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+
+ if (hosts == null)
+ hosts = EMPTY_HOSTS;
+
+ return new HadoopFileBlock(hosts, split.getPath().toUri(), split.getStart(), split.getLength());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Task.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Task.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Task.java
new file mode 100644
index 0000000..b7da700
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v1/HadoopV1Task.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v1;
+
+import org.apache.hadoop.mapred.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.text.*;
+
+/**
+ * Extended Hadoop v1 task.
+ */
+public abstract class HadoopV1Task extends HadoopTask {
+ /** Indicates that this task is to be cancelled. */
+ private volatile boolean cancelled;
+
+ /**
+ * Constructor.
+ *
+ * @param taskInfo Task info.
+ */
+ protected HadoopV1Task(HadoopTaskInfo taskInfo) {
+ super(taskInfo);
+ }
+
+ /**
+ * Gets file name for that task result.
+ *
+ * @return File name.
+ */
+ public String fileName() {
+ NumberFormat numFormat = NumberFormat.getInstance();
+
+ numFormat.setMinimumIntegerDigits(5);
+ numFormat.setGroupingUsed(false);
+
+ return "part-" + numFormat.format(info().taskNumber());
+ }
+
+ /**
+ *
+ * @param jobConf Job configuration.
+ * @param taskCtx Task context.
+ * @param directWrite Direct write flag.
+ * @param fileName File name.
+ * @param attempt Attempt of task.
+ * @return Collector.
+ * @throws IOException In case of IO exception.
+ */
+ protected HadoopV1OutputCollector collector(JobConf jobConf, HadoopV2TaskContext taskCtx,
+ boolean directWrite, @Nullable String fileName, TaskAttemptID attempt) throws IOException {
+ HadoopV1OutputCollector collector = new HadoopV1OutputCollector(jobConf, taskCtx, directWrite,
+ fileName, attempt) {
+ /** {@inheritDoc} */
+ @Override public void collect(Object key, Object val) throws IOException {
+ if (cancelled)
+ throw new HadoopTaskCancelledException("Task cancelled.");
+
+ super.collect(key, val);
+ }
+ };
+
+ collector.setup();
+
+ return collector;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancel() {
+ cancelled = true;
+ }
+
+ /** Returns true if task is cancelled. */
+ public boolean isCancelled() {
+ return cancelled;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopExternalSplit.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopExternalSplit.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopExternalSplit.java
deleted file mode 100644
index 36b40a2..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopExternalSplit.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-
-import java.io.*;
-
-/**
- * Split serialized in external file.
- */
-public class GridHadoopExternalSplit extends GridHadoopInputSplit {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private long off;
-
- /**
- * For {@link Externalizable}.
- */
- public GridHadoopExternalSplit() {
- // No-op.
- }
-
- /**
- * @param hosts Hosts.
- * @param off Offset of this split in external file.
- */
- public GridHadoopExternalSplit(String[] hosts, long off) {
- assert off >= 0 : off;
- assert hosts != null;
-
- this.hosts = hosts;
- this.off = off;
- }
-
- /**
- * @return Offset of this input split in external file.
- */
- public long offset() {
- return off;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeLong(off);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- off = in.readLong();
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- GridHadoopExternalSplit that = (GridHadoopExternalSplit) o;
-
- return off == that.off;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return (int)(off ^ (off >>> 32));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopNativeCodeLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopNativeCodeLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopNativeCodeLoader.java
deleted file mode 100644
index 5ef4759..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopNativeCodeLoader.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.classification.*;
-import org.apache.hadoop.conf.*;
-
-/**
- * A fake helper to load the native hadoop code i.e. libhadoop.so.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class GridHadoopNativeCodeLoader {
- /**
- * Check if native-hadoop code is loaded for this platform.
- *
- * @return <code>true</code> if native-hadoop is loaded,
- * else <code>false</code>
- */
- public static boolean isNativeCodeLoaded() {
- return false;
- }
-
- /**
- * Returns true only if this build was compiled with support for snappy.
- */
- public static boolean buildSupportsSnappy() {
- return false;
- }
-
- /**
- * @return Library name.
- */
- public static String getLibraryName() {
- throw new IllegalStateException();
- }
-
- /**
- * Return if native hadoop libraries, if present, can be used for this job.
- * @param conf configuration
- *
- * @return <code>true</code> if native hadoop libraries, if present, can be
- * used for this job; <code>false</code> otherwise.
- */
- public boolean getLoadNativeLibraries(Configuration conf) {
- return false;
- }
-
- /**
- * Set if native hadoop libraries, if present, can be used for this job.
- *
- * @param conf configuration
- * @param loadNativeLibraries can native hadoop libraries be loaded
- */
- public void setLoadNativeLibraries(Configuration conf, boolean loadNativeLibraries) {
- // No-op.
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopSerializationWrapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopSerializationWrapper.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopSerializationWrapper.java
deleted file mode 100644
index 0f38548..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopSerializationWrapper.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.io.serializer.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * The wrapper around external serializer.
- */
-public class GridHadoopSerializationWrapper<T> implements GridHadoopSerialization {
- /** External serializer - writer. */
- private final Serializer<T> serializer;
-
- /** External serializer - reader. */
- private final Deserializer<T> deserializer;
-
- /** Data output for current write operation. */
- private OutputStream currOut;
-
- /** Data input for current read operation. */
- private InputStream currIn;
-
- /** Wrapper around current output to provide OutputStream interface. */
- private final OutputStream outStream = new OutputStream() {
- /** {@inheritDoc} */
- @Override public void write(int b) throws IOException {
- currOut.write(b);
- }
-
- /** {@inheritDoc} */
- @Override public void write(byte[] b, int off, int len) throws IOException {
- currOut.write(b, off, len);
- }
- };
-
- /** Wrapper around current input to provide InputStream interface. */
- private final InputStream inStream = new InputStream() {
- /** {@inheritDoc} */
- @Override public int read() throws IOException {
- return currIn.read();
- }
-
- /** {@inheritDoc} */
- @Override public int read(byte[] b, int off, int len) throws IOException {
- return currIn.read(b, off, len);
- }
- };
-
- /**
- * @param serialization External serializer to wrap.
- * @param cls The class to serialize.
- */
- public GridHadoopSerializationWrapper(Serialization<T> serialization, Class<T> cls) throws IgniteCheckedException {
- assert cls != null;
-
- serializer = serialization.getSerializer(cls);
- deserializer = serialization.getDeserializer(cls);
-
- try {
- serializer.open(outStream);
- deserializer.open(inStream);
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void write(DataOutput out, Object obj) throws IgniteCheckedException {
- assert out != null;
- assert obj != null;
-
- try {
- currOut = (OutputStream)out;
-
- serializer.serialize((T)obj);
-
- currOut = null;
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Object read(DataInput in, @Nullable Object obj) throws IgniteCheckedException {
- assert in != null;
-
- try {
- currIn = (InputStream)in;
-
- T res = deserializer.deserialize((T) obj);
-
- currIn = null;
-
- return res;
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteCheckedException {
- try {
- serializer.close();
- deserializer.close();
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-}
[44/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java
new file mode 100644
index 0000000..f679fc0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+/**
+ * {@code IGFS} configuration. More than one file system can be configured within grid.
+ * {@code IGFS} configuration is provided via {@link IgniteConfiguration#getFileSystemConfiguration()}
+ * method.
+ */
+public class FileSystemConfiguration {
+ /** Default file system user name. */
+ public static final String DFLT_USER_NAME = System.getProperty("user.name", "anonymous");
+
+ /** Default IPC port. */
+ public static final int DFLT_IPC_PORT = 10500;
+
+ /** Default fragmentizer throttling block length. */
+ public static final long DFLT_FRAGMENTIZER_THROTTLING_BLOCK_LENGTH = 16 * 1024 * 1024;
+
+ /** Default fragmentizer throttling delay. */
+ public static final long DFLT_FRAGMENTIZER_THROTTLING_DELAY = 200;
+
+ /** Default fragmentizer concurrent files. */
+ public static final int DFLT_FRAGMENTIZER_CONCURRENT_FILES = 0;
+
+ /** Default fragmentizer local writes ratio. */
+ public static final float DFLT_FRAGMENTIZER_LOCAL_WRITES_RATIO = 0.8f;
+
+ /** Fragmentizer enabled property. */
+ public static final boolean DFLT_FRAGMENTIZER_ENABLED = true;
+
+ /** Default batch size for logging. */
+ public static final int DFLT_IGFS_LOG_BATCH_SIZE = 100;
+
+ /** Default {@code IGFS} log directory. */
+ public static final String DFLT_IGFS_LOG_DIR = "work/igfs/log";
+
+ /** Default per node buffer size. */
+ public static final int DFLT_PER_NODE_BATCH_SIZE = 100;
+
+ /** Default number of per node parallel operations. */
+ public static final int DFLT_PER_NODE_PARALLEL_BATCH_CNT = 8;
+
+ /** Default IGFS mode. */
+ public static final IgfsMode DFLT_MODE = IgfsMode.DUAL_ASYNC;
+
+ /** Default file's data block size (bytes). */
+ public static final int DFLT_BLOCK_SIZE = 1 << 16;
+
+ /** Default read/write buffers size (bytes). */
+ public static final int DFLT_BUF_SIZE = 1 << 16;
+
+ /** Default trash directory purge await timeout in case data cache oversize is detected. */
+ public static final long DFLT_TRASH_PURGE_TIMEOUT = 1000;
+
+ /** Default management port. */
+ public static final int DFLT_MGMT_PORT = 11400;
+
+ /** Default IPC endpoint enabled flag. */
+ public static final boolean DFLT_IPC_ENDPOINT_ENABLED = true;
+
+ /** IGFS instance name. */
+ private String name;
+
+ /** Cache name to store IGFS meta information. */
+ private String metaCacheName;
+
+ /** Cache name to store file's data blocks. */
+ private String dataCacheName;
+
+ /** File's data block size (bytes). */
+ private int blockSize = DFLT_BLOCK_SIZE;
+
+ /** The number of pre-fetched blocks if specific file's chunk is requested. */
+ private int prefetchBlocks;
+
+ /** Amount of sequential block reads before prefetch is triggered. */
+ private int seqReadsBeforePrefetch;
+
+ /** Read/write buffers size for stream operations (bytes). */
+ private int bufSize = DFLT_BUF_SIZE;
+
+ /** Per node buffer size. */
+ private int perNodeBatchSize = DFLT_PER_NODE_BATCH_SIZE;
+
+ /** Per node parallel operations. */
+ private int perNodeParallelBatchCnt = DFLT_PER_NODE_PARALLEL_BATCH_CNT;
+
+ /** IPC endpoint properties to publish IGFS over. */
+ private Map<String, String> ipcEndpointCfg;
+
+ /** IPC endpoint enabled flag. */
+ private boolean ipcEndpointEnabled = DFLT_IPC_ENDPOINT_ENABLED;
+
+ /** Management port. */
+ private int mgmtPort = DFLT_MGMT_PORT;
+
+ /** Secondary file system */
+ private IgfsSecondaryFileSystem secondaryFs;
+
+ /** IGFS mode. */
+ private IgfsMode dfltMode = DFLT_MODE;
+
+ /** Fragmentizer throttling block length. */
+ private long fragmentizerThrottlingBlockLen = DFLT_FRAGMENTIZER_THROTTLING_BLOCK_LENGTH;
+
+ /** Fragmentizer throttling delay. */
+ private long fragmentizerThrottlingDelay = DFLT_FRAGMENTIZER_THROTTLING_DELAY;
+
+ /** Fragmentizer concurrent files. */
+ private int fragmentizerConcurrentFiles = DFLT_FRAGMENTIZER_CONCURRENT_FILES;
+
+ /** Fragmentizer local writes ratio. */
+ private float fragmentizerLocWritesRatio = DFLT_FRAGMENTIZER_LOCAL_WRITES_RATIO;
+
+ /** Fragmentizer enabled flag. */
+ private boolean fragmentizerEnabled = DFLT_FRAGMENTIZER_ENABLED;
+
+ /** Path modes. */
+ private Map<String, IgfsMode> pathModes;
+
+ /** Maximum space. */
+ private long maxSpace;
+
+ /** Trash purge await timeout. */
+ private long trashPurgeTimeout = DFLT_TRASH_PURGE_TIMEOUT;
+
+ /** Dual mode PUT operations executor service. */
+ private ExecutorService dualModePutExec;
+
+ /** Dual mode PUT operations executor service shutdown flag. */
+ private boolean dualModePutExecShutdown;
+
+ /** Maximum amount of data in pending puts. */
+ private long dualModeMaxPendingPutsSize;
+
+ /** Maximum range length. */
+ private long maxTaskRangeLen;
+
+ /**
+ * Constructs default configuration.
+ */
+ public FileSystemConfiguration() {
+ // No-op.
+ }
+
+ /**
+ * Constructs the copy of the configuration.
+ *
+ * @param cfg Configuration to copy.
+ */
+ public FileSystemConfiguration(FileSystemConfiguration cfg) {
+ assert cfg != null;
+
+ /*
+ * Must preserve alphabetical order!
+ */
+ blockSize = cfg.getBlockSize();
+ bufSize = cfg.getStreamBufferSize();
+ dataCacheName = cfg.getDataCacheName();
+ dfltMode = cfg.getDefaultMode();
+ dualModeMaxPendingPutsSize = cfg.getDualModeMaxPendingPutsSize();
+ dualModePutExec = cfg.getDualModePutExecutorService();
+ dualModePutExecShutdown = cfg.getDualModePutExecutorServiceShutdown();
+ fragmentizerConcurrentFiles = cfg.getFragmentizerConcurrentFiles();
+ fragmentizerLocWritesRatio = cfg.getFragmentizerLocalWritesRatio();
+ fragmentizerEnabled = cfg.isFragmentizerEnabled();
+ fragmentizerThrottlingBlockLen = cfg.getFragmentizerThrottlingBlockLength();
+ fragmentizerThrottlingDelay = cfg.getFragmentizerThrottlingDelay();
+ secondaryFs = cfg.getSecondaryFileSystem();
+ ipcEndpointCfg = cfg.getIpcEndpointConfiguration();
+ ipcEndpointEnabled = cfg.isIpcEndpointEnabled();
+ maxSpace = cfg.getMaxSpaceSize();
+ maxTaskRangeLen = cfg.getMaximumTaskRangeLength();
+ metaCacheName = cfg.getMetaCacheName();
+ mgmtPort = cfg.getManagementPort();
+ name = cfg.getName();
+ pathModes = cfg.getPathModes();
+ perNodeBatchSize = cfg.getPerNodeBatchSize();
+ perNodeParallelBatchCnt = cfg.getPerNodeParallelBatchCount();
+ prefetchBlocks = cfg.getPrefetchBlocks();
+ seqReadsBeforePrefetch = cfg.getSequentialReadsBeforePrefetch();
+ trashPurgeTimeout = cfg.getTrashPurgeTimeout();
+ }
+
+ /**
+ * Gets IGFS instance name. If {@code null}, then instance with default
+ * name will be used.
+ *
+ * @return IGFS instance name.
+ */
+ @Nullable public String getName() {
+ return name;
+ }
+
+ /**
+ * Sets IGFS instance name.
+ *
+ * @param name IGFS instance name.
+ */
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Cache name to store IGFS meta information. If {@code null}, then instance
+ * with default meta-cache name will be used.
+ *
+ * @return Cache name to store IGFS meta information.
+ */
+ @Nullable public String getMetaCacheName() {
+ return metaCacheName;
+ }
+
+ /**
+ * Sets cache name to store IGFS meta information.
+ *
+ * @param metaCacheName Cache name to store IGFS meta information.
+ */
+ public void setMetaCacheName(String metaCacheName) {
+ this.metaCacheName = metaCacheName;
+ }
+
+ /**
+ * Cache name to store IGFS data.
+ *
+ * @return Cache name to store IGFS data.
+ */
+ @Nullable public String getDataCacheName() {
+ return dataCacheName;
+ }
+
+ /**
+ * Sets cache name to store IGFS data.
+ *
+ * @param dataCacheName Cache name to store IGFS data.
+ */
+ public void setDataCacheName(String dataCacheName) {
+ this.dataCacheName = dataCacheName;
+ }
+
+ /**
+ * Get file's data block size.
+ *
+ * @return File's data block size.
+ */
+ public int getBlockSize() {
+ return blockSize;
+ }
+
+ /**
+ * Sets file's data block size.
+ *
+ * @param blockSize File's data block size (bytes) or {@code 0} to reset default value.
+ */
+ public void setBlockSize(int blockSize) {
+ A.ensure(blockSize >= 0, "blockSize >= 0");
+
+ this.blockSize = blockSize == 0 ? DFLT_BLOCK_SIZE : blockSize;
+ }
+
+ /**
+ * Get number of pre-fetched blocks if specific file's chunk is requested.
+ *
+ * @return The number of pre-fetched blocks.
+ */
+ public int getPrefetchBlocks() {
+ return prefetchBlocks;
+ }
+
+ /**
+ * Sets the number of pre-fetched blocks if specific file's chunk is requested.
+ *
+ * @param prefetchBlocks New number of pre-fetched blocks.
+ */
+ public void setPrefetchBlocks(int prefetchBlocks) {
+ A.ensure(prefetchBlocks >= 0, "prefetchBlocks >= 0");
+
+ this.prefetchBlocks = prefetchBlocks;
+ }
+
+ /**
+ * Get amount of sequential block reads before prefetch is triggered. The
+ * higher this value, the longer IGFS will wait before starting to prefetch
+ * values ahead of time. Depending on the use case, this can either help
+ * or hurt performance.
+ * <p>
+ * Default is {@code 0} which means that pre-fetching will start right away.
+ * <h1 class="header">Integration With Hadoop</h1>
+ * This parameter can be also overridden for individual Hadoop MapReduce tasks by passing
+ * {@code fs.igfs.[name].open.sequential_reads_before_prefetch} configuration property directly to Hadoop
+ * MapReduce task.
+ * <p>
+ * <b>NOTE:</b> Integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition.
+ *
+ * @return Amount of sequential block reads.
+ */
+ public int getSequentialReadsBeforePrefetch() {
+ return seqReadsBeforePrefetch;
+ }
+
+ /**
+ * Sets amount of sequential block reads before prefetch is triggered. The
+ * higher this value, the longer IGFS will wait before starting to prefetch
+ * values ahead of time. Depending on the use case, this can either help
+ * or hurt performance.
+ * <p>
+ * Default is {@code 0} which means that pre-fetching will start right away.
+ * <h1 class="header">Integration With Hadoop</h1>
+ * This parameter can be also overridden for individual Hadoop MapReduce tasks by passing
+ * {@code fs.igfs.[name].open.sequential_reads_before_prefetch} configuration property directly to Hadoop
+ * MapReduce task.
+ * <p>
+ * <b>NOTE:</b> Integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition.
+ *
+ * @param seqReadsBeforePrefetch Amount of sequential block reads before prefetch is triggered.
+ */
+ public void setSequentialReadsBeforePrefetch(int seqReadsBeforePrefetch) {
+ A.ensure(seqReadsBeforePrefetch >= 0, "seqReadsBeforePrefetch >= 0");
+
+ this.seqReadsBeforePrefetch = seqReadsBeforePrefetch;
+ }
+
+ /**
+ * Get read/write buffer size for {@code IGFS} stream operations in bytes.
+ *
+ * @return Read/write buffers size (bytes).
+ */
+ public int getStreamBufferSize() {
+ return bufSize;
+ }
+
+ /**
+ * Sets read/write buffers size for {@code IGFS} stream operations (bytes).
+ *
+ * @param bufSize Read/write buffers size for stream operations (bytes) or {@code 0} to reset default value.
+ */
+ public void setStreamBufferSize(int bufSize) {
+ A.ensure(bufSize >= 0, "bufSize >= 0");
+
+ this.bufSize = bufSize == 0 ? DFLT_BUF_SIZE : bufSize;
+ }
+
+ /**
+ * Gets number of file blocks buffered on local node before sending batch to remote node.
+ *
+ * @return Per node buffer size.
+ */
+ public int getPerNodeBatchSize() {
+ return perNodeBatchSize;
+ }
+
+ /**
+ * Sets number of file blocks collected on local node before sending batch to remote node.
+ *
+ * @param perNodeBatchSize Per node buffer size.
+ */
+ public void setPerNodeBatchSize(int perNodeBatchSize) {
+ this.perNodeBatchSize = perNodeBatchSize;
+ }
+
+ /**
+ * Gets number of batches that can be concurrently sent to remote node.
+ *
+ * @return Number of batches for each node.
+ */
+ public int getPerNodeParallelBatchCount() {
+ return perNodeParallelBatchCnt;
+ }
+
+ /**
+ * Sets number of file block batches that can be concurrently sent to remote node.
+ *
+ * @param perNodeParallelBatchCnt Per node parallel load operations.
+ */
+ public void setPerNodeParallelBatchCount(int perNodeParallelBatchCnt) {
+ this.perNodeParallelBatchCnt = perNodeParallelBatchCnt;
+ }
+
+ /**
+ * Gets map of IPC endpoint configuration properties. There are 2 different
+ * types of endpoint supported: {@code shared-memory}, and {@code TCP}.
+ * <p>
+ * The following configuration properties are supported for {@code shared-memory}
+ * endpoint:
+ * <ul>
+ * <li>{@code type} - value is {@code shmem} to specify {@code shared-memory} approach.</li>
+ * <li>{@code port} - endpoint port.</li>
+ * <li>{@code size} - memory size allocated for single endpoint communication.</li>
+ * <li>
+ * {@code tokenDirectoryPath} - path, either absolute or relative to {@code IGNITE_HOME} to
+ * store shared memory tokens.
+ * </li>
+ * </ul>
+ * <p>
+ * The following configuration properties are supported for {@code TCP} approach:
+ * <ul>
+ * <li>{@code type} - value is {@code tcp} to specify {@code TCP} approach.</li>
+ * <li>{@code port} - endpoint bind port.</li>
+ * <li>
+ * {@code host} - endpoint bind host. If omitted '127.0.0.1' will be used.
+ * </li>
+ * </ul>
+ * <p>
+ * Note that {@code shared-memory} approach is not supported on Windows environments.
+ * In case IGFS is failed to bind to particular port, further attempts will be performed every 3 seconds.
+ *
+ * @return Map of IPC endpoint configuration properties. In case the value is not set, defaults will be used. Default
+ * type for Windows is "tcp", for all other platforms - "shmem". Default port is {@link #DFLT_IPC_PORT}.
+ */
+ @Nullable public Map<String,String> getIpcEndpointConfiguration() {
+ return ipcEndpointCfg;
+ }
+
+ /**
+ * Sets IPC endpoint configuration to publish IGFS over.
+ *
+ * @param ipcEndpointCfg Map of IPC endpoint config properties.
+ */
+ public void setIpcEndpointConfiguration(@Nullable Map<String,String> ipcEndpointCfg) {
+ this.ipcEndpointCfg = ipcEndpointCfg;
+ }
+
+ /**
+ * Get IPC endpoint enabled flag. In case it is set to {@code true} endpoint will be created and bound to specific
+ * port. Otherwise endpoint will not be created. Default value is {@link #DFLT_IPC_ENDPOINT_ENABLED}.
+ *
+ * @return {@code True} in case endpoint is enabled.
+ */
+ public boolean isIpcEndpointEnabled() {
+ return ipcEndpointEnabled;
+ }
+
+ /**
+ * Set IPC endpoint enabled flag. See {@link #isIpcEndpointEnabled()}.
+ *
+ * @param ipcEndpointEnabled IPC endpoint enabled flag.
+ */
+ public void setIpcEndpointEnabled(boolean ipcEndpointEnabled) {
+ this.ipcEndpointEnabled = ipcEndpointEnabled;
+ }
+
+ /**
+ * Gets port number for management endpoint. All IGFS nodes should have this port open
+ * for Visor Management Console to work with IGFS.
+ * <p>
+ * Default value is {@link #DFLT_MGMT_PORT}
+ *
+ * @return Port number or {@code -1} if management endpoint should be disabled.
+ */
+ public int getManagementPort() {
+ return mgmtPort;
+ }
+
+ /**
+ * Sets management endpoint port.
+ *
+ * @param mgmtPort port number or {@code -1} to disable management endpoint.
+ */
+ public void setManagementPort(int mgmtPort) {
+ this.mgmtPort = mgmtPort;
+ }
+
+ /**
+ * Gets mode to specify how {@code IGFS} interacts with Hadoop file system, like {@code HDFS}.
+ * Secondary Hadoop file system is provided for pass-through, write-through, and read-through
+ * purposes.
+ * <p>
+ * Default mode is {@link org.apache.ignite.igfs.IgfsMode#DUAL_ASYNC}. If secondary Hadoop file system is
+ * not configured, this mode will work just like {@link org.apache.ignite.igfs.IgfsMode#PRIMARY} mode.
+ *
+ * @return Mode to specify how IGFS interacts with secondary HDFS file system.
+ */
+ public IgfsMode getDefaultMode() {
+ return dfltMode;
+ }
+
+ /**
+ * Sets {@code IGFS} mode to specify how it should interact with secondary
+ * Hadoop file system, like {@code HDFS}. Secondary Hadoop file system is provided
+ * for pass-through, write-through, and read-through purposes.
+ *
+ * @param dfltMode {@code IGFS} mode.
+ */
+ public void setDefaultMode(IgfsMode dfltMode) {
+ this.dfltMode = dfltMode;
+ }
+
+ /**
+ * Gets the secondary file system. Secondary file system is provided for pass-through, write-through,
+ * and read-through purposes.
+ *
+ * @return Secondary file system.
+ */
+ public IgfsSecondaryFileSystem getSecondaryFileSystem() {
+ return secondaryFs;
+ }
+
+ /**
+ * Sets the secondary file system. Secondary file system is provided for pass-through, write-through,
+ * and read-through purposes.
+ *
+ * @param fileSystem
+ */
+ public void setSecondaryFileSystem(IgfsSecondaryFileSystem fileSystem) {
+ secondaryFs = fileSystem;
+ }
+
+ /**
+ * Gets map of path prefixes to {@code IGFS} modes used for them.
+ * <p>
+ * If path doesn't correspond to any specified prefix or mappings are not provided, then
+ * {@link #getDefaultMode()} is used.
+ * <p>
+ * Several folders under {@code '/apache/ignite'} folder have predefined mappings which cannot be overridden.
+ * <li>{@code /apache/ignite/primary} and all it's sub-folders will always work in {@code PRIMARY} mode.</li>
+ * <p>
+ * And in case secondary file system URI is provided:
+ * <li>{@code /apache/ignite/proxy} and all it's sub-folders will always work in {@code PROXY} mode.</li>
+ * <li>{@code /apache/ignite/sync} and all it's sub-folders will always work in {@code DUAL_SYNC} mode.</li>
+ * <li>{@code /apache/ignite/async} and all it's sub-folders will always work in {@code DUAL_ASYNC} mode.</li>
+ *
+ * @return Map of paths to {@code IGFS} modes.
+ */
+ @Nullable public Map<String, IgfsMode> getPathModes() {
+ return pathModes;
+ }
+
+ /**
+ * Sets map of path prefixes to {@code IGFS} modes used for them.
+ * <p>
+ * If path doesn't correspond to any specified prefix or mappings are not provided, then
+ * {@link #getDefaultMode()} is used.
+ *
+ * @param pathModes Map of paths to {@code IGFS} modes.
+ */
+ public void setPathModes(Map<String, IgfsMode> pathModes) {
+ this.pathModes = pathModes;
+ }
+
+ /**
+ * Gets the length of file chunk to send before delaying the fragmentizer.
+ *
+ * @return File chunk length in bytes.
+ */
+ public long getFragmentizerThrottlingBlockLength() {
+ return fragmentizerThrottlingBlockLen;
+ }
+
+ /**
+ * Sets length of file chunk to transmit before throttling is delayed.
+ *
+ * @param fragmentizerThrottlingBlockLen Block length in bytes.
+ */
+ public void setFragmentizerThrottlingBlockLength(long fragmentizerThrottlingBlockLen) {
+ this.fragmentizerThrottlingBlockLen = fragmentizerThrottlingBlockLen;
+ }
+
+ /**
+ * Gets throttle delay for fragmentizer.
+ *
+ * @return Throttle delay in milliseconds.
+ */
+ public long getFragmentizerThrottlingDelay() {
+ return fragmentizerThrottlingDelay;
+ }
+
+ /**
+ * Sets delay in milliseconds for which fragmentizer is paused.
+ *
+ * @param fragmentizerThrottlingDelay Delay in milliseconds.
+ */
+ public void setFragmentizerThrottlingDelay(long fragmentizerThrottlingDelay) {
+ this.fragmentizerThrottlingDelay = fragmentizerThrottlingDelay;
+ }
+
+ /**
+ * Gets number of files that can be processed by fragmentizer concurrently.
+ *
+ * @return Number of files to process concurrently.
+ */
+ public int getFragmentizerConcurrentFiles() {
+ return fragmentizerConcurrentFiles;
+ }
+
+ /**
+ * Sets number of files to process concurrently by fragmentizer.
+ *
+ * @param fragmentizerConcurrentFiles Number of files to process concurrently.
+ */
+ public void setFragmentizerConcurrentFiles(int fragmentizerConcurrentFiles) {
+ this.fragmentizerConcurrentFiles = fragmentizerConcurrentFiles;
+ }
+
+ /**
+ * Gets amount of local memory (in % of local IGFS max space size) available for local writes
+ * during file creation.
+ * <p>
+ * If current IGFS space size is less than {@code fragmentizerLocalWritesRatio * maxSpaceSize},
+ * then file blocks will be written to the local node first and then asynchronously distributed
+ * among cluster nodes (fragmentized).
+ * <p>
+ * Default value is {@link #DFLT_FRAGMENTIZER_LOCAL_WRITES_RATIO}.
+ *
+ * @return Ratio for local writes space.
+ */
+ public float getFragmentizerLocalWritesRatio() {
+ return fragmentizerLocWritesRatio;
+ }
+
+ /**
+ * Sets ratio for space available for local file writes.
+ *
+ * @param fragmentizerLocWritesRatio Ratio for local file writes.
+ * @see #getFragmentizerLocalWritesRatio()
+ */
+ public void setFragmentizerLocalWritesRatio(float fragmentizerLocWritesRatio) {
+ this.fragmentizerLocWritesRatio = fragmentizerLocWritesRatio;
+ }
+
+ /**
+ * Gets flag indicating whether IGFS fragmentizer is enabled. If fragmentizer is disabled, files will be
+ * written in distributed fashion.
+ *
+ * @return Flag indicating whether fragmentizer is enabled.
+ */
+ public boolean isFragmentizerEnabled() {
+ return fragmentizerEnabled;
+ }
+
+ /**
+ * Sets property indicating whether fragmentizer is enabled.
+ *
+ * @param fragmentizerEnabled {@code True} if fragmentizer is enabled.
+ */
+ public void setFragmentizerEnabled(boolean fragmentizerEnabled) {
+ this.fragmentizerEnabled = fragmentizerEnabled;
+ }
+
+ /**
+ * Get maximum space available for data cache to store file system entries.
+ *
+ * @return Maximum space available for data cache.
+ */
+ public long getMaxSpaceSize() {
+ return maxSpace;
+ }
+
+ /**
+ * Set maximum space in bytes available in data cache.
+ *
+ * @param maxSpace Maximum space available in data cache.
+ */
+ public void setMaxSpaceSize(long maxSpace) {
+ this.maxSpace = maxSpace;
+ }
+
+ /**
+ * Gets maximum timeout awaiting for trash purging in case data cache oversize is detected.
+ *
+ * @return Maximum timeout awaiting for trash purging in case data cache oversize is detected.
+ */
+ public long getTrashPurgeTimeout() {
+ return trashPurgeTimeout;
+ }
+
+ /**
+ * Sets maximum timeout awaiting for trash purging in case data cache oversize is detected.
+ *
+ * @param trashPurgeTimeout Maximum timeout awaiting for trash purging in case data cache oversize is detected.
+ */
+ public void setTrashPurgeTimeout(long trashPurgeTimeout) {
+ this.trashPurgeTimeout = trashPurgeTimeout;
+ }
+
+ /**
+ * Get DUAL mode put operation executor service. This executor service will process cache PUT requests for
+ * data which came from the secondary file system and about to be written to IGFS data cache.
+ * In case no executor service is provided, default one will be created with maximum amount of threads equals
+ * to amount of processor cores.
+ *
+ * @return Get DUAL mode put operation executor service
+ */
+ @Nullable public ExecutorService getDualModePutExecutorService() {
+ return dualModePutExec;
+ }
+
+ /**
+ * Set DUAL mode put operations executor service.
+ *
+ * @param dualModePutExec Dual mode put operations executor service.
+ */
+ public void setDualModePutExecutorService(ExecutorService dualModePutExec) {
+ this.dualModePutExec = dualModePutExec;
+ }
+
+ /**
+ * Get DUAL mode put operation executor service shutdown flag.
+ *
+ * @return DUAL mode put operation executor service shutdown flag.
+ */
+ public boolean getDualModePutExecutorServiceShutdown() {
+ return dualModePutExecShutdown;
+ }
+
+ /**
+ * Set DUAL mode put operations executor service shutdown flag.
+ *
+ * @param dualModePutExecShutdown Dual mode put operations executor service shutdown flag.
+ */
+ public void setDualModePutExecutorServiceShutdown(boolean dualModePutExecShutdown) {
+ this.dualModePutExecShutdown = dualModePutExecShutdown;
+ }
+
+ /**
+ * Get maximum amount of pending data read from the secondary file system and waiting to be written to data
+ * cache. {@code 0} or negative value stands for unlimited size.
+ * <p>
+ * By default this value is set to {@code 0}. It is recommended to set positive value in case your
+ * application performs frequent reads of large amount of data from the secondary file system in order to
+ * avoid issues with increasing GC pauses or out-of-memory error.
+ *
+ * @return Maximum amount of pending data read from the secondary file system
+ */
+ public long getDualModeMaxPendingPutsSize() {
+ return dualModeMaxPendingPutsSize;
+ }
+
+ /**
+ * Set maximum amount of data in pending put operations.
+ *
+ * @param dualModeMaxPendingPutsSize Maximum amount of data in pending put operations.
+ */
+ public void setDualModeMaxPendingPutsSize(long dualModeMaxPendingPutsSize) {
+ this.dualModeMaxPendingPutsSize = dualModeMaxPendingPutsSize;
+ }
+
+ /**
+ * Get maximum default range size of a file being split during IGFS task execution. When IGFS task is about to
+ * be executed, it requests file block locations first. Each location is defined as {@link org.apache.ignite.igfs.mapreduce.IgfsFileRange} which
+ * has length. In case this parameter is set to positive value, then IGFS will split single file range into smaller
+ * ranges with length not greater that this parameter. The only exception to this case is when maximum task range
+ * length is smaller than file block size. In this case maximum task range size will be overridden and set to file
+ * block size.
+ * <p>
+ * Note that this parameter is applied when task is split into jobs before {@link org.apache.ignite.igfs.mapreduce.IgfsRecordResolver} is
+ * applied. Therefore, final file ranges being assigned to particular jobs could be greater than value of this
+ * parameter depending on file data layout and selected resolver type.
+ * <p>
+ * Setting this parameter might be useful when file is highly colocated and have very long consequent data chunks
+ * so that task execution suffers from insufficient parallelism. E.g., in case you have one IGFS node in topology
+ * and want to process 1Gb file, then only single range of length 1Gb will be returned. This will result in
+ * a single job which will be processed in one thread. But in case you provide this configuration parameter and set
+ * maximum range length to 16Mb, then 64 ranges will be returned resulting in 64 jobs which could be executed in
+ * parallel.
+ * <p>
+ * Note that some {@code IgniteFs.execute()} methods can override value of this parameter.
+ * <p>
+ * In case value of this parameter is set to {@code 0} or negative value, it is simply ignored. Default value is
+ * {@code 0}.
+ *
+ * @return Maximum range size of a file being split during IGFS task execution.
+ */
+ public long getMaximumTaskRangeLength() {
+ return maxTaskRangeLen;
+ }
+
+ /**
+ * Set maximum default range size of a file being split during IGFS task execution.
+ * See {@link #getMaximumTaskRangeLength()} for more details.
+ *
+ * @param maxTaskRangeLen Set maximum default range size of a file being split during IGFS task execution.
+ */
+ public void setMaximumTaskRangeLength(long maxTaskRangeLen) {
+ this.maxTaskRangeLen = maxTaskRangeLen;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(FileSystemConfiguration.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java
new file mode 100644
index 0000000..01ef8b0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+/**
+ * Ignite Hadoop Accelerator configuration.
+ */
+public class HadoopConfiguration {
+ /** Default finished job info time-to-live. */
+ public static final long DFLT_FINISHED_JOB_INFO_TTL = 10_000;
+
+ /** Default value for external execution flag. */
+ public static final boolean DFLT_EXTERNAL_EXECUTION = false;
+
+ /** Default value for the max parallel tasks. */
+ public static final int DFLT_MAX_PARALLEL_TASKS = Runtime.getRuntime().availableProcessors();
+
+ /** Default value for the max task queue size. */
+ public static final int DFLT_MAX_TASK_QUEUE_SIZE = 1000;
+
+ /** Map reduce planner. */
+ private HadoopMapReducePlanner planner;
+
+ /** */
+ private boolean extExecution = DFLT_EXTERNAL_EXECUTION;
+
+ /** Finished job info TTL. */
+ private long finishedJobInfoTtl = DFLT_FINISHED_JOB_INFO_TTL;
+
+ /** */
+ private int maxParallelTasks = DFLT_MAX_PARALLEL_TASKS;
+
+ /** */
+ private int maxTaskQueueSize = DFLT_MAX_TASK_QUEUE_SIZE;
+
+ /**
+ * Default constructor.
+ */
+ public HadoopConfiguration() {
+ // No-op.
+ }
+
+ /**
+ * Copy constructor.
+ *
+ * @param cfg Configuration to copy.
+ */
+ public HadoopConfiguration(HadoopConfiguration cfg) {
+ // Preserve alphabetic order.
+ extExecution = cfg.isExternalExecution();
+ finishedJobInfoTtl = cfg.getFinishedJobInfoTtl();
+ planner = cfg.getMapReducePlanner();
+ maxParallelTasks = cfg.getMaxParallelTasks();
+ maxTaskQueueSize = cfg.getMaxTaskQueueSize();
+ }
+
+ /**
+ * Gets max number of local tasks that may be executed in parallel.
+ *
+ * @return Max number of local tasks that may be executed in parallel.
+ */
+ public int getMaxParallelTasks() {
+ return maxParallelTasks;
+ }
+
+ /**
+ * Sets max number of local tasks that may be executed in parallel.
+ *
+ * @param maxParallelTasks Max number of local tasks that may be executed in parallel.
+ */
+ public void setMaxParallelTasks(int maxParallelTasks) {
+ this.maxParallelTasks = maxParallelTasks;
+ }
+
+ /**
+ * Gets max task queue size.
+ *
+ * @return Max task queue size.
+ */
+ public int getMaxTaskQueueSize() {
+ return maxTaskQueueSize;
+ }
+
+ /**
+ * Sets max task queue size.
+ *
+ * @param maxTaskQueueSize Max task queue size.
+ */
+ public void setMaxTaskQueueSize(int maxTaskQueueSize) {
+ this.maxTaskQueueSize = maxTaskQueueSize;
+ }
+
+ /**
+ * Gets finished job info time-to-live in milliseconds.
+ *
+ * @return Finished job info time-to-live.
+ */
+ public long getFinishedJobInfoTtl() {
+ return finishedJobInfoTtl;
+ }
+
+ /**
+ * Sets finished job info time-to-live.
+ *
+ * @param finishedJobInfoTtl Finished job info time-to-live.
+ */
+ public void setFinishedJobInfoTtl(long finishedJobInfoTtl) {
+ this.finishedJobInfoTtl = finishedJobInfoTtl;
+ }
+
+ /**
+ * Gets external task execution flag. If {@code true}, hadoop job tasks will be executed in an external
+ * (relative to node) process.
+ *
+ * @return {@code True} if external execution.
+ */
+ public boolean isExternalExecution() {
+ return extExecution;
+ }
+
+ /**
+ * Sets external task execution flag.
+ *
+ * @param extExecution {@code True} if tasks should be executed in an external process.
+ * @see #isExternalExecution()
+ */
+ public void setExternalExecution(boolean extExecution) {
+ this.extExecution = extExecution;
+ }
+
+ /**
+ * Gets Hadoop map-reduce planner, a component which defines job execution plan based on job
+ * configuration and current grid topology.
+ *
+ * @return Map-reduce planner.
+ */
+ public HadoopMapReducePlanner getMapReducePlanner() {
+ return planner;
+ }
+
+ /**
+ * Sets Hadoop map-reduce planner, a component which defines job execution plan based on job
+ * configuration and current grid topology.
+ *
+ * @param planner Map-reduce planner.
+ */
+ public void setMapReducePlanner(HadoopMapReducePlanner planner) {
+ this.planner = planner;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopConfiguration.class, this, super.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/configuration/IgfsConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgfsConfiguration.java
deleted file mode 100644
index 308471d..0000000
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgfsConfiguration.java
+++ /dev/null
@@ -1,807 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.configuration;
-
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-/**
- * {@code IGFS} configuration. More than one file system can be configured within grid.
- * {@code IGFS} configuration is provided via {@link org.apache.ignite.configuration.IgniteConfiguration#getIgfsConfiguration()}
- * method.
- * <p>
- * Refer to {@code config/hadoop/default-config.xml} or {@code config/hadoop/default-config-client.xml}
- * configuration files under Ignite installation to see sample {@code IGFS} configuration.
- */
-public class IgfsConfiguration {
- /** Default file system user name. */
- public static final String DFLT_USER_NAME = System.getProperty("user.name", "anonymous");
-
- /** Default IPC port. */
- public static final int DFLT_IPC_PORT = 10500;
-
- /** Default fragmentizer throttling block length. */
- public static final long DFLT_FRAGMENTIZER_THROTTLING_BLOCK_LENGTH = 16 * 1024 * 1024;
-
- /** Default fragmentizer throttling delay. */
- public static final long DFLT_FRAGMENTIZER_THROTTLING_DELAY = 200;
-
- /** Default fragmentizer concurrent files. */
- public static final int DFLT_FRAGMENTIZER_CONCURRENT_FILES = 0;
-
- /** Default fragmentizer local writes ratio. */
- public static final float DFLT_FRAGMENTIZER_LOCAL_WRITES_RATIO = 0.8f;
-
- /** Fragmentizer enabled property. */
- public static final boolean DFLT_FRAGMENTIZER_ENABLED = true;
-
- /** Default batch size for logging. */
- public static final int DFLT_IGFS_LOG_BATCH_SIZE = 100;
-
- /** Default {@code IGFS} log directory. */
- public static final String DFLT_IGFS_LOG_DIR = "work/igfs/log";
-
- /** Default per node buffer size. */
- public static final int DFLT_PER_NODE_BATCH_SIZE = 100;
-
- /** Default number of per node parallel operations. */
- public static final int DFLT_PER_NODE_PARALLEL_BATCH_CNT = 8;
-
- /** Default IGFS mode. */
- public static final IgfsMode DFLT_MODE = IgfsMode.DUAL_ASYNC;
-
- /** Default file's data block size (bytes). */
- public static final int DFLT_BLOCK_SIZE = 1 << 16;
-
- /** Default read/write buffers size (bytes). */
- public static final int DFLT_BUF_SIZE = 1 << 16;
-
- /** Default trash directory purge await timeout in case data cache oversize is detected. */
- public static final long DFLT_TRASH_PURGE_TIMEOUT = 1000;
-
- /** Default management port. */
- public static final int DFLT_MGMT_PORT = 11400;
-
- /** Default IPC endpoint enabled flag. */
- public static final boolean DFLT_IPC_ENDPOINT_ENABLED = true;
-
- /** IGFS instance name. */
- private String name;
-
- /** Cache name to store IGFS meta information. */
- private String metaCacheName;
-
- /** Cache name to store file's data blocks. */
- private String dataCacheName;
-
- /** File's data block size (bytes). */
- private int blockSize = DFLT_BLOCK_SIZE;
-
- /** The number of pre-fetched blocks if specific file's chunk is requested. */
- private int prefetchBlocks;
-
- /** Amount of sequential block reads before prefetch is triggered. */
- private int seqReadsBeforePrefetch;
-
- /** Read/write buffers size for stream operations (bytes). */
- private int bufSize = DFLT_BUF_SIZE;
-
- /** Per node buffer size. */
- private int perNodeBatchSize = DFLT_PER_NODE_BATCH_SIZE;
-
- /** Per node parallel operations. */
- private int perNodeParallelBatchCnt = DFLT_PER_NODE_PARALLEL_BATCH_CNT;
-
- /** IPC endpoint properties to publish IGFS over. */
- private Map<String, String> ipcEndpointCfg;
-
- /** IPC endpoint enabled flag. */
- private boolean ipcEndpointEnabled = DFLT_IPC_ENDPOINT_ENABLED;
-
- /** Management port. */
- private int mgmtPort = DFLT_MGMT_PORT;
-
- /** Secondary file system */
- private Igfs secondaryFs;
-
- /** IGFS mode. */
- private IgfsMode dfltMode = DFLT_MODE;
-
- /** Fragmentizer throttling block length. */
- private long fragmentizerThrottlingBlockLen = DFLT_FRAGMENTIZER_THROTTLING_BLOCK_LENGTH;
-
- /** Fragmentizer throttling delay. */
- private long fragmentizerThrottlingDelay = DFLT_FRAGMENTIZER_THROTTLING_DELAY;
-
- /** Fragmentizer concurrent files. */
- private int fragmentizerConcurrentFiles = DFLT_FRAGMENTIZER_CONCURRENT_FILES;
-
- /** Fragmentizer local writes ratio. */
- private float fragmentizerLocWritesRatio = DFLT_FRAGMENTIZER_LOCAL_WRITES_RATIO;
-
- /** Fragmentizer enabled flag. */
- private boolean fragmentizerEnabled = DFLT_FRAGMENTIZER_ENABLED;
-
- /** Path modes. */
- private Map<String, IgfsMode> pathModes;
-
- /** Maximum space. */
- private long maxSpace;
-
- /** Trash purge await timeout. */
- private long trashPurgeTimeout = DFLT_TRASH_PURGE_TIMEOUT;
-
- /** Dual mode PUT operations executor service. */
- private ExecutorService dualModePutExec;
-
- /** Dual mode PUT operations executor service shutdown flag. */
- private boolean dualModePutExecShutdown;
-
- /** Maximum amount of data in pending puts. */
- private long dualModeMaxPendingPutsSize;
-
- /** Maximum range length. */
- private long maxTaskRangeLen;
-
- /**
- * Constructs default configuration.
- */
- public IgfsConfiguration() {
- // No-op.
- }
-
- /**
- * Constructs the copy of the configuration.
- *
- * @param cfg Configuration to copy.
- */
- public IgfsConfiguration(IgfsConfiguration cfg) {
- assert cfg != null;
-
- /*
- * Must preserve alphabetical order!
- */
- blockSize = cfg.getBlockSize();
- bufSize = cfg.getStreamBufferSize();
- dataCacheName = cfg.getDataCacheName();
- dfltMode = cfg.getDefaultMode();
- dualModeMaxPendingPutsSize = cfg.getDualModeMaxPendingPutsSize();
- dualModePutExec = cfg.getDualModePutExecutorService();
- dualModePutExecShutdown = cfg.getDualModePutExecutorServiceShutdown();
- fragmentizerConcurrentFiles = cfg.getFragmentizerConcurrentFiles();
- fragmentizerLocWritesRatio = cfg.getFragmentizerLocalWritesRatio();
- fragmentizerEnabled = cfg.isFragmentizerEnabled();
- fragmentizerThrottlingBlockLen = cfg.getFragmentizerThrottlingBlockLength();
- fragmentizerThrottlingDelay = cfg.getFragmentizerThrottlingDelay();
- secondaryFs = cfg.getSecondaryFileSystem();
- ipcEndpointCfg = cfg.getIpcEndpointConfiguration();
- ipcEndpointEnabled = cfg.isIpcEndpointEnabled();
- maxSpace = cfg.getMaxSpaceSize();
- maxTaskRangeLen = cfg.getMaximumTaskRangeLength();
- metaCacheName = cfg.getMetaCacheName();
- mgmtPort = cfg.getManagementPort();
- name = cfg.getName();
- pathModes = cfg.getPathModes();
- perNodeBatchSize = cfg.getPerNodeBatchSize();
- perNodeParallelBatchCnt = cfg.getPerNodeParallelBatchCount();
- prefetchBlocks = cfg.getPrefetchBlocks();
- seqReadsBeforePrefetch = cfg.getSequentialReadsBeforePrefetch();
- trashPurgeTimeout = cfg.getTrashPurgeTimeout();
- }
-
- /**
- * Gets IGFS instance name. If {@code null}, then instance with default
- * name will be used.
- *
- * @return IGFS instance name.
- */
- @Nullable public String getName() {
- return name;
- }
-
- /**
- * Sets IGFS instance name.
- *
- * @param name IGFS instance name.
- */
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * Cache name to store IGFS meta information. If {@code null}, then instance
- * with default meta-cache name will be used.
- *
- * @return Cache name to store IGFS meta information.
- */
- @Nullable public String getMetaCacheName() {
- return metaCacheName;
- }
-
- /**
- * Sets cache name to store IGFS meta information.
- *
- * @param metaCacheName Cache name to store IGFS meta information.
- */
- public void setMetaCacheName(String metaCacheName) {
- this.metaCacheName = metaCacheName;
- }
-
- /**
- * Cache name to store IGFS data.
- *
- * @return Cache name to store IGFS data.
- */
- @Nullable public String getDataCacheName() {
- return dataCacheName;
- }
-
- /**
- * Sets cache name to store IGFS data.
- *
- * @param dataCacheName Cache name to store IGFS data.
- */
- public void setDataCacheName(String dataCacheName) {
- this.dataCacheName = dataCacheName;
- }
-
- /**
- * Get file's data block size.
- *
- * @return File's data block size.
- */
- public int getBlockSize() {
- return blockSize;
- }
-
- /**
- * Sets file's data block size.
- *
- * @param blockSize File's data block size (bytes) or {@code 0} to reset default value.
- */
- public void setBlockSize(int blockSize) {
- A.ensure(blockSize >= 0, "blockSize >= 0");
-
- this.blockSize = blockSize == 0 ? DFLT_BLOCK_SIZE : blockSize;
- }
-
- /**
- * Get number of pre-fetched blocks if specific file's chunk is requested.
- *
- * @return The number of pre-fetched blocks.
- */
- public int getPrefetchBlocks() {
- return prefetchBlocks;
- }
-
- /**
- * Sets the number of pre-fetched blocks if specific file's chunk is requested.
- *
- * @param prefetchBlocks New number of pre-fetched blocks.
- */
- public void setPrefetchBlocks(int prefetchBlocks) {
- A.ensure(prefetchBlocks >= 0, "prefetchBlocks >= 0");
-
- this.prefetchBlocks = prefetchBlocks;
- }
-
- /**
- * Get amount of sequential block reads before prefetch is triggered. The
- * higher this value, the longer IGFS will wait before starting to prefetch
- * values ahead of time. Depending on the use case, this can either help
- * or hurt performance.
- * <p>
- * Default is {@code 0} which means that pre-fetching will start right away.
- * <h1 class="header">Integration With Hadoop</h1>
- * This parameter can be also overridden for individual Hadoop MapReduce tasks by passing
- * {@code org.apache.ignite.igfs.hadoop.IgfsHadoopParameters.PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH}
- * configuration property directly to Hadoop MapReduce task.
- * <p>
- * <b>NOTE:</b> Integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition.
- *
- * @return Amount of sequential block reads.
- */
- public int getSequentialReadsBeforePrefetch() {
- return seqReadsBeforePrefetch;
- }
-
- /**
- * Sets amount of sequential block reads before prefetch is triggered. The
- * higher this value, the longer IGFS will wait before starting to prefetch
- * values ahead of time. Depending on the use case, this can either help
- * or hurt performance.
- * <p>
- * Default is {@code 0} which means that pre-fetching will start right away.
- * <h1 class="header">Integration With Hadoop</h1>
- * This parameter can be also overridden for individual Hadoop MapReduce tasks by passing
- * {@code org.apache.ignite.igfs.hadoop.IgfsHadoopParameters.PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH}
- * configuration property directly to Hadoop MapReduce task.
- * <p>
- * <b>NOTE:</b> Integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition.
- *
- * @param seqReadsBeforePrefetch Amount of sequential block reads before prefetch is triggered.
- */
- public void setSequentialReadsBeforePrefetch(int seqReadsBeforePrefetch) {
- A.ensure(seqReadsBeforePrefetch >= 0, "seqReadsBeforePrefetch >= 0");
-
- this.seqReadsBeforePrefetch = seqReadsBeforePrefetch;
- }
-
- /**
- * Get read/write buffer size for {@code IGFS} stream operations in bytes.
- *
- * @return Read/write buffers size (bytes).
- */
- public int getStreamBufferSize() {
- return bufSize;
- }
-
- /**
- * Sets read/write buffers size for {@code IGFS} stream operations (bytes).
- *
- * @param bufSize Read/write buffers size for stream operations (bytes) or {@code 0} to reset default value.
- */
- public void setStreamBufferSize(int bufSize) {
- A.ensure(bufSize >= 0, "bufSize >= 0");
-
- this.bufSize = bufSize == 0 ? DFLT_BUF_SIZE : bufSize;
- }
-
- /**
- * Gets number of file blocks buffered on local node before sending batch to remote node.
- *
- * @return Per node buffer size.
- */
- public int getPerNodeBatchSize() {
- return perNodeBatchSize;
- }
-
- /**
- * Sets number of file blocks collected on local node before sending batch to remote node.
- *
- * @param perNodeBatchSize Per node buffer size.
- */
- public void setPerNodeBatchSize(int perNodeBatchSize) {
- this.perNodeBatchSize = perNodeBatchSize;
- }
-
- /**
- * Gets number of batches that can be concurrently sent to remote node.
- *
- * @return Number of batches for each node.
- */
- public int getPerNodeParallelBatchCount() {
- return perNodeParallelBatchCnt;
- }
-
- /**
- * Sets number of file block batches that can be concurrently sent to remote node.
- *
- * @param perNodeParallelBatchCnt Per node parallel load operations.
- */
- public void setPerNodeParallelBatchCount(int perNodeParallelBatchCnt) {
- this.perNodeParallelBatchCnt = perNodeParallelBatchCnt;
- }
-
- /**
- * Gets map of IPC endpoint configuration properties. There are 2 different
- * types of endpoint supported: {@code shared-memory}, and {@code TCP}.
- * <p>
- * The following configuration properties are supported for {@code shared-memory}
- * endpoint:
- * <ul>
- * <li>{@code type} - value is {@code shmem} to specify {@code shared-memory} approach.</li>
- * <li>{@code port} - endpoint port.</li>
- * <li>{@code size} - memory size allocated for single endpoint communication.</li>
- * <li>
- * {@code tokenDirectoryPath} - path, either absolute or relative to {@code IGNITE_HOME} to
- * store shared memory tokens.
- * </li>
- * </ul>
- * <p>
- * The following configuration properties are supported for {@code TCP} approach:
- * <ul>
- * <li>{@code type} - value is {@code tcp} to specify {@code TCP} approach.</li>
- * <li>{@code port} - endpoint bind port.</li>
- * <li>
- * {@code host} - endpoint bind host. If omitted '127.0.0.1' will be used.
- * </li>
- * </ul>
- * <p>
- * Note that {@code shared-memory} approach is not supported on Windows environments.
- * In case IGFS is failed to bind to particular port, further attempts will be performed every 3 seconds.
- *
- * @return Map of IPC endpoint configuration properties. In case the value is not set, defaults will be used. Default
- * type for Windows is "tcp", for all other platforms - "shmem". Default port is {@link #DFLT_IPC_PORT}.
- */
- @Nullable public Map<String,String> getIpcEndpointConfiguration() {
- return ipcEndpointCfg;
- }
-
- /**
- * Sets IPC endpoint configuration to publish IGFS over.
- *
- * @param ipcEndpointCfg Map of IPC endpoint config properties.
- */
- public void setIpcEndpointConfiguration(@Nullable Map<String,String> ipcEndpointCfg) {
- this.ipcEndpointCfg = ipcEndpointCfg;
- }
-
- /**
- * Get IPC endpoint enabled flag. In case it is set to {@code true} endpoint will be created and bound to specific
- * port. Otherwise endpoint will not be created. Default value is {@link #DFLT_IPC_ENDPOINT_ENABLED}.
- *
- * @return {@code True} in case endpoint is enabled.
- */
- public boolean isIpcEndpointEnabled() {
- return ipcEndpointEnabled;
- }
-
- /**
- * Set IPC endpoint enabled flag. See {@link #isIpcEndpointEnabled()}.
- *
- * @param ipcEndpointEnabled IPC endpoint enabled flag.
- */
- public void setIpcEndpointEnabled(boolean ipcEndpointEnabled) {
- this.ipcEndpointEnabled = ipcEndpointEnabled;
- }
-
- /**
- * Gets port number for management endpoint. All IGFS nodes should have this port open
- * for Visor Management Console to work with IGFS.
- * <p>
- * Default value is {@link #DFLT_MGMT_PORT}
- *
- * @return Port number or {@code -1} if management endpoint should be disabled.
- */
- public int getManagementPort() {
- return mgmtPort;
- }
-
- /**
- * Sets management endpoint port.
- *
- * @param mgmtPort port number or {@code -1} to disable management endpoint.
- */
- public void setManagementPort(int mgmtPort) {
- this.mgmtPort = mgmtPort;
- }
-
- /**
- * Gets mode to specify how {@code IGFS} interacts with Hadoop file system, like {@code HDFS}.
- * Secondary Hadoop file system is provided for pass-through, write-through, and read-through
- * purposes.
- * <p>
- * Default mode is {@link org.apache.ignite.igfs.IgfsMode#DUAL_ASYNC}. If secondary Hadoop file system is
- * not configured, this mode will work just like {@link org.apache.ignite.igfs.IgfsMode#PRIMARY} mode.
- *
- * @return Mode to specify how IGFS interacts with secondary HDFS file system.
- */
- public IgfsMode getDefaultMode() {
- return dfltMode;
- }
-
- /**
- * Sets {@code IGFS} mode to specify how it should interact with secondary
- * Hadoop file system, like {@code HDFS}. Secondary Hadoop file system is provided
- * for pass-through, write-through, and read-through purposes.
- *
- * @param dfltMode {@code IGFS} mode.
- */
- public void setDefaultMode(IgfsMode dfltMode) {
- this.dfltMode = dfltMode;
- }
-
- /**
- * Gets the secondary file system. Secondary file system is provided for pass-through, write-through,
- * and read-through purposes.
- *
- * @return Secondary file system.
- */
- public Igfs getSecondaryFileSystem() {
- return secondaryFs;
- }
-
- /**
- * Sets the secondary file system. Secondary file system is provided for pass-through, write-through,
- * and read-through purposes.
- *
- * @param fileSystem
- */
- public void setSecondaryFileSystem(Igfs fileSystem) {
- secondaryFs = fileSystem;
- }
-
- /**
- * Gets map of path prefixes to {@code IGFS} modes used for them.
- * <p>
- * If path doesn't correspond to any specified prefix or mappings are not provided, then
- * {@link #getDefaultMode()} is used.
- * <p>
- * Several folders under {@code '/apache/ignite'} folder have predefined mappings which cannot be overridden.
- * <li>{@code /apache/ignite/primary} and all it's sub-folders will always work in {@code PRIMARY} mode.</li>
- * <p>
- * And in case secondary file system URI is provided:
- * <li>{@code /apache/ignite/proxy} and all it's sub-folders will always work in {@code PROXY} mode.</li>
- * <li>{@code /apache/ignite/sync} and all it's sub-folders will always work in {@code DUAL_SYNC} mode.</li>
- * <li>{@code /apache/ignite/async} and all it's sub-folders will always work in {@code DUAL_ASYNC} mode.</li>
- *
- * @return Map of paths to {@code IGFS} modes.
- */
- @Nullable public Map<String, IgfsMode> getPathModes() {
- return pathModes;
- }
-
- /**
- * Sets map of path prefixes to {@code IGFS} modes used for them.
- * <p>
- * If path doesn't correspond to any specified prefix or mappings are not provided, then
- * {@link #getDefaultMode()} is used.
- *
- * @param pathModes Map of paths to {@code IGFS} modes.
- */
- public void setPathModes(Map<String, IgfsMode> pathModes) {
- this.pathModes = pathModes;
- }
-
- /**
- * Gets the length of file chunk to send before delaying the fragmentizer.
- *
- * @return File chunk length in bytes.
- */
- public long getFragmentizerThrottlingBlockLength() {
- return fragmentizerThrottlingBlockLen;
- }
-
- /**
- * Sets length of file chunk to transmit before throttling is delayed.
- *
- * @param fragmentizerThrottlingBlockLen Block length in bytes.
- */
- public void setFragmentizerThrottlingBlockLength(long fragmentizerThrottlingBlockLen) {
- this.fragmentizerThrottlingBlockLen = fragmentizerThrottlingBlockLen;
- }
-
- /**
- * Gets throttle delay for fragmentizer.
- *
- * @return Throttle delay in milliseconds.
- */
- public long getFragmentizerThrottlingDelay() {
- return fragmentizerThrottlingDelay;
- }
-
- /**
- * Sets delay in milliseconds for which fragmentizer is paused.
- *
- * @param fragmentizerThrottlingDelay Delay in milliseconds.
- */
- public void setFragmentizerThrottlingDelay(long fragmentizerThrottlingDelay) {
- this.fragmentizerThrottlingDelay = fragmentizerThrottlingDelay;
- }
-
- /**
- * Gets number of files that can be processed by fragmentizer concurrently.
- *
- * @return Number of files to process concurrently.
- */
- public int getFragmentizerConcurrentFiles() {
- return fragmentizerConcurrentFiles;
- }
-
- /**
- * Sets number of files to process concurrently by fragmentizer.
- *
- * @param fragmentizerConcurrentFiles Number of files to process concurrently.
- */
- public void setFragmentizerConcurrentFiles(int fragmentizerConcurrentFiles) {
- this.fragmentizerConcurrentFiles = fragmentizerConcurrentFiles;
- }
-
- /**
- * Gets amount of local memory (in % of local IGFS max space size) available for local writes
- * during file creation.
- * <p>
- * If current IGFS space size is less than {@code fragmentizerLocalWritesRatio * maxSpaceSize},
- * then file blocks will be written to the local node first and then asynchronously distributed
- * among cluster nodes (fragmentized).
- * <p>
- * Default value is {@link #DFLT_FRAGMENTIZER_LOCAL_WRITES_RATIO}.
- *
- * @return Ratio for local writes space.
- */
- public float getFragmentizerLocalWritesRatio() {
- return fragmentizerLocWritesRatio;
- }
-
- /**
- * Sets ratio for space available for local file writes.
- *
- * @param fragmentizerLocWritesRatio Ratio for local file writes.
- * @see #getFragmentizerLocalWritesRatio()
- */
- public void setFragmentizerLocalWritesRatio(float fragmentizerLocWritesRatio) {
- this.fragmentizerLocWritesRatio = fragmentizerLocWritesRatio;
- }
-
- /**
- * Gets flag indicating whether IGFS fragmentizer is enabled. If fragmentizer is disabled, files will be
- * written in distributed fashion.
- *
- * @return Flag indicating whether fragmentizer is enabled.
- */
- public boolean isFragmentizerEnabled() {
- return fragmentizerEnabled;
- }
-
- /**
- * Sets property indicating whether fragmentizer is enabled.
- *
- * @param fragmentizerEnabled {@code True} if fragmentizer is enabled.
- */
- public void setFragmentizerEnabled(boolean fragmentizerEnabled) {
- this.fragmentizerEnabled = fragmentizerEnabled;
- }
-
- /**
- * Get maximum space available for data cache to store file system entries.
- *
- * @return Maximum space available for data cache.
- */
- public long getMaxSpaceSize() {
- return maxSpace;
- }
-
- /**
- * Set maximum space in bytes available in data cache.
- *
- * @param maxSpace Maximum space available in data cache.
- */
- public void setMaxSpaceSize(long maxSpace) {
- this.maxSpace = maxSpace;
- }
-
- /**
- * Gets maximum timeout awaiting for trash purging in case data cache oversize is detected.
- *
- * @return Maximum timeout awaiting for trash purging in case data cache oversize is detected.
- */
- public long getTrashPurgeTimeout() {
- return trashPurgeTimeout;
- }
-
- /**
- * Sets maximum timeout awaiting for trash purging in case data cache oversize is detected.
- *
- * @param trashPurgeTimeout Maximum timeout awaiting for trash purging in case data cache oversize is detected.
- */
- public void setTrashPurgeTimeout(long trashPurgeTimeout) {
- this.trashPurgeTimeout = trashPurgeTimeout;
- }
-
- /**
- * Get DUAL mode put operation executor service. This executor service will process cache PUT requests for
- * data which came from the secondary file system and about to be written to IGFS data cache.
- * In case no executor service is provided, default one will be created with maximum amount of threads equals
- * to amount of processor cores.
- *
- * @return Get DUAL mode put operation executor service
- */
- @Nullable public ExecutorService getDualModePutExecutorService() {
- return dualModePutExec;
- }
-
- /**
- * Set DUAL mode put operations executor service.
- *
- * @param dualModePutExec Dual mode put operations executor service.
- */
- public void setDualModePutExecutorService(ExecutorService dualModePutExec) {
- this.dualModePutExec = dualModePutExec;
- }
-
- /**
- * Get DUAL mode put operation executor service shutdown flag.
- *
- * @return DUAL mode put operation executor service shutdown flag.
- */
- public boolean getDualModePutExecutorServiceShutdown() {
- return dualModePutExecShutdown;
- }
-
- /**
- * Set DUAL mode put operations executor service shutdown flag.
- *
- * @param dualModePutExecShutdown Dual mode put operations executor service shutdown flag.
- */
- public void setDualModePutExecutorServiceShutdown(boolean dualModePutExecShutdown) {
- this.dualModePutExecShutdown = dualModePutExecShutdown;
- }
-
- /**
- * Get maximum amount of pending data read from the secondary file system and waiting to be written to data
- * cache. {@code 0} or negative value stands for unlimited size.
- * <p>
- * By default this value is set to {@code 0}. It is recommended to set positive value in case your
- * application performs frequent reads of large amount of data from the secondary file system in order to
- * avoid issues with increasing GC pauses or out-of-memory error.
- *
- * @return Maximum amount of pending data read from the secondary file system
- */
- public long getDualModeMaxPendingPutsSize() {
- return dualModeMaxPendingPutsSize;
- }
-
- /**
- * Set maximum amount of data in pending put operations.
- *
- * @param dualModeMaxPendingPutsSize Maximum amount of data in pending put operations.
- */
- public void setDualModeMaxPendingPutsSize(long dualModeMaxPendingPutsSize) {
- this.dualModeMaxPendingPutsSize = dualModeMaxPendingPutsSize;
- }
-
- /**
- * Get maximum default range size of a file being split during IGFS task execution. When IGFS task is about to
- * be executed, it requests file block locations first. Each location is defined as {@link org.apache.ignite.igfs.mapreduce.IgfsFileRange} which
- * has length. In case this parameter is set to positive value, then IGFS will split single file range into smaller
- * ranges with length not greater that this parameter. The only exception to this case is when maximum task range
- * length is smaller than file block size. In this case maximum task range size will be overridden and set to file
- * block size.
- * <p>
- * Note that this parameter is applied when task is split into jobs before {@link org.apache.ignite.igfs.mapreduce.IgfsRecordResolver} is
- * applied. Therefore, final file ranges being assigned to particular jobs could be greater than value of this
- * parameter depending on file data layout and selected resolver type.
- * <p>
- * Setting this parameter might be useful when file is highly colocated and have very long consequent data chunks
- * so that task execution suffers from insufficient parallelism. E.g., in case you have one IGFS node in topology
- * and want to process 1Gb file, then only single range of length 1Gb will be returned. This will result in
- * a single job which will be processed in one thread. But in case you provide this configuration parameter and set
- * maximum range length to 16Mb, then 64 ranges will be returned resulting in 64 jobs which could be executed in
- * parallel.
- * <p>
- * Note that some {@code IgniteFs.execute()} methods can override value of this parameter.
- * <p>
- * In case value of this parameter is set to {@code 0} or negative value, it is simply ignored. Default value is
- * {@code 0}.
- *
- * @return Maximum range size of a file being split during IGFS task execution.
- */
- public long getMaximumTaskRangeLength() {
- return maxTaskRangeLen;
- }
-
- /**
- * Set maximum default range size of a file being split during IGFS task execution.
- * See {@link #getMaximumTaskRangeLength()} for more details.
- *
- * @param maxTaskRangeLen Set maximum default range size of a file being split during IGFS task execution.
- */
- public void setMaximumTaskRangeLength(long maxTaskRangeLen) {
- this.maxTaskRangeLen = maxTaskRangeLen;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(IgfsConfiguration.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index cf88778..8bd2f83 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -20,7 +20,6 @@ package org.apache.ignite.configuration;
import org.apache.ignite.*;
import org.apache.ignite.events.*;
import org.apache.ignite.internal.managers.eventstorage.*;
-import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.apache.ignite.lang.*;
import org.apache.ignite.lifecycle.*;
@@ -345,7 +344,7 @@ public class IgniteConfiguration {
private Map<IgnitePredicate<? extends Event>, int[]> lsnrs;
/** IGFS configuration. */
- private IgfsConfiguration[] igfsCfg;
+ private FileSystemConfiguration[] igfsCfg;
/** Streamer configuration. */
private StreamerConfiguration[] streamerCfg;
@@ -354,7 +353,7 @@ public class IgniteConfiguration {
private ServiceConfiguration[] svcCfgs;
/** Hadoop configuration. */
- private GridHadoopConfiguration hadoopCfg;
+ private HadoopConfiguration hadoopCfg;
/** Client access configuration. */
private ConnectorConfiguration connectorCfg = new ConnectorConfiguration();
@@ -418,7 +417,7 @@ public class IgniteConfiguration {
ggHome = cfg.getIgniteHome();
ggWork = cfg.getWorkDirectory();
gridName = cfg.getGridName();
- igfsCfg = cfg.getIgfsConfiguration();
+ igfsCfg = cfg.getFileSystemConfiguration();
igfsPoolSize = cfg.getIgfsThreadPoolSize();
hadoopCfg = cfg.getHadoopConfiguration();
inclEvtTypes = cfg.getIncludeEventTypes();
@@ -1728,20 +1727,20 @@ public class IgniteConfiguration {
}
/**
- * Gets IGFS configurations.
+ * Gets IGFS (Ignite In-Memory File System) configurations.
*
* @return IGFS configurations.
*/
- public IgfsConfiguration[] getIgfsConfiguration() {
+ public FileSystemConfiguration[] getFileSystemConfiguration() {
return igfsCfg;
}
/**
- * Sets IGFS configurations.
+ * Sets IGFS (Ignite In-Memory File System) configurations.
*
* @param igfsCfg IGFS configurations.
*/
- public void setIgfsConfiguration(IgfsConfiguration... igfsCfg) {
+ public void setFileSystemConfiguration(FileSystemConfiguration... igfsCfg) {
this.igfsCfg = igfsCfg;
}
@@ -1768,7 +1767,7 @@ public class IgniteConfiguration {
*
* @return Hadoop configuration.
*/
- public GridHadoopConfiguration getHadoopConfiguration() {
+ public HadoopConfiguration getHadoopConfiguration() {
return hadoopCfg;
}
@@ -1777,7 +1776,7 @@ public class IgniteConfiguration {
*
* @param hadoopCfg Hadoop configuration.
*/
- public void setHadoopConfiguration(GridHadoopConfiguration hadoopCfg) {
+ public void setHadoopConfiguration(HadoopConfiguration hadoopCfg) {
this.hadoopCfg = hadoopCfg;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/Igfs.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/Igfs.java b/modules/core/src/main/java/org/apache/ignite/igfs/Igfs.java
deleted file mode 100644
index 48b9b58..0000000
--- a/modules/core/src/main/java/org/apache/ignite/igfs/Igfs.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.ignite.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Common file system interface. It provides a typical generalized "view" of any file system:
- * <ul>
- * <li>list directories or get information for a single path</li>
- * <li>create/move/delete files or directories</li>
- * <li>write/read data streams into/from files</li>
- * </ul>
- *
- * This is the minimum of functionality that is needed to work as secondary file system in dual modes of IGFS.
- */
-public interface Igfs {
- /** File property: user name. */
- public static final String PROP_USER_NAME = "usrName";
-
- /** File property: group name. */
- public static final String PROP_GROUP_NAME = "grpName";
-
- /** File property: permission. */
- public static final String PROP_PERMISSION = "permission";
-
- /**
- * Checks if the specified path exists in the file system.
- *
- * @param path Path to check for existence in the file system.
- * @return {@code True} if such file exists, otherwise - {@code false}.
- * @throws IgniteException In case of error.
- */
- public boolean exists(IgfsPath path);
-
- /**
- * Updates file information for the specified path. Existent properties, not listed in the passed collection,
- * will not be affected. Other properties will be added or overwritten. Passed properties with {@code null} values
- * will be removed from the stored properties or ignored if they don't exist in the file info.
- * <p>
- * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties will be propagated
- * to the secondary file system:
- * <ul>
- * <li>{@code usrName} - file owner name;</li>
- * <li>{@code grpName} - file owner group;</li>
- * <li>{@code permission} - Unix-style string representing file permissions.</li>
- * </ul>
- *
- * @param path File path to set properties for.
- * @param props Properties to update.
- * @return File information for specified path or {@code null} if such path does not exist.
- * @throws IgniteException In case of error.
- */
- public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteException;
-
- /**
- * Renames/moves a file.
- * <p>
- * You are free to rename/move data files as you wish, but directories can be only renamed.
- * You cannot move the directory between different parent directories.
- * <p>
- * Examples:
- * <ul>
- * <li>"/work/file.txt" => "/home/project/Presentation Scenario.txt"</li>
- * <li>"/work" => "/work-2012.bkp"</li>
- * <li>"/work" => "<strike>/backups/work</strike>" - such operation is restricted for directories.</li>
- * </ul>
- *
- * @param src Source file path to rename.
- * @param dest Destination file path. If destination path is a directory, then source file will be placed
- * into destination directory with original name.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If source file doesn't exist.
- */
- public void rename(IgfsPath src, IgfsPath dest) throws IgniteException;
-
- /**
- * Deletes file.
- *
- * @param path File path to delete.
- * @param recursive Delete non-empty directories recursively.
- * @return {@code True} in case of success, {@code false} otherwise.
- * @throws IgniteException In case of error.
- */
- public boolean delete(IgfsPath path, boolean recursive) throws IgniteException;
-
- /**
- * Creates directories under specified path.
- *
- * @param path Path of directories chain to create.
- * @throws IgniteException In case of error.
- */
- public void mkdirs(IgfsPath path) throws IgniteException;
-
- /**
- * Creates directories under specified path with the specified properties.
- *
- * @param path Path of directories chain to create.
- * @param props Metadata properties to set on created directories.
- * @throws IgniteException In case of error.
- */
- public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Lists file paths under the specified path.
- *
- * @param path Path to list files under.
- * @return List of files under the specified path.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteException;
-
- /**
- * Lists files under the specified path.
- *
- * @param path Path to list files under.
- * @return List of files under the specified path.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteException;
-
- /**
- * Opens a file for reading.
- *
- * @param path File path to read.
- * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
- * @return File input stream to read data from.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist.
- */
- public IgfsReader open(IgfsPath path, int bufSize) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- public OutputStream create(IgfsPath path, boolean overwrite) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @param replication Replication factor.
- * @param blockSize Block size.
- * @param props File properties to set.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication, long blockSize,
- @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Opens an output stream to an existing file for appending data.
- *
- * @param path File path to append.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param create Create file if it doesn't exist yet.
- * @param props File properties to set only in case it file was just created.
- * @return File output stream to append data to.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
- */
- public OutputStream append(IgfsPath path, int bufSize, boolean create, @Nullable Map<String, String> props)
- throws IgniteException;
-
- /**
- * Gets file information for the specified path.
- *
- * @param path Path to get information for.
- * @return File information for specified path or {@code null} if such path does not exist.
- * @throws IgniteException In case of error.
- */
- public IgfsFile info(IgfsPath path) throws IgniteException;
-
- /**
- * Gets used space in bytes.
- *
- * @return Used space in bytes.
- * @throws IgniteException In case of error.
- */
- public long usedSpaceSize() throws IgniteException;
-
- /**
- * Gets the implementation specific properties of file system.
- *
- * @return Map of properties.
- */
- public Map<String,String> properties();
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/IgfsBlockLocation.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsBlockLocation.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsBlockLocation.java
index c4f28c6..afd0314 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsBlockLocation.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsBlockLocation.java
@@ -22,7 +22,7 @@ import java.util.*;
/**
* {@code IGFS} file's data block location in the grid. It is used to determine
* node affinity of a certain file block within the Grid by calling
- * {@link org.apache.ignite.IgniteFs#affinity(IgfsPath, long, long)} method.
+ * {@link org.apache.ignite.IgniteFileSystem#affinity(IgfsPath, long, long)} method.
*/
public interface IgfsBlockLocation {
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/IgfsFile.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsFile.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsFile.java
index 172dca1..550679a 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsFile.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsFile.java
@@ -77,7 +77,7 @@ public interface IgfsFile {
/**
* Gets file last access time. File last access time is not updated automatically due to
* performance considerations and can be updated on demand with
- * {@link org.apache.ignite.IgniteFs#setTimes(IgfsPath, long, long)} method.
+ * {@link org.apache.ignite.IgniteFileSystem#setTimes(IgfsPath, long, long)} method.
* <p>
* By default last access time equals file creation time.
*
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
index 308dbcb..c2ddbb0 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
@@ -17,13 +17,15 @@
package org.apache.ignite.igfs;
+import org.apache.ignite.igfs.secondary.*;
+
import java.io.*;
/**
* {@code IGFS} input stream to read data from the file system.
* It provides several additional methods for asynchronous access.
*/
-public abstract class IgfsInputStream extends InputStream implements IgfsReader {
+public abstract class IgfsInputStream extends InputStream {
/**
* Gets file length during file open.
*
@@ -76,5 +78,5 @@ public abstract class IgfsInputStream extends InputStream implements IgfsReader
* @return Total number of bytes read into the buffer, or -1 if there is no more data (EOF).
* @throws IOException In case of IO exception.
*/
- @Override public abstract int read(long pos, byte[] buf, int off, int len) throws IOException;
+ public abstract int read(long pos, byte[] buf, int off, int len) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMetrics.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMetrics.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMetrics.java
index afdae1a..50b5435 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMetrics.java
@@ -33,7 +33,7 @@ public interface IgfsMetrics {
/**
* Gets maximum amount of data that can be stored on local node. This metrics is either
- * equal to {@link org.apache.ignite.configuration.IgfsConfiguration#getMaxSpaceSize()}, or, if it is {@code 0}, equal to
+ * equal to {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaxSpaceSize()}, or, if it is {@code 0}, equal to
* {@code 80%} of maximum heap size allocated for JVM.
*
* @return Maximum IGFS local space size.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMode.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMode.java
index 3c440ab..2c9fcdd 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMode.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMode.java
@@ -24,7 +24,7 @@ import org.jetbrains.annotations.*;
* Secondary Hadoop file system is provided for pass-through, write-through, and
* read-through purposes.
* <p>
- * This mode is configured via {@link org.apache.ignite.configuration.IgfsConfiguration#getDefaultMode()}
+ * This mode is configured via {@link org.apache.ignite.configuration.FileSystemConfiguration#getDefaultMode()}
* configuration property.
*/
public enum IgfsMode {
@@ -39,7 +39,7 @@ public enum IgfsMode {
* through to secondary Hadoop file system. If this mode is enabled, then
* secondary Hadoop file system must be configured.
*
- * @see org.apache.ignite.configuration.IgfsConfiguration#getSecondaryFileSystem()
+ * @see org.apache.ignite.configuration.FileSystemConfiguration#getSecondaryFileSystem()
*/
PROXY,
@@ -50,7 +50,7 @@ public enum IgfsMode {
* If secondary Hadoop file system is not configured, then this mode behaves like
* {@link #PRIMARY} mode.
*
- * @see org.apache.ignite.configuration.IgfsConfiguration#getSecondaryFileSystem()
+ * @see org.apache.ignite.configuration.FileSystemConfiguration#getSecondaryFileSystem()
*/
DUAL_SYNC,
@@ -61,7 +61,7 @@ public enum IgfsMode {
* If secondary Hadoop file system is not configured, then this mode behaves like
* {@link #PRIMARY} mode.
*
- * @see org.apache.ignite.configuration.IgfsConfiguration#getSecondaryFileSystem()
+ * @see org.apache.ignite.configuration.FileSystemConfiguration#getSecondaryFileSystem()
*/
DUAL_ASYNC;
[30/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsInProc.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsInProc.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsInProc.java
new file mode 100644
index 0000000..a8eb58c
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsInProc.java
@@ -0,0 +1,409 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.commons.logging.*;
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+/**
+ * Communication with grid in the same process.
+ */
+public class HadoopIgfsInProc implements HadoopIgfsEx {
+ /** Target IGFS. */
+ private final IgfsEx igfs;
+
+ /** Buffer size. */
+ private final int bufSize;
+
+ /** Event listeners. */
+ private final Map<HadoopIgfsStreamDelegate, HadoopIgfsStreamEventListener> lsnrs =
+ new ConcurrentHashMap<>();
+
+ /** Logger. */
+ private final Log log;
+
+ /**
+ * Constructor.
+ *
+ * @param igfs Target IGFS.
+ * @param log Log.
+ */
+ public HadoopIgfsInProc(IgfsEx igfs, Log log) {
+ this.igfs = igfs;
+ this.log = log;
+
+ bufSize = igfs.configuration().getBlockSize() * 2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsHandshakeResponse handshake(String logDir) {
+ igfs.clientLogDirectory(logDir);
+
+ return new IgfsHandshakeResponse(igfs.name(), igfs.proxyPaths(), igfs.groupBlockSize(),
+ igfs.globalSampling());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close(boolean force) {
+ // Perform cleanup.
+ for (HadoopIgfsStreamEventListener lsnr : lsnrs.values()) {
+ try {
+ lsnr.onClose();
+ }
+ catch (IgniteCheckedException e) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to notify stream event listener", e);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsFile info(IgfsPath path) throws IgniteCheckedException {
+ try {
+ return igfs.info(path);
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to get file info because Grid is stopping: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteCheckedException {
+ try {
+ return igfs.update(path, props);
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to update file because Grid is stopping: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Boolean setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteCheckedException {
+ try {
+ igfs.setTimes(path, accessTime, modificationTime);
+
+ return true;
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to set path times because Grid is stopping: " +
+ path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Boolean rename(IgfsPath src, IgfsPath dest) throws IgniteCheckedException {
+ try {
+ igfs.rename(src, dest);
+
+ return true;
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to rename path because Grid is stopping: " + src);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Boolean delete(IgfsPath path, boolean recursive) throws IgniteCheckedException {
+ try {
+ return igfs.delete(path, recursive);
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to delete path because Grid is stopping: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsStatus fsStatus() throws IgniteCheckedException {
+ try {
+ return igfs.globalSpace();
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to get file system status because Grid is " +
+ "stopping.");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteCheckedException {
+ try {
+ return igfs.listPaths(path);
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to list paths because Grid is stopping: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteCheckedException {
+ try {
+ return igfs.listFiles(path);
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to list files because Grid is stopping: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Boolean mkdirs(IgfsPath path, Map<String, String> props) throws IgniteCheckedException {
+ try {
+ igfs.mkdirs(path, props);
+
+ return true;
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to create directory because Grid is stopping: " +
+ path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsPathSummary contentSummary(IgfsPath path) throws IgniteCheckedException {
+ try {
+ return igfs.summary(path);
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to get content summary because Grid is stopping: " +
+ path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len)
+ throws IgniteCheckedException {
+ try {
+ return igfs.affinity(path, start, len);
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to get affinity because Grid is stopping: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopIgfsStreamDelegate open(IgfsPath path) throws IgniteCheckedException {
+ try {
+ IgfsInputStreamAdapter stream = igfs.open(path, bufSize);
+
+ return new HadoopIgfsStreamDelegate(this, stream, stream.fileInfo().length());
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to open file because Grid is stopping: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopIgfsStreamDelegate open(IgfsPath path, int seqReadsBeforePrefetch)
+ throws IgniteCheckedException {
+ try {
+ IgfsInputStreamAdapter stream = igfs.open(path, bufSize, seqReadsBeforePrefetch);
+
+ return new HadoopIgfsStreamDelegate(this, stream, stream.fileInfo().length());
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to open file because Grid is stopping: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopIgfsStreamDelegate create(IgfsPath path, boolean overwrite, boolean colocate,
+ int replication, long blockSize, @Nullable Map<String, String> props) throws IgniteCheckedException {
+ try {
+ IgfsOutputStream stream = igfs.create(path, bufSize, overwrite,
+ colocate ? igfs.nextAffinityKey() : null, replication, blockSize, props);
+
+ return new HadoopIgfsStreamDelegate(this, stream);
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to create file because Grid is stopping: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopIgfsStreamDelegate append(IgfsPath path, boolean create,
+ @Nullable Map<String, String> props) throws IgniteCheckedException {
+ try {
+ IgfsOutputStream stream = igfs.append(path, bufSize, create, props);
+
+ return new HadoopIgfsStreamDelegate(this, stream);
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+ catch (IllegalStateException e) {
+ throw new HadoopIgfsCommunicationException("Failed to append file because Grid is stopping: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridPlainFuture<byte[]> readData(HadoopIgfsStreamDelegate delegate, long pos, int len,
+ @Nullable byte[] outBuf, int outOff, int outLen) {
+ IgfsInputStreamAdapter stream = delegate.target();
+
+ try {
+ byte[] res = null;
+
+ if (outBuf != null) {
+ int outTailLen = outBuf.length - outOff;
+
+ if (len <= outTailLen)
+ stream.readFully(pos, outBuf, outOff, len);
+ else {
+ stream.readFully(pos, outBuf, outOff, outTailLen);
+
+ int remainderLen = len - outTailLen;
+
+ res = new byte[remainderLen];
+
+ stream.readFully(pos, res, 0, remainderLen);
+ }
+ } else {
+ res = new byte[len];
+
+ stream.readFully(pos, res, 0, len);
+ }
+
+ return new GridPlainFutureAdapter<>(res);
+ }
+ catch (IllegalStateException | IOException e) {
+ HadoopIgfsStreamEventListener lsnr = lsnrs.get(delegate);
+
+ if (lsnr != null)
+ lsnr.onError(e.getMessage());
+
+ return new GridPlainFutureAdapter<>(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeData(HadoopIgfsStreamDelegate delegate, byte[] data, int off, int len)
+ throws IOException {
+ try {
+ IgfsOutputStream stream = delegate.target();
+
+ stream.write(data, off, len);
+ }
+ catch (IllegalStateException | IOException e) {
+ HadoopIgfsStreamEventListener lsnr = lsnrs.get(delegate);
+
+ if (lsnr != null)
+ lsnr.onError(e.getMessage());
+
+ if (e instanceof IllegalStateException)
+ throw new IOException("Failed to write data to IGFS stream because Grid is stopping.", e);
+ else
+ throw e;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void flush(HadoopIgfsStreamDelegate delegate) throws IOException {
+ try {
+ IgfsOutputStream stream = delegate.target();
+
+ stream.flush();
+ }
+ catch (IllegalStateException | IOException e) {
+ HadoopIgfsStreamEventListener lsnr = lsnrs.get(delegate);
+
+ if (lsnr != null)
+ lsnr.onError(e.getMessage());
+
+ if (e instanceof IllegalStateException)
+ throw new IOException("Failed to flush data to IGFS stream because Grid is stopping.", e);
+ else
+ throw e;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void closeStream(HadoopIgfsStreamDelegate desc) throws IOException {
+ Closeable closeable = desc.target();
+
+ try {
+ closeable.close();
+ }
+ catch (IllegalStateException e) {
+ throw new IOException("Failed to close IGFS stream because Grid is stopping.", e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void addEventListener(HadoopIgfsStreamDelegate delegate,
+ HadoopIgfsStreamEventListener lsnr) {
+ HadoopIgfsStreamEventListener lsnr0 = lsnrs.put(delegate, lsnr);
+
+ assert lsnr0 == null || lsnr0 == lsnr;
+
+ if (log.isDebugEnabled())
+ log.debug("Added stream event listener [delegate=" + delegate + ']');
+ }
+
+ /** {@inheritDoc} */
+ @Override public void removeEventListener(HadoopIgfsStreamDelegate delegate) {
+ HadoopIgfsStreamEventListener lsnr0 = lsnrs.remove(delegate);
+
+ if (lsnr0 != null && log.isDebugEnabled())
+ log.debug("Removed stream event listener [delegate=" + delegate + ']');
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsInputStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsInputStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsInputStream.java
new file mode 100644
index 0000000..c335a62
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsInputStream.java
@@ -0,0 +1,626 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.commons.logging.*;
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.igfs.common.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * IGFS input stream wrapper for hadoop interfaces.
+ */
+@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
+public final class HadoopIgfsInputStream extends InputStream implements Seekable, PositionedReadable,
+ HadoopIgfsStreamEventListener {
+ /** Minimum buffer size. */
+ private static final int MIN_BUF_SIZE = 4 * 1024;
+
+ /** Server stream delegate. */
+ private HadoopIgfsStreamDelegate delegate;
+
+ /** Stream ID used by logger. */
+ private long logStreamId;
+
+ /** Stream position. */
+ private long pos;
+
+ /** Stream read limit. */
+ private long limit;
+
+ /** Mark position. */
+ private long markPos = -1;
+
+ /** Prefetch buffer. */
+ private DoubleFetchBuffer buf = new DoubleFetchBuffer();
+
+ /** Buffer half size for double-buffering. */
+ private int bufHalfSize;
+
+ /** Closed flag. */
+ private volatile boolean closed;
+
+ /** Flag set if stream was closed due to connection breakage. */
+ private boolean connBroken;
+
+ /** Logger. */
+ private Log log;
+
+ /** Client logger. */
+ private IgfsLogger clientLog;
+
+ /** Read time. */
+ private long readTime;
+
+ /** User time. */
+ private long userTime;
+
+ /** Last timestamp. */
+ private long lastTs;
+
+ /** Amount of read bytes. */
+ private long total;
+
+ /**
+ * Creates input stream.
+ *
+ * @param delegate Server stream delegate.
+ * @param limit Read limit.
+ * @param bufSize Buffer size.
+ * @param log Log.
+ * @param clientLog Client logger.
+ */
+ public HadoopIgfsInputStream(HadoopIgfsStreamDelegate delegate, long limit, int bufSize, Log log,
+ IgfsLogger clientLog, long logStreamId) {
+ assert limit >= 0;
+
+ this.delegate = delegate;
+ this.limit = limit;
+ this.log = log;
+ this.clientLog = clientLog;
+ this.logStreamId = logStreamId;
+
+ bufHalfSize = Math.max(bufSize, MIN_BUF_SIZE);
+
+ lastTs = System.nanoTime();
+
+ delegate.hadoop().addEventListener(delegate, this);
+ }
+
+ /**
+ * Read start.
+ */
+ private void readStart() {
+ long now = System.nanoTime();
+
+ userTime += now - lastTs;
+
+ lastTs = now;
+ }
+
+ /**
+ * Read end.
+ */
+ private void readEnd() {
+ long now = System.nanoTime();
+
+ readTime += now - lastTs;
+
+ lastTs = now;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int read() throws IOException {
+ checkClosed();
+
+ readStart();
+
+ try {
+ if (eof())
+ return -1;
+
+ buf.refreshAhead(pos);
+
+ int res = buf.atPosition(pos);
+
+ pos++;
+ total++;
+
+ buf.refreshAhead(pos);
+
+ return res;
+ }
+ catch (IgniteCheckedException e) {
+ throw HadoopIgfsUtils.cast(e);
+ }
+ finally {
+ readEnd();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int read(@NotNull byte[] b, int off, int len) throws IOException {
+ checkClosed();
+
+ if (eof())
+ return -1;
+
+ readStart();
+
+ try {
+ long remaining = limit - pos;
+
+ int read = buf.flatten(b, pos, off, len);
+
+ pos += read;
+ total += read;
+ remaining -= read;
+
+ if (remaining > 0 && read != len) {
+ int readAmt = (int)Math.min(remaining, len - read);
+
+ delegate.hadoop().readData(delegate, pos, readAmt, b, off + read, len - read).get();
+
+ read += readAmt;
+ pos += readAmt;
+ total += readAmt;
+ }
+
+ buf.refreshAhead(pos);
+
+ return read;
+ }
+ catch (IgniteCheckedException e) {
+ throw HadoopIgfsUtils.cast(e);
+ }
+ finally {
+ readEnd();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized long skip(long n) throws IOException {
+ checkClosed();
+
+ if (clientLog.isLogEnabled())
+ clientLog.logSkip(logStreamId, n);
+
+ long oldPos = pos;
+
+ if (pos + n <= limit)
+ pos += n;
+ else
+ pos = limit;
+
+ buf.refreshAhead(pos);
+
+ return pos - oldPos;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int available() throws IOException {
+ checkClosed();
+
+ int available = buf.available(pos);
+
+ assert available >= 0;
+
+ return available;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void close() throws IOException {
+ if (!closed) {
+ readStart();
+
+ if (log.isDebugEnabled())
+ log.debug("Closing input stream: " + delegate);
+
+ delegate.hadoop().closeStream(delegate);
+
+ readEnd();
+
+ if (clientLog.isLogEnabled())
+ clientLog.logCloseIn(logStreamId, userTime, readTime, total);
+
+ markClosed(false);
+
+ if (log.isDebugEnabled())
+ log.debug("Closed stream [delegate=" + delegate + ", readTime=" + readTime +
+ ", userTime=" + userTime + ']');
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void mark(int readLimit) {
+ markPos = pos;
+
+ if (clientLog.isLogEnabled())
+ clientLog.logMark(logStreamId, readLimit);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void reset() throws IOException {
+ checkClosed();
+
+ if (clientLog.isLogEnabled())
+ clientLog.logReset(logStreamId);
+
+ if (markPos == -1)
+ throw new IOException("Stream was not marked.");
+
+ pos = markPos;
+
+ buf.refreshAhead(pos);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean markSupported() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int read(long position, byte[] buf, int off, int len) throws IOException {
+ long remaining = limit - position;
+
+ int read = (int)Math.min(len, remaining);
+
+ // Return -1 at EOF.
+ if (read == 0)
+ return -1;
+
+ readFully(position, buf, off, read);
+
+ return read;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void readFully(long position, byte[] buf, int off, int len) throws IOException {
+ long remaining = limit - position;
+
+ checkClosed();
+
+ if (len > remaining)
+ throw new EOFException("End of stream reached before data was fully read.");
+
+ readStart();
+
+ try {
+ int read = this.buf.flatten(buf, position, off, len);
+
+ total += read;
+
+ if (read != len) {
+ int readAmt = len - read;
+
+ delegate.hadoop().readData(delegate, position + read, readAmt, buf, off + read, readAmt).get();
+
+ total += readAmt;
+ }
+
+ if (clientLog.isLogEnabled())
+ clientLog.logRandomRead(logStreamId, position, len);
+ }
+ catch (IgniteCheckedException e) {
+ throw HadoopIgfsUtils.cast(e);
+ }
+ finally {
+ readEnd();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFully(long position, byte[] buf) throws IOException {
+ readFully(position, buf, 0, buf.length);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void seek(long pos) throws IOException {
+ A.ensure(pos >= 0, "position must be non-negative");
+
+ checkClosed();
+
+ if (clientLog.isLogEnabled())
+ clientLog.logSeek(logStreamId, pos);
+
+ if (pos > limit)
+ pos = limit;
+
+ if (log.isDebugEnabled())
+ log.debug("Seek to position [delegate=" + delegate + ", pos=" + pos + ", oldPos=" + this.pos + ']');
+
+ this.pos = pos;
+
+ buf.refreshAhead(pos);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized long getPos() {
+ return pos;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized boolean seekToNewSource(long targetPos) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onClose() {
+ markClosed(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onError(String errMsg) {
+ // No-op.
+ }
+
+ /**
+ * Marks stream as closed.
+ *
+ * @param connBroken {@code True} if connection with server was lost.
+ */
+ private void markClosed(boolean connBroken) {
+ // It is ok to have race here.
+ if (!closed) {
+ closed = true;
+
+ this.connBroken = connBroken;
+
+ delegate.hadoop().removeEventListener(delegate);
+ }
+ }
+
+ /**
+ * @throws IOException If check failed.
+ */
+ private void checkClosed() throws IOException {
+ if (closed) {
+ if (connBroken)
+ throw new IOException("Server connection was lost.");
+ else
+ throw new IOException("Stream is closed.");
+ }
+ }
+
+ /**
+ * @return {@code True} if end of stream reached.
+ */
+ private boolean eof() {
+ return limit == pos;
+ }
+
+ /**
+ * Asynchronous prefetch buffer.
+ */
+ private static class FetchBufferPart {
+ /** Read future. */
+ private GridPlainFuture<byte[]> readFut;
+
+ /** Position of cached chunk in file. */
+ private long pos;
+
+ /** Prefetch length. Need to store as read future result might be not available yet. */
+ private int len;
+
+ /**
+ * Creates fetch buffer part.
+ *
+ * @param readFut Read future for this buffer.
+ * @param pos Read position.
+ * @param len Chunk length.
+ */
+ private FetchBufferPart(GridPlainFuture<byte[]> readFut, long pos, int len) {
+ this.readFut = readFut;
+ this.pos = pos;
+ this.len = len;
+ }
+
+ /**
+ * Copies cached data if specified position matches cached region.
+ *
+ * @param dst Destination buffer.
+ * @param pos Read position in file.
+ * @param dstOff Offset in destination buffer from which start writing.
+ * @param len Maximum number of bytes to copy.
+ * @return Number of bytes copied.
+ * @throws IgniteCheckedException If read future failed.
+ */
+ public int flatten(byte[] dst, long pos, int dstOff, int len) throws IgniteCheckedException {
+ // If read start position is within cached boundaries.
+ if (contains(pos)) {
+ byte[] data = readFut.get();
+
+ int srcPos = (int)(pos - this.pos);
+ int cpLen = Math.min(len, data.length - srcPos);
+
+ U.arrayCopy(data, srcPos, dst, dstOff, cpLen);
+
+ return cpLen;
+ }
+
+ return 0;
+ }
+
+ /**
+ * @return {@code True} if data is ready to be read.
+ */
+ public boolean ready() {
+ return readFut.isDone();
+ }
+
+ /**
+ * Checks if current buffer part contains given position.
+ *
+ * @param pos Position to check.
+ * @return {@code True} if position matches buffer region.
+ */
+ public boolean contains(long pos) {
+ return this.pos <= pos && this.pos + len > pos;
+ }
+ }
+
+ private class DoubleFetchBuffer {
+ /** */
+ private FetchBufferPart first;
+
+ /** */
+ private FetchBufferPart second;
+
+ /**
+ * Copies fetched data from both buffers to destination array if cached region matched read position.
+ *
+ * @param dst Destination buffer.
+ * @param pos Read position in file.
+ * @param dstOff Destination buffer offset.
+ * @param len Maximum number of bytes to copy.
+ * @return Number of bytes copied.
+ * @throws IgniteCheckedException If any read operation failed.
+ */
+ public int flatten(byte[] dst, long pos, int dstOff, int len) throws IgniteCheckedException {
+ assert dstOff >= 0;
+ assert dstOff + len <= dst.length : "Invalid indices [dst.length=" + dst.length + ", dstOff=" + dstOff +
+ ", len=" + len + ']';
+
+ int bytesCopied = 0;
+
+ if (first != null) {
+ bytesCopied += first.flatten(dst, pos, dstOff, len);
+
+ if (bytesCopied != len && second != null) {
+ assert second.pos == first.pos + first.len;
+
+ bytesCopied += second.flatten(dst, pos + bytesCopied, dstOff + bytesCopied, len - bytesCopied);
+ }
+ }
+
+ return bytesCopied;
+ }
+
+ /**
+ * Gets byte at specified position in buffer.
+ *
+ * @param pos Stream position.
+ * @return Read byte.
+ * @throws IgniteCheckedException If read failed.
+ */
+ public int atPosition(long pos) throws IgniteCheckedException {
+ // Should not reach here if stream contains no data.
+ assert first != null;
+
+ if (first.contains(pos)) {
+ byte[] bytes = first.readFut.get();
+
+ return bytes[((int)(pos - first.pos))] & 0xFF;
+ }
+ else {
+ assert second != null;
+ assert second.contains(pos);
+
+ byte[] bytes = second.readFut.get();
+
+ return bytes[((int)(pos - second.pos))] & 0xFF;
+ }
+ }
+
+ /**
+ * Starts asynchronous buffer refresh if needed, depending on current position.
+ *
+ * @param pos Current stream position.
+ */
+ public void refreshAhead(long pos) {
+ if (fullPrefetch(pos)) {
+ first = fetch(pos, bufHalfSize);
+ second = fetch(pos + bufHalfSize, bufHalfSize);
+ }
+ else if (needFlip(pos)) {
+ first = second;
+
+ second = fetch(first.pos + first.len, bufHalfSize);
+ }
+ }
+
+ /**
+ * @param pos Position from which read is expected.
+ * @return Number of bytes available to be read without blocking.
+ */
+ public int available(long pos) {
+ int available = 0;
+
+ if (first != null) {
+ if (first.contains(pos)) {
+ if (first.ready()) {
+ available += (pos - first.pos);
+
+ if (second != null && second.ready())
+ available += second.len;
+ }
+ }
+ else {
+ if (second != null && second.contains(pos) && second.ready())
+ available += (pos - second.pos);
+ }
+ }
+
+ return available;
+ }
+
+ /**
+ * Checks if position shifted enough to forget previous buffer.
+ *
+ * @param pos Current position.
+ * @return {@code True} if need flip buffers.
+ */
+ private boolean needFlip(long pos) {
+ // Return true if we read more then half of second buffer.
+ return second != null && second.contains(pos);
+ }
+
+ /**
+ * Determines if all cached bytes should be discarded and new region should be
+ * prefetched.
+ *
+ * @param curPos Current stream position.
+ * @return {@code True} if need to refresh both blocks.
+ */
+ private boolean fullPrefetch(long curPos) {
+ // If no data was prefetched yet, return true.
+ return first == null || curPos < first.pos || (second != null && curPos >= second.pos + second.len);
+ }
+
+ /**
+ * Starts asynchronous fetch for given region.
+ *
+ * @param pos Position to read from.
+ * @param size Number of bytes to read.
+ * @return Fetch buffer part.
+ */
+ private FetchBufferPart fetch(long pos, int size) {
+ long remaining = limit - pos;
+
+ size = (int)Math.min(size, remaining);
+
+ return size <= 0 ? null :
+ new FetchBufferPart(delegate.hadoop().readData(delegate, pos, size, null, 0, 0), pos, size);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsIo.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsIo.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsIo.java
new file mode 100644
index 0000000..88dd896
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsIo.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.igfs.common.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.jetbrains.annotations.*;
+
+/**
+ * IO abstraction layer for IGFS client. Two kind of messages are expected to be sent: requests with response
+ * and request without response.
+ */
+public interface HadoopIgfsIo {
+ /**
+ * Sends given IGFS client message and asynchronously awaits for response.
+ *
+ * @param msg Message to send.
+ * @return Future that will be completed.
+ * @throws IgniteCheckedException If a message cannot be sent (connection is broken or client was closed).
+ */
+ public GridPlainFuture<IgfsMessage> send(IgfsMessage msg) throws IgniteCheckedException;
+
+ /**
+ * Sends given IGFS client message and asynchronously awaits for response. When IO detects response
+ * beginning for given message it stops reading data and passes input stream to closure which can read
+ * response in a specific way.
+ *
+ * @param msg Message to send.
+ * @param outBuf Output buffer. If {@code null}, the output buffer is not used.
+ * @param outOff Output buffer offset.
+ * @param outLen Output buffer length.
+ * @return Future that will be completed when response is returned from closure.
+ * @throws IgniteCheckedException If a message cannot be sent (connection is broken or client was closed).
+ */
+ public <T> GridPlainFuture<T> send(IgfsMessage msg, @Nullable byte[] outBuf, int outOff, int outLen)
+ throws IgniteCheckedException;
+
+ /**
+ * Sends given message and does not wait for response.
+ *
+ * @param msg Message to send.
+ * @throws IgniteCheckedException If send failed.
+ */
+ public void sendPlain(IgfsMessage msg) throws IgniteCheckedException;
+
+ /**
+ * Adds event listener that will be invoked when connection with server is lost or remote error has occurred.
+ * If connection is closed already, callback will be invoked synchronously inside this method.
+ *
+ * @param lsnr Event listener.
+ */
+ public void addEventListener(HadoopIgfsIpcIoListener lsnr);
+
+ /**
+ * Removes event listener that will be invoked when connection with server is lost or remote error has occurred.
+ *
+ * @param lsnr Event listener.
+ */
+ public void removeEventListener(HadoopIgfsIpcIoListener lsnr);
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsIpcIo.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsIpcIo.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsIpcIo.java
new file mode 100644
index 0000000..c9c61fe
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsIpcIo.java
@@ -0,0 +1,599 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.commons.logging.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.igfs.common.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.ipc.*;
+import org.apache.ignite.internal.util.ipc.shmem.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jdk8.backport.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import java.util.concurrent.locks.*;
+
+/**
+ * IO layer implementation based on blocking IPC streams.
+ */
+@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
+public class HadoopIgfsIpcIo implements HadoopIgfsIo {
+ /** Logger. */
+ private Log log;
+
+ /** Request futures map. */
+ private ConcurrentMap<Long, HadoopIgfsFuture> reqMap =
+ new ConcurrentHashMap8<>();
+
+ /** Request ID counter. */
+ private AtomicLong reqIdCnt = new AtomicLong();
+
+ /** Endpoint. */
+ private IpcEndpoint endpoint;
+
+ /** Endpoint output stream. */
+ private IgfsDataOutputStream out;
+
+ /** Protocol. */
+ private final IgfsMarshaller marsh;
+
+ /** Client reader thread. */
+ private Thread reader;
+
+ /** Lock for graceful shutdown. */
+ private final ReadWriteLock busyLock = new ReentrantReadWriteLock();
+
+ /** Stopping flag. */
+ private volatile boolean stopping;
+
+ /** Server endpoint address. */
+ private final String endpointAddr;
+
+ /** Number of open file system sessions. */
+ private final AtomicInteger activeCnt = new AtomicInteger(1);
+
+ /** Event listeners. */
+ private final Collection<HadoopIgfsIpcIoListener> lsnrs =
+ new GridConcurrentHashSet<>();
+
+ /** Cached connections. */
+ private static final ConcurrentMap<String, HadoopIgfsIpcIo> ipcCache =
+ new ConcurrentHashMap8<>();
+
+ /** Striped lock that prevents multiple instance creation in {@link #get(Log, String)}. */
+ private static final GridStripedLock initLock = new GridStripedLock(32);
+
+ /**
+ * @param endpointAddr Endpoint.
+ * @param marsh Protocol.
+ * @param log Logger to use.
+ */
+ public HadoopIgfsIpcIo(String endpointAddr, IgfsMarshaller marsh, Log log) {
+ assert endpointAddr != null;
+ assert marsh != null;
+
+ this.endpointAddr = endpointAddr;
+ this.marsh = marsh;
+ this.log = log;
+ }
+
+ /**
+ * Returns a started and valid instance of this class
+ * for a given endpoint.
+ *
+ * @param log Logger to use for new instance.
+ * @param endpoint Endpoint string.
+ * @return New or existing cached instance, which is started and operational.
+ * @throws IOException If new instance was created but failed to start.
+ */
+ public static HadoopIgfsIpcIo get(Log log, String endpoint) throws IOException {
+ while (true) {
+ HadoopIgfsIpcIo clientIo = ipcCache.get(endpoint);
+
+ if (clientIo != null) {
+ if (clientIo.acquire())
+ return clientIo;
+ else
+ // If concurrent close.
+ ipcCache.remove(endpoint, clientIo);
+ }
+ else {
+ Lock lock = initLock.getLock(endpoint);
+
+ lock.lock();
+
+ try {
+ clientIo = ipcCache.get(endpoint);
+
+ if (clientIo != null) { // Perform double check.
+ if (clientIo.acquire())
+ return clientIo;
+ else
+ // If concurrent close.
+ ipcCache.remove(endpoint, clientIo);
+ }
+
+ // Otherwise try creating a new one.
+ clientIo = new HadoopIgfsIpcIo(endpoint, new IgfsMarshaller(), log);
+
+ try {
+ clientIo.start();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException(e.getMessage(), e);
+ }
+
+ HadoopIgfsIpcIo old = ipcCache.putIfAbsent(endpoint, clientIo);
+
+ // Put in exclusive lock.
+ assert old == null;
+
+ return clientIo;
+ }
+ finally {
+ lock.unlock();
+ }
+ }
+ }
+ }
+
+ /**
+ * Increases usage count for this instance.
+ *
+ * @return {@code true} if usage count is greater than zero.
+ */
+ private boolean acquire() {
+ while (true) {
+ int cnt = activeCnt.get();
+
+ if (cnt == 0) {
+ if (log.isDebugEnabled())
+ log.debug("IPC IO not acquired (count was 0): " + this);
+
+ return false;
+ }
+
+ // Need to make sure that no-one decremented count in between.
+ if (activeCnt.compareAndSet(cnt, cnt + 1)) {
+ if (log.isDebugEnabled())
+ log.debug("IPC IO acquired: " + this);
+
+ return true;
+ }
+ }
+ }
+
+ /**
+ * Releases this instance, decrementing usage count.
+ * <p>
+ * If usage count becomes zero, the instance is stopped
+ * and removed from cache.
+ */
+ public void release() {
+ while (true) {
+ int cnt = activeCnt.get();
+
+ if (cnt == 0) {
+ if (log.isDebugEnabled())
+ log.debug("IPC IO not released (count was 0): " + this);
+
+ return;
+ }
+
+ if (activeCnt.compareAndSet(cnt, cnt - 1)) {
+ if (cnt == 1) {
+ ipcCache.remove(endpointAddr, this);
+
+ if (log.isDebugEnabled())
+ log.debug("IPC IO stopping as unused: " + this);
+
+ stop();
+ }
+ else if (log.isDebugEnabled())
+ log.debug("IPC IO released: " + this);
+
+ return;
+ }
+ }
+ }
+
+ /**
+ * Closes this IO instance, removing it from cache.
+ */
+ public void forceClose() {
+ if (ipcCache.remove(endpointAddr, this))
+ stop();
+ }
+
+ /**
+ * Starts the IO.
+ *
+ * @throws IgniteCheckedException If failed to connect the endpoint.
+ */
+ private void start() throws IgniteCheckedException {
+ boolean success = false;
+
+ try {
+ endpoint = IpcEndpointFactory.connectEndpoint(
+ endpointAddr, new GridLoggerProxy(new HadoopIgfsJclLogger(log), null, null, ""));
+
+ out = new IgfsDataOutputStream(new BufferedOutputStream(endpoint.outputStream()));
+
+ reader = new ReaderThread();
+
+ // Required for Hadoop 2.x
+ reader.setDaemon(true);
+
+ reader.start();
+
+ success = true;
+ }
+ catch (IgniteCheckedException e) {
+ IpcOutOfSystemResourcesException resEx = e.getCause(IpcOutOfSystemResourcesException.class);
+
+ if (resEx != null)
+ throw new IgniteCheckedException(IpcSharedMemoryServerEndpoint.OUT_OF_RESOURCES_MSG, resEx);
+
+ throw e;
+ }
+ finally {
+ if (!success)
+ stop();
+ }
+ }
+
+ /**
+ * Shuts down the IO. No send requests will be accepted anymore, all pending futures will be failed.
+ * Close listeners will be invoked as if connection is closed by server.
+ */
+ private void stop() {
+ close0(null);
+
+ if (reader != null) {
+ try {
+ U.interrupt(reader);
+ U.join(reader);
+
+ reader = null;
+ }
+ catch (IgniteInterruptedCheckedException ignored) {
+ Thread.currentThread().interrupt();
+
+ log.warn("Got interrupted while waiting for reader thread to shut down (will return).");
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void addEventListener(HadoopIgfsIpcIoListener lsnr) {
+ if (!busyLock.readLock().tryLock()) {
+ lsnr.onClose();
+
+ return;
+ }
+
+ boolean invokeNow = false;
+
+ try {
+ invokeNow = stopping;
+
+ if (!invokeNow)
+ lsnrs.add(lsnr);
+ }
+ finally {
+ busyLock.readLock().unlock();
+
+ if (invokeNow)
+ lsnr.onClose();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void removeEventListener(HadoopIgfsIpcIoListener lsnr) {
+ lsnrs.remove(lsnr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridPlainFuture<IgfsMessage> send(IgfsMessage msg) throws IgniteCheckedException {
+ return send(msg, null, 0, 0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T> GridPlainFuture<T> send(IgfsMessage msg, @Nullable byte[] outBuf, int outOff,
+ int outLen) throws IgniteCheckedException {
+ assert outBuf == null || msg.command() == IgfsIpcCommand.READ_BLOCK;
+
+ if (!busyLock.readLock().tryLock())
+ throw new HadoopIgfsCommunicationException("Failed to send message (client is being concurrently " +
+ "closed).");
+
+ try {
+ if (stopping)
+ throw new HadoopIgfsCommunicationException("Failed to send message (client is being concurrently " +
+ "closed).");
+
+ long reqId = reqIdCnt.getAndIncrement();
+
+ HadoopIgfsFuture<T> fut = new HadoopIgfsFuture<>();
+
+ fut.outputBuffer(outBuf);
+ fut.outputOffset(outOff);
+ fut.outputLength(outLen);
+ fut.read(msg.command() == IgfsIpcCommand.READ_BLOCK);
+
+ HadoopIgfsFuture oldFut = reqMap.putIfAbsent(reqId, fut);
+
+ assert oldFut == null;
+
+ if (log.isDebugEnabled())
+ log.debug("Sending IGFS message [reqId=" + reqId + ", msg=" + msg + ']');
+
+ byte[] hdr = IgfsMarshaller.createHeader(reqId, msg.command());
+
+ IgniteCheckedException err = null;
+
+ try {
+ synchronized (this) {
+ marsh.marshall(msg, hdr, out);
+
+ out.flush(); // Blocking operation + sometimes system call.
+ }
+ }
+ catch (IgniteCheckedException e) {
+ err = e;
+ }
+ catch (IOException e) {
+ err = new HadoopIgfsCommunicationException(e);
+ }
+
+ if (err != null) {
+ reqMap.remove(reqId, fut);
+
+ fut.onDone(err);
+ }
+
+ return fut;
+ }
+ finally {
+ busyLock.readLock().unlock();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void sendPlain(IgfsMessage msg) throws IgniteCheckedException {
+ if (!busyLock.readLock().tryLock())
+ throw new HadoopIgfsCommunicationException("Failed to send message (client is being " +
+ "concurrently closed).");
+
+ try {
+ if (stopping)
+ throw new HadoopIgfsCommunicationException("Failed to send message (client is being concurrently closed).");
+
+ assert msg.command() == IgfsIpcCommand.WRITE_BLOCK;
+
+ IgfsStreamControlRequest req = (IgfsStreamControlRequest)msg;
+
+ byte[] hdr = IgfsMarshaller.createHeader(-1, IgfsIpcCommand.WRITE_BLOCK);
+
+ U.longToBytes(req.streamId(), hdr, 12);
+ U.intToBytes(req.length(), hdr, 20);
+
+ synchronized (this) {
+ out.write(hdr);
+ out.write(req.data(), (int)req.position(), req.length());
+
+ out.flush();
+ }
+ }
+ catch (IOException e) {
+ throw new HadoopIgfsCommunicationException(e);
+ }
+ finally {
+ busyLock.readLock().unlock();
+ }
+ }
+
+ /**
+ * Closes client but does not wait.
+ *
+ * @param err Error.
+ */
+ private void close0(@Nullable Throwable err) {
+ busyLock.writeLock().lock();
+
+ try {
+ if (stopping)
+ return;
+
+ stopping = true;
+ }
+ finally {
+ busyLock.writeLock().unlock();
+ }
+
+ if (err == null)
+ err = new IgniteCheckedException("Failed to perform request (connection was concurrently closed before response " +
+ "is received).");
+
+ // Clean up resources.
+ U.closeQuiet(out);
+
+ if (endpoint != null)
+ endpoint.close();
+
+ // Unwind futures. We can safely iterate here because no more futures will be added.
+ Iterator<HadoopIgfsFuture> it = reqMap.values().iterator();
+
+ while (it.hasNext()) {
+ HadoopIgfsFuture fut = it.next();
+
+ fut.onDone(err);
+
+ it.remove();
+ }
+
+ for (HadoopIgfsIpcIoListener lsnr : lsnrs)
+ lsnr.onClose();
+ }
+
+ /**
+ * Do not extend {@code GridThread} to minimize class dependencies.
+ */
+ private class ReaderThread extends Thread {
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public void run() {
+ // Error to fail pending futures.
+ Throwable err = null;
+
+ try {
+ InputStream in = endpoint.inputStream();
+
+ IgfsDataInputStream dis = new IgfsDataInputStream(in);
+
+ byte[] hdr = new byte[IgfsMarshaller.HEADER_SIZE];
+ byte[] msgHdr = new byte[IgfsControlResponse.RES_HEADER_SIZE];
+
+ while (!Thread.currentThread().isInterrupted()) {
+ dis.readFully(hdr);
+
+ long reqId = U.bytesToLong(hdr, 0);
+
+ // We don't wait for write responses, therefore reqId is -1.
+ if (reqId == -1) {
+ // We received a response which normally should not be sent. It must contain an error.
+ dis.readFully(msgHdr);
+
+ assert msgHdr[4] != 0;
+
+ String errMsg = dis.readUTF();
+
+ // Error code.
+ dis.readInt();
+
+ long streamId = dis.readLong();
+
+ for (HadoopIgfsIpcIoListener lsnr : lsnrs)
+ lsnr.onError(streamId, errMsg);
+ }
+ else {
+ HadoopIgfsFuture<Object> fut = reqMap.remove(reqId);
+
+ if (fut == null) {
+ String msg = "Failed to read response from server: response closure is unavailable for " +
+ "requestId (will close connection):" + reqId;
+
+ log.warn(msg);
+
+ err = new IgniteCheckedException(msg);
+
+ break;
+ }
+ else {
+ try {
+ IgfsIpcCommand cmd = IgfsIpcCommand.valueOf(U.bytesToInt(hdr, 8));
+
+ if (log.isDebugEnabled())
+ log.debug("Received IGFS response [reqId=" + reqId + ", cmd=" + cmd + ']');
+
+ Object res = null;
+
+ if (fut.read()) {
+ dis.readFully(msgHdr);
+
+ boolean hasErr = msgHdr[4] != 0;
+
+ if (hasErr) {
+ String errMsg = dis.readUTF();
+
+ // Error code.
+ Integer errCode = dis.readInt();
+
+ IgfsControlResponse.throwError(errCode, errMsg);
+ }
+
+ int blockLen = U.bytesToInt(msgHdr, 5);
+
+ int readLen = Math.min(blockLen, fut.outputLength());
+
+ if (readLen > 0) {
+ assert fut.outputBuffer() != null;
+
+ dis.readFully(fut.outputBuffer(), fut.outputOffset(), readLen);
+ }
+
+ if (readLen != blockLen) {
+ byte[] buf = new byte[blockLen - readLen];
+
+ dis.readFully(buf);
+
+ res = buf;
+ }
+ }
+ else
+ res = marsh.unmarshall(cmd, hdr, dis);
+
+ fut.onDone(res);
+ }
+ catch (IgniteCheckedException e) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to apply response closure (will fail request future): " +
+ e.getMessage());
+
+ fut.onDone(e);
+
+ err = e;
+ }
+ }
+ }
+ }
+ }
+ catch (EOFException ignored) {
+ err = new IgniteCheckedException("Failed to read response from server (connection was closed by remote peer).");
+ }
+ catch (IOException e) {
+ if (!stopping)
+ log.error("Failed to read data (connection will be closed)", e);
+
+ err = new HadoopIgfsCommunicationException(e);
+ }
+ catch (IgniteCheckedException e) {
+ if (!stopping)
+ log.error("Failed to obtain endpoint input stream (connection will be closed)", e);
+
+ err = e;
+ }
+ finally {
+ close0(err);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return getClass().getSimpleName() + " [endpointAddr=" + endpointAddr + ", activeCnt=" + activeCnt +
+ ", stopping=" + stopping + ']';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsIpcIoListener.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsIpcIoListener.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsIpcIoListener.java
new file mode 100644
index 0000000..c2dad82
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsIpcIoListener.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+/**
+ * Listens to the events of {@link HadoopIgfsIpcIo}.
+ */
+public interface HadoopIgfsIpcIoListener {
+ /**
+ * Callback invoked when the IO is being closed.
+ */
+ public void onClose();
+
+ /**
+ * Callback invoked when remote error occurs.
+ *
+ * @param streamId Stream ID.
+ * @param errMsg Error message.
+ */
+ public void onError(long streamId, String errMsg);
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsJclLogger.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsJclLogger.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsJclLogger.java
new file mode 100644
index 0000000..0c29454
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsJclLogger.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.commons.logging.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.util.tostring.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+/**
+ * JCL logger wrapper for Hadoop.
+ */
+public class HadoopIgfsJclLogger implements IgniteLogger {
+ /** JCL implementation proxy. */
+ @GridToStringInclude
+ private Log impl;
+
+ /**
+ * Constructor.
+ *
+ * @param impl JCL implementation to use.
+ */
+ HadoopIgfsJclLogger(Log impl) {
+ assert impl != null;
+
+ this.impl = impl;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteLogger getLogger(Object ctgr) {
+ return new HadoopIgfsJclLogger(LogFactory.getLog(
+ ctgr instanceof Class ? ((Class)ctgr).getName() : String.valueOf(ctgr)));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void trace(String msg) {
+ impl.trace(msg);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void debug(String msg) {
+ impl.debug(msg);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void info(String msg) {
+ impl.info(msg);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void warning(String msg) {
+ impl.warn(msg);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void warning(String msg, @Nullable Throwable e) {
+ impl.warn(msg, e);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void error(String msg) {
+ impl.error(msg);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isQuiet() {
+ return !isInfoEnabled() && !isDebugEnabled();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void error(String msg, @Nullable Throwable e) {
+ impl.error(msg, e);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isTraceEnabled() {
+ return impl.isTraceEnabled();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isDebugEnabled() {
+ return impl.isDebugEnabled();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isInfoEnabled() {
+ return impl.isInfoEnabled();
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public String fileName() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopIgfsJclLogger.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsOutProc.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsOutProc.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsOutProc.java
new file mode 100644
index 0000000..662541a
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsOutProc.java
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.commons.logging.*;
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.igfs.common.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.jdk8.backport.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.igfs.common.IgfsIpcCommand.*;
+
+/**
+ * Communication with external process (TCP or shmem).
+ */
+public class HadoopIgfsOutProc implements HadoopIgfsEx, HadoopIgfsIpcIoListener {
+ /** Expected result is boolean. */
+ private static final GridPlainClosure<GridPlainFuture<IgfsMessage>, Boolean> BOOL_RES = createClosure();
+
+ /** Expected result is boolean. */
+ private static final GridPlainClosure<GridPlainFuture<IgfsMessage>, Long> LONG_RES = createClosure();
+
+ /** Expected result is {@code IgfsFile}. */
+ private static final GridPlainClosure<GridPlainFuture<IgfsMessage>, IgfsFile> FILE_RES = createClosure();
+
+ /** Expected result is {@code IgfsHandshakeResponse} */
+ private static final GridPlainClosure<GridPlainFuture<IgfsMessage>,
+ IgfsHandshakeResponse> HANDSHAKE_RES = createClosure();
+
+ /** Expected result is {@code IgfsStatus} */
+ private static final GridPlainClosure<GridPlainFuture<IgfsMessage>, IgfsStatus> STATUS_RES =
+ createClosure();
+
+ /** Expected result is {@code IgfsFile}. */
+ private static final GridPlainClosure<GridPlainFuture<IgfsMessage>,
+ IgfsInputStreamDescriptor> STREAM_DESCRIPTOR_RES = createClosure();
+
+ /** Expected result is {@code IgfsFile}. */
+ private static final GridPlainClosure<GridPlainFuture<IgfsMessage>,
+ Collection<IgfsFile>> FILE_COL_RES = createClosure();
+
+ /** Expected result is {@code IgfsFile}. */
+ private static final GridPlainClosure<GridPlainFuture<IgfsMessage>,
+ Collection<IgfsPath>> PATH_COL_RES = createClosure();
+
+ /** Expected result is {@code IgfsPathSummary}. */
+ private static final GridPlainClosure<GridPlainFuture<IgfsMessage>, IgfsPathSummary> SUMMARY_RES =
+ createClosure();
+
+ /** Expected result is {@code IgfsFile}. */
+ private static final GridPlainClosure<GridPlainFuture<IgfsMessage>,
+ Collection<IgfsBlockLocation>> BLOCK_LOCATION_COL_RES = createClosure();
+
+ /** Grid name. */
+ private final String grid;
+
+ /** IGFS name. */
+ private final String igfs;
+
+ /** Client log. */
+ private final Log log;
+
+ /** Client IO. */
+ private final HadoopIgfsIpcIo io;
+
+ /** Event listeners. */
+ private final Map<Long, HadoopIgfsStreamEventListener> lsnrs = new ConcurrentHashMap8<>();
+
+ /**
+ * Constructor for TCP endpoint.
+ *
+ * @param host Host.
+ * @param port Port.
+ * @param grid Grid name.
+ * @param igfs IGFS name.
+ * @param log Client logger.
+ * @throws IOException If failed.
+ */
+ public HadoopIgfsOutProc(String host, int port, String grid, String igfs, Log log) throws IOException {
+ this(host, port, grid, igfs, false, log);
+ }
+
+ /**
+ * Constructor for shmem endpoint.
+ *
+ * @param port Port.
+ * @param grid Grid name.
+ * @param igfs IGFS name.
+ * @param log Client logger.
+ * @throws IOException If failed.
+ */
+ public HadoopIgfsOutProc(int port, String grid, String igfs, Log log) throws IOException {
+ this(null, port, grid, igfs, true, log);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param host Host.
+ * @param port Port.
+ * @param grid Grid name.
+ * @param igfs IGFS name.
+ * @param shmem Shared memory flag.
+ * @param log Client logger.
+ * @throws IOException If failed.
+ */
+ private HadoopIgfsOutProc(String host, int port, String grid, String igfs, boolean shmem, Log log)
+ throws IOException {
+ assert host != null && !shmem || host == null && shmem :
+ "Invalid arguments [host=" + host + ", port=" + port + ", shmem=" + shmem + ']';
+
+ String endpoint = host != null ? host + ":" + port : "shmem:" + port;
+
+ this.grid = grid;
+ this.igfs = igfs;
+ this.log = log;
+
+ io = HadoopIgfsIpcIo.get(log, endpoint);
+
+ io.addEventListener(this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsHandshakeResponse handshake(String logDir) throws IgniteCheckedException {
+ final IgfsHandshakeRequest req = new IgfsHandshakeRequest();
+
+ req.gridName(grid);
+ req.igfsName(igfs);
+ req.logDirectory(logDir);
+
+ return io.send(req).chain(HANDSHAKE_RES).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close(boolean force) {
+ assert io != null;
+
+ io.removeEventListener(this);
+
+ if (force)
+ io.forceClose();
+ else
+ io.release();
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsFile info(IgfsPath path) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(INFO);
+ msg.path(path);
+
+ return io.send(msg).chain(FILE_RES).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(UPDATE);
+ msg.path(path);
+ msg.properties(props);
+
+ return io.send(msg).chain(FILE_RES).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Boolean setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(SET_TIMES);
+ msg.path(path);
+ msg.accessTime(accessTime);
+ msg.modificationTime(modificationTime);
+
+ return io.send(msg).chain(BOOL_RES).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Boolean rename(IgfsPath src, IgfsPath dest) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(RENAME);
+ msg.path(src);
+ msg.destinationPath(dest);
+
+ return io.send(msg).chain(BOOL_RES).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Boolean delete(IgfsPath path, boolean recursive) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(DELETE);
+ msg.path(path);
+ msg.flag(recursive);
+
+ return io.send(msg).chain(BOOL_RES).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len)
+ throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(AFFINITY);
+ msg.path(path);
+ msg.start(start);
+ msg.length(len);
+
+ return io.send(msg).chain(BLOCK_LOCATION_COL_RES).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsPathSummary contentSummary(IgfsPath path) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(PATH_SUMMARY);
+ msg.path(path);
+
+ return io.send(msg).chain(SUMMARY_RES).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Boolean mkdirs(IgfsPath path, Map<String, String> props) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(MAKE_DIRECTORIES);
+ msg.path(path);
+ msg.properties(props);
+
+ return io.send(msg).chain(BOOL_RES).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(LIST_FILES);
+ msg.path(path);
+
+ return io.send(msg).chain(FILE_COL_RES).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(LIST_PATHS);
+ msg.path(path);
+
+ return io.send(msg).chain(PATH_COL_RES).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsStatus fsStatus() throws IgniteCheckedException {
+ return io.send(new IgfsStatusRequest()).chain(STATUS_RES).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopIgfsStreamDelegate open(IgfsPath path) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(OPEN_READ);
+ msg.path(path);
+ msg.flag(false);
+
+ IgfsInputStreamDescriptor rmtDesc = io.send(msg).chain(STREAM_DESCRIPTOR_RES).get();
+
+ return new HadoopIgfsStreamDelegate(this, rmtDesc.streamId(), rmtDesc.length());
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopIgfsStreamDelegate open(IgfsPath path,
+ int seqReadsBeforePrefetch) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(OPEN_READ);
+ msg.path(path);
+ msg.flag(true);
+ msg.sequentialReadsBeforePrefetch(seqReadsBeforePrefetch);
+
+ IgfsInputStreamDescriptor rmtDesc = io.send(msg).chain(STREAM_DESCRIPTOR_RES).get();
+
+ return new HadoopIgfsStreamDelegate(this, rmtDesc.streamId(), rmtDesc.length());
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopIgfsStreamDelegate create(IgfsPath path, boolean overwrite, boolean colocate,
+ int replication, long blockSize, @Nullable Map<String, String> props) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(OPEN_CREATE);
+ msg.path(path);
+ msg.flag(overwrite);
+ msg.colocate(colocate);
+ msg.properties(props);
+ msg.replication(replication);
+ msg.blockSize(blockSize);
+
+ Long streamId = io.send(msg).chain(LONG_RES).get();
+
+ return new HadoopIgfsStreamDelegate(this, streamId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopIgfsStreamDelegate append(IgfsPath path, boolean create,
+ @Nullable Map<String, String> props) throws IgniteCheckedException {
+ final IgfsPathControlRequest msg = new IgfsPathControlRequest();
+
+ msg.command(OPEN_APPEND);
+ msg.path(path);
+ msg.flag(create);
+ msg.properties(props);
+
+ Long streamId = io.send(msg).chain(LONG_RES).get();
+
+ return new HadoopIgfsStreamDelegate(this, streamId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridPlainFuture<byte[]> readData(HadoopIgfsStreamDelegate desc, long pos, int len,
+ final @Nullable byte[] outBuf, final int outOff, final int outLen) {
+ assert len > 0;
+
+ final IgfsStreamControlRequest msg = new IgfsStreamControlRequest();
+
+ msg.command(READ_BLOCK);
+ msg.streamId((long) desc.target());
+ msg.position(pos);
+ msg.length(len);
+
+ try {
+ return io.send(msg, outBuf, outOff, outLen);
+ }
+ catch (IgniteCheckedException e) {
+ return new GridPlainFutureAdapter<>(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeData(HadoopIgfsStreamDelegate desc, byte[] data, int off, int len)
+ throws IOException {
+ final IgfsStreamControlRequest msg = new IgfsStreamControlRequest();
+
+ msg.command(WRITE_BLOCK);
+ msg.streamId((long) desc.target());
+ msg.data(data);
+ msg.position(off);
+ msg.length(len);
+
+ try {
+ io.sendPlain(msg);
+ }
+ catch (IgniteCheckedException e) {
+ throw HadoopIgfsUtils.cast(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void flush(HadoopIgfsStreamDelegate delegate) throws IOException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void closeStream(HadoopIgfsStreamDelegate desc) throws IOException {
+ final IgfsStreamControlRequest msg = new IgfsStreamControlRequest();
+
+ msg.command(CLOSE);
+ msg.streamId((long)desc.target());
+
+ try {
+ io.send(msg).chain(BOOL_RES).get();
+ }
+ catch (IgniteCheckedException e) {
+ throw HadoopIgfsUtils.cast(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void addEventListener(HadoopIgfsStreamDelegate desc,
+ HadoopIgfsStreamEventListener lsnr) {
+ long streamId = desc.target();
+
+ HadoopIgfsStreamEventListener lsnr0 = lsnrs.put(streamId, lsnr);
+
+ assert lsnr0 == null || lsnr0 == lsnr;
+
+ if (log.isDebugEnabled())
+ log.debug("Added stream event listener [streamId=" + streamId + ']');
+ }
+
+ /** {@inheritDoc} */
+ @Override public void removeEventListener(HadoopIgfsStreamDelegate desc) {
+ long streamId = desc.target();
+
+ HadoopIgfsStreamEventListener lsnr0 = lsnrs.remove(streamId);
+
+ if (lsnr0 != null && log.isDebugEnabled())
+ log.debug("Removed stream event listener [streamId=" + streamId + ']');
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onClose() {
+ for (HadoopIgfsStreamEventListener lsnr : lsnrs.values()) {
+ try {
+ lsnr.onClose();
+ }
+ catch (IgniteCheckedException e) {
+ log.warn("Got exception from stream event listener (will ignore): " + lsnr, e);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onError(long streamId, String errMsg) {
+ HadoopIgfsStreamEventListener lsnr = lsnrs.get(streamId);
+
+ if (lsnr != null)
+ lsnr.onError(errMsg);
+ else
+ log.warn("Received write error response for not registered output stream (will ignore) " +
+ "[streamId= " + streamId + ']');
+ }
+
+ /**
+ * Creates conversion closure for given type.
+ *
+ * @param <T> Type of expected result.
+ * @return Conversion closure.
+ */
+ @SuppressWarnings("unchecked")
+ private static <T> GridPlainClosure<GridPlainFuture<IgfsMessage>, T> createClosure() {
+ return new GridPlainClosure<GridPlainFuture<IgfsMessage>, T>() {
+ @Override public T apply(GridPlainFuture<IgfsMessage> fut) throws IgniteCheckedException {
+ IgfsControlResponse res = (IgfsControlResponse)fut.get();
+
+ if (res.hasError())
+ res.throwError();
+
+ return (T)res.response();
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsOutputStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsOutputStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsOutputStream.java
new file mode 100644
index 0000000..902d710
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsOutputStream.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.commons.logging.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.igfs.common.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * IGFS Hadoop output stream implementation.
+ */
+public class HadoopIgfsOutputStream extends OutputStream implements HadoopIgfsStreamEventListener {
+ /** Log instance. */
+ private Log log;
+
+ /** Client logger. */
+ private IgfsLogger clientLog;
+
+ /** Log stream ID. */
+ private long logStreamId;
+
+ /** Server stream delegate. */
+ private HadoopIgfsStreamDelegate delegate;
+
+ /** Closed flag. */
+ private volatile boolean closed;
+
+ /** Flag set if stream was closed due to connection breakage. */
+ private boolean connBroken;
+
+ /** Error message. */
+ private volatile String errMsg;
+
+ /** Read time. */
+ private long writeTime;
+
+ /** User time. */
+ private long userTime;
+
+ /** Last timestamp. */
+ private long lastTs;
+
+ /** Amount of written bytes. */
+ private long total;
+
+ /**
+ * Creates light output stream.
+ *
+ * @param delegate Server stream delegate.
+ * @param log Logger to use.
+ * @param clientLog Client logger.
+ */
+ public HadoopIgfsOutputStream(HadoopIgfsStreamDelegate delegate, Log log,
+ IgfsLogger clientLog, long logStreamId) {
+ this.delegate = delegate;
+ this.log = log;
+ this.clientLog = clientLog;
+ this.logStreamId = logStreamId;
+
+ lastTs = System.nanoTime();
+
+ delegate.hadoop().addEventListener(delegate, this);
+ }
+
+ /**
+ * Read start.
+ */
+ private void writeStart() {
+ long now = System.nanoTime();
+
+ userTime += now - lastTs;
+
+ lastTs = now;
+ }
+
+ /**
+ * Read end.
+ */
+ private void writeEnd() {
+ long now = System.nanoTime();
+
+ writeTime += now - lastTs;
+
+ lastTs = now;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(@NotNull byte[] b, int off, int len) throws IOException {
+ check();
+
+ writeStart();
+
+ try {
+ delegate.hadoop().writeData(delegate, b, off, len);
+
+ total += len;
+ }
+ finally {
+ writeEnd();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(int b) throws IOException {
+ write(new byte[] {(byte)b});
+
+ total++;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void flush() throws IOException {
+ delegate.hadoop().flush(delegate);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IOException {
+ if (!closed) {
+ if (log.isDebugEnabled())
+ log.debug("Closing output stream: " + delegate);
+
+ writeStart();
+
+ delegate.hadoop().closeStream(delegate);
+
+ markClosed(false);
+
+ writeEnd();
+
+ if (clientLog.isLogEnabled())
+ clientLog.logCloseOut(logStreamId, userTime, writeTime, total);
+
+ if (log.isDebugEnabled())
+ log.debug("Closed output stream [delegate=" + delegate + ", writeTime=" + writeTime / 1000 +
+ ", userTime=" + userTime / 1000 + ']');
+ }
+ else if(connBroken)
+ throw new IOException(
+ "Failed to close stream, because connection was broken (data could have been lost).");
+ }
+
+ /**
+ * Marks stream as closed.
+ *
+ * @param connBroken {@code True} if connection with server was lost.
+ */
+ private void markClosed(boolean connBroken) {
+ // It is ok to have race here.
+ if (!closed) {
+ closed = true;
+
+ delegate.hadoop().removeEventListener(delegate);
+
+ this.connBroken = connBroken;
+ }
+ }
+
+ /**
+ * @throws IOException If check failed.
+ */
+ private void check() throws IOException {
+ String errMsg0 = errMsg;
+
+ if (errMsg0 != null)
+ throw new IOException(errMsg0);
+
+ if (closed) {
+ if (connBroken)
+ throw new IOException("Server connection was lost.");
+ else
+ throw new IOException("Stream is closed.");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onClose() throws IgniteCheckedException {
+ markClosed(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onError(String errMsg) {
+ this.errMsg = errMsg;
+ }
+}
[14/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
new file mode 100644
index 0000000..d907a6c
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
@@ -0,0 +1,1967 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.*;
+import org.apache.ignite.*;
+import org.apache.ignite.cache.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.hadoop.fs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
+import org.apache.ignite.spi.communication.*;
+import org.apache.ignite.spi.communication.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+import org.apache.ignite.testframework.*;
+import org.jdk8.backport.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.events.EventType.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * Hadoop 2.x compliant file system.
+ */
+public abstract class HadoopIgfs20FileSystemAbstractSelfTest extends IgfsCommonAbstractTest {
+ /** Group size. */
+ public static final int GRP_SIZE = 128;
+
+ /** Thread count for multithreaded tests. */
+ private static final int THREAD_CNT = 8;
+
+ /** IP finder. */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** Barrier for multithreaded tests. */
+ private static CyclicBarrier barrier;
+
+ /** File system. */
+ private static AbstractFileSystem fs;
+
+ /** Default IGFS mode. */
+ protected IgfsMode mode;
+
+ /** Primary file system URI. */
+ protected URI primaryFsUri;
+
+ /** Primary file system configuration. */
+ protected Configuration primaryFsCfg;
+
+ /**
+ * Constructor.
+ *
+ * @param mode Default IGFS mode.
+ */
+ protected HadoopIgfs20FileSystemAbstractSelfTest(IgfsMode mode) {
+ this.mode = mode;
+ }
+
+ /**
+ * Gets primary file system URI path.
+ *
+ * @return Primary file system URI path.
+ */
+ protected abstract String primaryFileSystemUriPath();
+
+ /**
+ * Gets primary file system config path.
+ *
+ * @return Primary file system config path.
+ */
+ protected abstract String primaryFileSystemConfigPath();
+
+ /**
+ * Get primary IPC endpoint configuration.
+ *
+ * @param gridName Grid name.
+ * @return IPC primary endpoint configuration.
+ */
+ protected abstract Map<String, String> primaryIpcEndpointConfiguration(String gridName);
+
+ /**
+ * Gets secondary file system URI path.
+ *
+ * @return Secondary file system URI path.
+ */
+ protected abstract String secondaryFileSystemUriPath();
+
+ /**
+ * Gets secondary file system config path.
+ *
+ * @return Secondary file system config path.
+ */
+ protected abstract String secondaryFileSystemConfigPath();
+
+ /**
+ * Get secondary IPC endpoint configuration.
+ *
+ * @return Secondary IPC endpoint configuration.
+ */
+ protected abstract Map<String, String> secondaryIpcEndpointConfiguration();
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ startNodes();
+ }
+
+ /**
+ * Starts the nodes for this test.
+ *
+ * @throws Exception If failed.
+ */
+ private void startNodes() throws Exception {
+ if (mode != PRIMARY) {
+ // Start secondary IGFS.
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName("igfs_secondary");
+ igfsCfg.setIpcEndpointConfiguration(secondaryIpcEndpointConfiguration());
+ igfsCfg.setManagementPort(-1);
+ igfsCfg.setBlockSize(512 * 1024);
+ igfsCfg.setPrefetchBlocks(1);
+
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("grid_secondary");
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
+ cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
+ cfg.setLocalHost(U.getLocalHost().getHostAddress());
+ cfg.setCommunicationSpi(communicationSpi());
+
+ G.start(cfg);
+ }
+
+ startGrids(4);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getTestGridName() {
+ return "grid";
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(cacheConfiguration(gridName));
+ cfg.setFileSystemConfiguration(igfsConfiguration(gridName));
+ cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
+ cfg.setLocalHost("127.0.0.1");
+ cfg.setCommunicationSpi(communicationSpi());
+
+ return cfg;
+ }
+
+ /**
+ * Gets cache configuration.
+ *
+ * @param gridName Grid name.
+ * @return Cache configuration.
+ */
+ protected CacheConfiguration[] cacheConfiguration(String gridName) {
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ return new CacheConfiguration[] {metaCacheCfg, cacheCfg};
+ }
+
+ /**
+ * Gets IGFS configuration.
+ *
+ * @param gridName Grid name.
+ * @return IGFS configuration.
+ */
+ protected FileSystemConfiguration igfsConfiguration(String gridName) throws IgniteCheckedException {
+ FileSystemConfiguration cfg = new FileSystemConfiguration();
+
+ cfg.setDataCacheName("partitioned");
+ cfg.setMetaCacheName("replicated");
+ cfg.setName("igfs");
+ cfg.setPrefetchBlocks(1);
+ cfg.setMaxSpaceSize(64 * 1024 * 1024);
+ cfg.setDefaultMode(mode);
+
+ if (mode != PRIMARY)
+ cfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(secondaryFileSystemUriPath(),
+ secondaryFileSystemConfigPath()));
+
+ cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(gridName));
+ cfg.setManagementPort(-1);
+
+ cfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups.
+
+ return cfg;
+ }
+
+ /** @return Communication SPI. */
+ private CommunicationSpi communicationSpi() {
+ TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
+
+ commSpi.setSharedMemoryPort(-1);
+
+ return commSpi;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ G.stopAll(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ primaryFsUri = new URI(primaryFileSystemUriPath());
+
+ primaryFsCfg = new Configuration();
+
+ primaryFsCfg.addResource(U.resolveIgniteUrl(primaryFileSystemConfigPath()));
+
+ fs = AbstractFileSystem.get(primaryFsUri, primaryFsCfg);
+
+ barrier = new CyclicBarrier(THREAD_CNT);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ try {
+ fs.delete(new Path("/"), true);
+ }
+ catch (Exception ignore) {
+ // No-op.
+ }
+
+ U.closeQuiet((Closeable)fs);
+ }
+
+ /** @throws Exception If failed. */
+ public void testStatus() throws Exception {
+
+ try (FSDataOutputStream file = fs.create(new Path("/file1"), EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()))) {
+ file.write(new byte[1024 * 1024]);
+ }
+
+ FsStatus status = fs.getFsStatus();
+
+ assertEquals(4, grid(0).cluster().nodes().size());
+
+ long used = 0, max = 0;
+
+ for (int i = 0; i < 4; i++) {
+ IgniteFileSystem igfs = grid(i).fileSystem("igfs");
+
+ IgfsMetrics metrics = igfs.metrics();
+
+ used += metrics.localSpaceSize();
+ max += metrics.maxSpaceSize();
+ }
+
+ assertEquals(used, status.getUsed());
+ assertEquals(max, status.getCapacity());
+ }
+
+ /** @throws Exception If failed. */
+ public void testTimes() throws Exception {
+ Path file = new Path("/file1");
+
+ long now = System.currentTimeMillis();
+
+ try (FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()))) {
+ os.write(new byte[1024 * 1024]);
+ }
+
+ FileStatus status = fs.getFileStatus(file);
+
+ assertTrue(status.getAccessTime() >= now);
+ assertTrue(status.getModificationTime() >= now);
+
+ long accessTime = now - 10 * 60 * 1000;
+ long modificationTime = now - 5 * 60 * 1000;
+
+ fs.setTimes(file, modificationTime, accessTime);
+
+ status = fs.getFileStatus(file);
+ assertEquals(accessTime, status.getAccessTime());
+ assertEquals(modificationTime, status.getModificationTime());
+
+ // Check listing is updated as well.
+ FileStatus[] files = fs.listStatus(new Path("/"));
+
+ assertEquals(1, files.length);
+
+ assertEquals(file.getName(), files[0].getPath().getName());
+ assertEquals(accessTime, files[0].getAccessTime());
+ assertEquals(modificationTime, files[0].getModificationTime());
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.setTimes(new Path("/unknownFile"), 0, 0);
+
+ return null;
+ }
+ }, FileNotFoundException.class, null);
+ }
+
+ /** @throws Exception If failed. */
+ public void testCreateCheckParameters() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.create(null, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+ }
+ }, NullPointerException.class, null);
+ }
+
+ /** @throws Exception If failed. */
+ public void testCreateBase() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
+ Path file = new Path(dir, "someFile");
+
+ assertPathDoesNotExist(fs, file);
+
+ FsPermission fsPerm = new FsPermission((short)644);
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(fsPerm));
+
+ // Try to write something in file.
+ os.write("abc".getBytes());
+
+ os.close();
+
+ // Check file status.
+ FileStatus fileStatus = fs.getFileStatus(file);
+
+ assertFalse(fileStatus.isDirectory());
+ assertEquals(file, fileStatus.getPath());
+ assertEquals(fsPerm, fileStatus.getPermission());
+ }
+
+ /** @throws Exception If failed. */
+ public void testCreateCheckOverwrite() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
+ final Path file = new Path(dir, "someFile");
+
+ FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ out.close();
+
+ // Check intermediate directory permissions.
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir).getPermission());
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir.getParent()).getPermission());
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir.getParent().getParent()).getPermission());
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+ }
+ }, PathExistsException.class, null);
+
+ // Overwrite should be successful.
+ FSDataOutputStream out1 = fs.create(file, EnumSet.of(CreateFlag.OVERWRITE),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ out1.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteIfNoSuchPath() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
+
+ assertPathDoesNotExist(fs, dir);
+
+ assertFalse(fs.delete(dir, true));
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteSuccessfulIfPathIsOpenedToRead() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "myFile");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ final int cnt = 5 * FileSystemConfiguration.DFLT_BLOCK_SIZE; // Write 5 blocks.
+
+ for (int i = 0; i < cnt; i++)
+ os.writeInt(i);
+
+ os.close();
+
+ final FSDataInputStream is = fs.open(file, -1);
+
+ for (int i = 0; i < cnt / 2; i++)
+ assertEquals(i, is.readInt());
+
+ assert fs.delete(file, false);
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.getFileStatus(file);
+
+ return null;
+ }
+ }, FileNotFoundException.class, null);
+
+ is.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteIfFilePathExists() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "myFile");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ assertTrue(fs.delete(file, false));
+
+ assertPathDoesNotExist(fs, file);
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteIfDirectoryPathExists() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
+
+ FSDataOutputStream os = fs.create(dir, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ assertTrue(fs.delete(dir, false));
+
+ assertPathDoesNotExist(fs, dir);
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteFailsIfNonRecursive() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
+
+ FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ final Path someDir2 = new Path(fsHome, "/someDir1/someDir2");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.delete(someDir2, false);
+
+ return null;
+ }
+ }, PathIsNotEmptyDirectoryException.class, null);
+
+ assertPathExists(fs, someDir2);
+ assertPathExists(fs, someDir3);
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteRecursively() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
+
+ FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ Path someDir2 = new Path(fsHome, "/someDir1/someDir2");
+
+ assertTrue(fs.delete(someDir2, true));
+
+ assertPathDoesNotExist(fs, someDir2);
+ assertPathDoesNotExist(fs, someDir3);
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteRecursivelyFromRoot() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
+
+ FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ Path root = new Path(fsHome, "/");
+
+ assertTrue(fs.delete(root, true));
+
+ assertPathDoesNotExist(fs, someDir3);
+ assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));
+ assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1"));
+ assertPathExists(fs, root);
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetPermissionCheckDefaultPermission() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ fs.setPermission(file, null);
+
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(file.getParent()).getPermission());
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetPermissionCheckNonRecursiveness() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ Path tmpDir = new Path(fsHome, "/tmp");
+
+ FsPermission perm = new FsPermission((short)123);
+
+ fs.setPermission(tmpDir, perm);
+
+ assertEquals(perm, fs.getFileStatus(tmpDir).getPermission());
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
+ }
+
+ /** @throws Exception If failed. */
+ @SuppressWarnings("OctalInteger")
+ public void testSetPermission() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ for (short i = 0; i <= 0777; i += 7) {
+ FsPermission perm = new FsPermission(i);
+
+ fs.setPermission(file, perm);
+
+ assertEquals(perm, fs.getFileStatus(file).getPermission());
+ }
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetPermissionIfOutputStreamIsNotClosed() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "myFile");
+
+ FsPermission perm = new FsPermission((short)123);
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ fs.setPermission(file, perm);
+
+ os.close();
+
+ assertEquals(perm, fs.getFileStatus(file).getPermission());
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetOwnerCheckParametersPathIsNull() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.setOwner(null, "aUser", "aGroup");
+
+ return null;
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: p");
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetOwnerCheckParametersUserIsNull() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.setOwner(file, null, "aGroup");
+
+ return null;
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: username");
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetOwnerCheckParametersGroupIsNull() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ fs.setOwner(file, "aUser", null);
+
+ return null;
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: grpName");
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetOwner() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ fs.setOwner(file, "aUser", "aGroup");
+
+ assertEquals("aUser", fs.getFileStatus(file).getOwner());
+ assertEquals("aGroup", fs.getFileStatus(file).getGroup());
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetOwnerIfOutputStreamIsNotClosed() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "myFile");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ fs.setOwner(file, "aUser", "aGroup");
+
+ os.close();
+
+ assertEquals("aUser", fs.getFileStatus(file).getOwner());
+ assertEquals("aGroup", fs.getFileStatus(file).getGroup());
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetOwnerCheckNonRecursiveness() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ Path tmpDir = new Path(fsHome, "/tmp");
+
+ fs.setOwner(file, "fUser", "fGroup");
+ fs.setOwner(tmpDir, "dUser", "dGroup");
+
+ assertEquals("dUser", fs.getFileStatus(tmpDir).getOwner());
+ assertEquals("dGroup", fs.getFileStatus(tmpDir).getGroup());
+
+ assertEquals("fUser", fs.getFileStatus(file).getOwner());
+ assertEquals("fGroup", fs.getFileStatus(file).getGroup());
+ }
+
+ /** @throws Exception If failed. */
+ public void testOpenCheckParametersPathIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.open(null, 1024);
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: f");
+ }
+
+ /** @throws Exception If failed. */
+ public void testOpenNoSuchPath() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "someFile");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.open(file, 1024);
+ }
+ }, FileNotFoundException.class, null);
+ }
+
+ /** @throws Exception If failed. */
+ public void testOpenIfPathIsAlreadyOpened() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "someFile");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ FSDataInputStream is1 = fs.open(file);
+ FSDataInputStream is2 = fs.open(file);
+
+ is1.close();
+ is2.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testOpen() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "someFile");
+
+ int cnt = 2 * 1024;
+
+ FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ for (long i = 0; i < cnt; i++)
+ out.writeLong(i);
+
+ out.close();
+
+ FSDataInputStream in = fs.open(file, 1024);
+
+ for (long i = 0; i < cnt; i++)
+ assertEquals(i, in.readLong());
+
+ in.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testAppendIfPathPointsToDirectory() throws Exception {
+ final Path fsHome = new Path(primaryFsUri);
+ final Path dir = new Path(fsHome, "/tmp");
+ Path file = new Path(dir, "my");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.create(new Path(fsHome, dir), EnumSet.of(CreateFlag.APPEND),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+ }
+ }, IOException.class, null);
+ }
+
+ /** @throws Exception If failed. */
+ public void testAppendIfFileIsAlreadyBeingOpenedToWrite() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "someFile");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ FSDataOutputStream appendOs = fs.create(file, EnumSet.of(CreateFlag.APPEND),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ return fs.create(file, EnumSet.of(CreateFlag.APPEND),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+ }
+ }, IOException.class, null);
+
+ appendOs.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testAppend() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "someFile");
+
+ int cnt = 1024;
+
+ FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ for (int i = 0; i < cnt; i++)
+ out.writeLong(i);
+
+ out.close();
+
+ out = fs.create(file, EnumSet.of(CreateFlag.APPEND),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ for (int i = cnt; i < cnt * 2; i++)
+ out.writeLong(i);
+
+ out.close();
+
+ FSDataInputStream in = fs.open(file, 1024);
+
+ for (int i = 0; i < cnt * 2; i++)
+ assertEquals(i, in.readLong());
+
+ in.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameCheckParametersSrcPathIsNull() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "someFile");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.rename(null, file);
+
+ return null;
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: f");
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameCheckParametersDstPathIsNull() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "someFile");
+
+ fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault())).close();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ fs.rename(file, null);
+
+ return null;
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: f");
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameIfSrcPathDoesNotExist() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path srcFile = new Path(fsHome, "srcFile");
+ final Path dstFile = new Path(fsHome, "dstFile");
+
+ assertPathDoesNotExist(fs, srcFile);
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.rename(srcFile, dstFile);
+
+ return null;
+ }
+ }, FileNotFoundException.class, null);
+
+ assertPathDoesNotExist(fs, dstFile);
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameIfSrcPathIsAlreadyBeingOpenedToWrite() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path srcFile = new Path(fsHome, "srcFile");
+ Path dstFile = new Path(fsHome, "dstFile");
+
+ FSDataOutputStream os = fs.create(srcFile, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ os = fs.create(srcFile, EnumSet.of(CreateFlag.APPEND),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ fs.rename(srcFile, dstFile);
+
+ assertPathExists(fs, dstFile);
+
+ String testStr = "Test";
+
+ try {
+ os.writeBytes(testStr);
+ }
+ finally {
+ os.close();
+ }
+
+ try (FSDataInputStream is = fs.open(dstFile)) {
+ byte[] buf = new byte[testStr.getBytes().length];
+
+ is.readFully(buf);
+
+ assertEquals(testStr, new String(buf));
+ }
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameFileIfDstPathExists() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path srcFile = new Path(fsHome, "srcFile");
+ final Path dstFile = new Path(fsHome, "dstFile");
+
+ FSDataOutputStream os = fs.create(srcFile, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ os = fs.create(dstFile, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.rename(srcFile, dstFile);
+
+ return null;
+ }
+ }, FileAlreadyExistsException.class, null);
+
+ assertPathExists(fs, srcFile);
+ assertPathExists(fs, dstFile);
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameFile() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path srcFile = new Path(fsHome, "/tmp/srcFile");
+ Path dstFile = new Path(fsHome, "/tmp/dstFile");
+
+ FSDataOutputStream os = fs.create(srcFile, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ fs.rename(srcFile, dstFile);
+
+ assertPathDoesNotExist(fs, srcFile);
+ assertPathExists(fs, dstFile);
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameIfSrcPathIsAlreadyBeingOpenedToRead() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path srcFile = new Path(fsHome, "srcFile");
+ Path dstFile = new Path(fsHome, "dstFile");
+
+ FSDataOutputStream os = fs.create(srcFile, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ int cnt = 1024;
+
+ for (int i = 0; i < cnt; i++)
+ os.writeInt(i);
+
+ os.close();
+
+ FSDataInputStream is = fs.open(srcFile);
+
+ for (int i = 0; i < cnt; i++) {
+ if (i == 100)
+ // Rename file during the read process.
+ fs.rename(srcFile, dstFile);
+
+ assertEquals(i, is.readInt());
+ }
+
+ assertPathDoesNotExist(fs, srcFile);
+ assertPathExists(fs, dstFile);
+
+ os.close();
+ is.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void _testRenameDirectoryIfDstPathExists() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path srcDir = new Path(fsHome, "/tmp/");
+ Path dstDir = new Path(fsHome, "/tmpNew/");
+
+ FSDataOutputStream os = fs.create(new Path(srcDir, "file1"), EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ os = fs.create(new Path(dstDir, "file2"), EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ fs.rename(srcDir, dstDir);
+
+ assertPathExists(fs, dstDir);
+ assertPathExists(fs, new Path(fsHome, "/tmpNew/tmp"));
+ assertPathExists(fs, new Path(fsHome, "/tmpNew/tmp/file1"));
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameDirectory() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path dir = new Path(fsHome, "/tmp/");
+ Path newDir = new Path(fsHome, "/tmpNew/");
+
+ FSDataOutputStream os = fs.create(new Path(dir, "myFile"), EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.close();
+
+ fs.rename(dir, newDir);
+
+ assertPathDoesNotExist(fs, dir);
+ assertPathExists(fs, newDir);
+ }
+
+ /** @throws Exception If failed. */
+ public void testListStatusIfPathIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.listStatus(null);
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: f");
+ }
+
+ /** @throws Exception If failed. */
+ public void testListStatusIfPathDoesNotExist() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.listStatus(new Path("/someDir"));
+ }
+ }, FileNotFoundException.class, null);
+ }
+
+ /**
+ * Test directory listing.
+ *
+ * @throws Exception If failed.
+ */
+ public void testListStatus() throws Exception {
+ Path igfsHome = new Path(primaryFsUri);
+
+ // Test listing of an empty directory.
+ Path dir = new Path(igfsHome, "dir");
+
+ fs.mkdir(dir, FsPermission.getDefault(), true);
+
+ FileStatus[] list = fs.listStatus(dir);
+
+ assert list.length == 0;
+
+ // Test listing of a not empty directory.
+ Path subDir = new Path(dir, "subDir");
+
+ fs.mkdir(subDir, FsPermission.getDefault(), true);
+
+ Path file = new Path(dir, "file");
+
+ FSDataOutputStream fos = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ fos.close();
+
+ list = fs.listStatus(dir);
+
+ assert list.length == 2;
+
+ String listRes1 = list[0].getPath().getName();
+ String listRes2 = list[1].getPath().getName();
+
+ assert "subDir".equals(listRes1) && "file".equals(listRes2) || "subDir".equals(listRes2) &&
+ "file".equals(listRes1);
+
+ // Test listing of a file.
+ list = fs.listStatus(file);
+
+ assert list.length == 1;
+
+ assert "file".equals(list[0].getPath().getName());
+ }
+
+ /** @throws Exception If failed. */
+ public void testMkdirsIfPathIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.mkdir(null, FsPermission.getDefault(), true);
+
+ return null;
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: f");
+ }
+
+ /** @throws Exception If failed. */
+ public void testMkdirsIfPermissionIsNull() throws Exception {
+ Path dir = new Path("/tmp");
+
+ fs.mkdir(dir, null, true);
+
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir).getPermission());
+ }
+
+ /** @throws Exception If failed. */
+ @SuppressWarnings("OctalInteger")
+ public void testMkdirs() throws Exception {
+ Path fsHome = new Path(primaryFileSystemUriPath());
+ Path dir = new Path(fsHome, "/tmp/staging");
+ Path nestedDir = new Path(dir, "nested");
+
+ FsPermission dirPerm = FsPermission.createImmutable((short)0700);
+ FsPermission nestedDirPerm = FsPermission.createImmutable((short)111);
+
+ fs.mkdir(dir, dirPerm, true);
+ fs.mkdir(nestedDir, nestedDirPerm, true);
+
+ assertEquals(dirPerm, fs.getFileStatus(dir).getPermission());
+ assertEquals(nestedDirPerm, fs.getFileStatus(nestedDir).getPermission());
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetFileStatusIfPathIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.getFileStatus(null);
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: f");
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetFileStatusIfPathDoesNotExist() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.getFileStatus(new Path("someDir"));
+ }
+ }, FileNotFoundException.class, "File not found: someDir");
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetFileBlockLocationsIfFileStatusIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ // Argument is checked by Hadoop.
+ return fs.getFileBlockLocations(null, 1, 2);
+ }
+ }, NullPointerException.class, null);
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetFileBlockLocationsIfFileStatusReferenceNotExistingPath() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.getFileBlockLocations(new Path("/someFile"), 1, 2);
+ }
+ }, FileNotFoundException.class, null);
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetFileBlockLocations() throws Exception {
+ Path igfsHome = new Path(primaryFsUri);
+
+ Path file = new Path(igfsHome, "someFile");
+
+ try (OutputStream out = new BufferedOutputStream(fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault())))) {
+ byte[] data = new byte[128 * 1024];
+
+ for (int i = 0; i < 100; i++)
+ out.write(data);
+
+ out.flush();
+ }
+
+ try (FSDataInputStream in = fs.open(file, 1024 * 1024)) {
+ byte[] data = new byte[128 * 1024];
+
+ int read;
+
+ do {
+ read = in.read(data);
+ }
+ while (read > 0);
+ }
+
+ FileStatus status = fs.getFileStatus(file);
+
+ int grpLen = 128 * 512 * 1024;
+
+ int grpCnt = (int)((status.getLen() + grpLen - 1) / grpLen);
+
+ BlockLocation[] locations = fs.getFileBlockLocations(file, 0, status.getLen());
+
+ assertEquals(grpCnt, locations.length);
+ }
+
+ /** @throws Exception If failed. */
+ public void testZeroReplicationFactor() throws Exception {
+ // This test doesn't make sense for any mode except of PRIMARY.
+ if (mode == PRIMARY) {
+ Path igfsHome = new Path(primaryFsUri);
+
+ Path file = new Path(igfsHome, "someFile");
+
+ try (FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()), Options.CreateOpts.repFac((short)1))) {
+ out.write(new byte[1024 * 1024]);
+ }
+
+ IgniteFileSystem igfs = grid(0).fileSystem("igfs");
+
+ IgfsPath filePath = new IgfsPath("/someFile");
+
+ IgfsFile fileInfo = igfs.info(filePath);
+
+ Collection<IgfsBlockLocation> locations = igfs.affinity(filePath, 0, fileInfo.length());
+
+ assertEquals(1, locations.size());
+
+ IgfsBlockLocation location = F.first(locations);
+
+ assertEquals(1, location.nodeIds().size());
+ }
+ }
+
+ /**
+ * Ensure that when running in multithreaded mode only one create() operation succeed.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMultithreadedCreate() throws Exception {
+ Path dir = new Path(new Path(primaryFsUri), "/dir");
+
+ fs.mkdir(dir, FsPermission.getDefault(), true);
+
+ final Path file = new Path(dir, "file");
+
+ fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault())).close();
+
+ final AtomicInteger cnt = new AtomicInteger();
+
+ final Collection<Integer> errs = new GridConcurrentHashSet<>(THREAD_CNT, 1.0f, THREAD_CNT);
+
+ multithreaded(new Runnable() {
+ @Override public void run() {
+ int idx = cnt.getAndIncrement();
+
+ byte[] data = new byte[256];
+
+ Arrays.fill(data, (byte)idx);
+
+ FSDataOutputStream os = null;
+
+ try {
+ os = fs.create(file, EnumSet.of(CreateFlag.OVERWRITE),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.write(data);
+ }
+ catch (IOException ignore) {
+ errs.add(idx);
+ }
+ finally {
+ U.awaitQuiet(barrier);
+
+ U.closeQuiet(os);
+ }
+ }
+ }, THREAD_CNT);
+
+ // Only one thread could obtain write lock on the file.
+ assert errs.size() == THREAD_CNT - 1 : "Invalid errors count [expected=" + (THREAD_CNT - 1) + ", actual=" +
+ errs.size() + ']';
+
+ int idx = -1;
+
+ for (int i = 0; i < THREAD_CNT; i++) {
+ if (!errs.remove(i)) {
+ idx = i;
+
+ break;
+ }
+ }
+
+ byte[] expData = new byte[256];
+
+ Arrays.fill(expData, (byte)idx);
+
+ FSDataInputStream is = fs.open(file);
+
+ byte[] data = new byte[256];
+
+ is.read(data);
+
+ is.close();
+
+ assert Arrays.equals(expData, data);
+ }
+
+ /**
+ * Ensure that when running in multithreaded mode only one append() operation succeed.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMultithreadedAppend() throws Exception {
+ Path dir = new Path(new Path(primaryFsUri), "/dir");
+
+ fs.mkdir(dir, FsPermission.getDefault(), true);
+
+ final Path file = new Path(dir, "file");
+
+ fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault())).close();
+
+ final AtomicInteger cnt = new AtomicInteger();
+
+ final Collection<Integer> errs = new GridConcurrentHashSet<>(THREAD_CNT, 1.0f, THREAD_CNT);
+
+ multithreaded(new Runnable() {
+ @Override public void run() {
+ int idx = cnt.getAndIncrement();
+
+ byte[] data = new byte[256];
+
+ Arrays.fill(data, (byte)idx);
+
+ U.awaitQuiet(barrier);
+
+ FSDataOutputStream os = null;
+
+ try {
+ os = fs.create(file, EnumSet.of(CreateFlag.APPEND),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ os.write(data);
+ }
+ catch (IOException ignore) {
+ errs.add(idx);
+ }
+ finally {
+ U.awaitQuiet(barrier);
+
+ U.closeQuiet(os);
+ }
+ }
+ }, THREAD_CNT);
+
+ // Only one thread could obtain write lock on the file.
+ assert errs.size() == THREAD_CNT - 1;
+
+ int idx = -1;
+
+ for (int i = 0; i < THREAD_CNT; i++) {
+ if (!errs.remove(i)) {
+ idx = i;
+
+ break;
+ }
+ }
+
+ byte[] expData = new byte[256];
+
+ Arrays.fill(expData, (byte)idx);
+
+ FSDataInputStream is = fs.open(file);
+
+ byte[] data = new byte[256];
+
+ is.read(data);
+
+ is.close();
+
+ assert Arrays.equals(expData, data);
+ }
+
+ /**
+ * Test concurrent reads within the file.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMultithreadedOpen() throws Exception {
+ final byte[] dataChunk = new byte[256];
+
+ for (int i = 0; i < dataChunk.length; i++)
+ dataChunk[i] = (byte)i;
+
+ Path dir = new Path(new Path(primaryFsUri), "/dir");
+
+ fs.mkdir(dir, FsPermission.getDefault(), true);
+
+ final Path file = new Path(dir, "file");
+
+ FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault()));
+
+ // Write 256 * 2048 = 512Kb of data.
+ for (int i = 0; i < 2048; i++)
+ os.write(dataChunk);
+
+ os.close();
+
+ final AtomicBoolean err = new AtomicBoolean();
+
+ multithreaded(new Runnable() {
+ @Override
+ public void run() {
+ FSDataInputStream is = null;
+
+ try {
+ int pos = ThreadLocalRandom8.current().nextInt(2048);
+
+ try {
+ is = fs.open(file);
+ }
+ finally {
+ U.awaitQuiet(barrier);
+ }
+
+ is.seek(256 * pos);
+
+ byte[] buf = new byte[256];
+
+ for (int i = pos; i < 2048; i++) {
+ // First perform normal read.
+ int read = is.read(buf);
+
+ assert read == 256;
+
+ Arrays.equals(dataChunk, buf);
+ }
+
+ int res = is.read(buf);
+
+ assert res == -1;
+ }
+ catch (IOException ignore) {
+ err.set(true);
+ }
+ finally {
+ U.closeQuiet(is);
+ }
+ }
+ }, THREAD_CNT);
+
+ assert !err.get();
+ }
+
+ /**
+ * Test concurrent creation of multiple directories.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMultithreadedMkdirs() throws Exception {
+ final Path dir = new Path(new Path("igfs:///"), "/dir");
+
+ fs.mkdir(dir, FsPermission.getDefault(), true);
+
+ final int depth = 3;
+ final int entryCnt = 5;
+
+ final AtomicBoolean err = new AtomicBoolean();
+
+ multithreaded(new Runnable() {
+ @Override public void run() {
+ Deque<IgniteBiTuple<Integer, Path>> queue = new ArrayDeque<>();
+
+ queue.add(F.t(0, dir));
+
+ U.awaitQuiet(barrier);
+
+ while (!queue.isEmpty()) {
+ IgniteBiTuple<Integer, Path> t = queue.pollFirst();
+
+ int curDepth = t.getKey();
+ Path curPath = t.getValue();
+
+ if (curDepth <= depth) {
+ int newDepth = curDepth + 1;
+
+ // Create directories.
+ for (int i = 0; i < entryCnt; i++) {
+ Path subDir = new Path(curPath, "dir-" + newDepth + "-" + i);
+
+ try {
+ fs.mkdir(subDir, FsPermission.getDefault(), true);
+ }
+ catch (IOException ignore) {
+ err.set(true);
+ }
+
+ queue.addLast(F.t(newDepth, subDir));
+ }
+ }
+ }
+ }
+ }, THREAD_CNT);
+
+ // Ensure there were no errors.
+ assert !err.get();
+
+ // Ensure correct folders structure.
+ Deque<IgniteBiTuple<Integer, Path>> queue = new ArrayDeque<>();
+
+ queue.add(F.t(0, dir));
+
+ while (!queue.isEmpty()) {
+ IgniteBiTuple<Integer, Path> t = queue.pollFirst();
+
+ int curDepth = t.getKey();
+ Path curPath = t.getValue();
+
+ if (curDepth <= depth) {
+ int newDepth = curDepth + 1;
+
+ // Create directories.
+ for (int i = 0; i < entryCnt; i++) {
+ Path subDir = new Path(curPath, "dir-" + newDepth + "-" + i);
+
+ assertNotNull(fs.getFileStatus(subDir));
+
+ queue.add(F.t(newDepth, subDir));
+ }
+ }
+ }
+ }
+
+ /**
+ * Test concurrent deletion of the same directory with advanced structure.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("TooBroadScope")
+ public void testMultithreadedDelete() throws Exception {
+ final Path dir = new Path(new Path(primaryFsUri), "/dir");
+
+ fs.mkdir(dir, FsPermission.getDefault(), true);
+
+ int depth = 3;
+ int entryCnt = 5;
+
+ Deque<IgniteBiTuple<Integer, Path>> queue = new ArrayDeque<>();
+
+ queue.add(F.t(0, dir));
+
+ while (!queue.isEmpty()) {
+ IgniteBiTuple<Integer, Path> t = queue.pollFirst();
+
+ int curDepth = t.getKey();
+ Path curPath = t.getValue();
+
+ if (curDepth < depth) {
+ int newDepth = curDepth + 1;
+
+ // Create directories.
+ for (int i = 0; i < entryCnt; i++) {
+ Path subDir = new Path(curPath, "dir-" + newDepth + "-" + i);
+
+ fs.mkdir(subDir, FsPermission.getDefault(), true);
+
+ queue.addLast(F.t(newDepth, subDir));
+ }
+ }
+ else {
+ // Create files.
+ for (int i = 0; i < entryCnt; i++) {
+ Path file = new Path(curPath, "file " + i);
+
+ fs.create(file, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault())).close();
+ }
+ }
+ }
+
+ final AtomicBoolean err = new AtomicBoolean();
+
+ multithreaded(new Runnable() {
+ @Override public void run() {
+ try {
+ U.awaitQuiet(barrier);
+
+ fs.delete(dir, true);
+ }
+ catch (FileNotFoundException ignore) {
+ // No-op.
+ }
+ catch (IOException ignore) {
+ err.set(true);
+ }
+ }
+ }, THREAD_CNT);
+
+ // Ensure there were no errors.
+ assert !err.get();
+
+ // Ensure the directory was actually deleted.
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.getFileStatus(dir);
+
+ return null;
+ }
+ }, FileNotFoundException.class, null);
+ }
+
+ /** @throws Exception If failed. */
+ public void testConsistency() throws Exception {
+ // Default buffers values
+ checkConsistency(-1, 1, -1, -1, 1, -1);
+ checkConsistency(-1, 10, -1, -1, 10, -1);
+ checkConsistency(-1, 100, -1, -1, 100, -1);
+ checkConsistency(-1, 1000, -1, -1, 1000, -1);
+ checkConsistency(-1, 10000, -1, -1, 10000, -1);
+ checkConsistency(-1, 100000, -1, -1, 100000, -1);
+
+ checkConsistency(65 * 1024 + 13, 100000, -1, -1, 100000, -1);
+
+ checkConsistency(-1, 100000, 2 * 4 * 1024 + 17, -1, 100000, -1);
+
+ checkConsistency(-1, 100000, -1, 65 * 1024 + 13, 100000, -1);
+
+ checkConsistency(-1, 100000, -1, -1, 100000, 2 * 4 * 1024 + 17);
+
+ checkConsistency(65 * 1024 + 13, 100000, 2 * 4 * 1024 + 13, 65 * 1024 + 149, 100000, 2 * 4 * 1024 + 157);
+ }
+
+ /**
+ * Verifies that client reconnects after connection to the server has been lost.
+ *
+ * @throws Exception If error occurs.
+ */
+ public void testClientReconnect() throws Exception {
+ final Path igfsHome = new Path(primaryFsUri);
+
+ final Path filePath = new Path(igfsHome, "someFile");
+
+ final FSDataOutputStream s = fs.create(filePath, EnumSet.noneOf(CreateFlag.class),
+ Options.CreateOpts.perms(FsPermission.getDefault())); // Open stream before stopping IGFS.
+
+ try {
+ G.stopAll(true); // Stop the server.
+
+ startNodes(); // Start server again.
+
+ // Check that client is again operational.
+ fs.mkdir(new Path("igfs:///dir1/dir2"), FsPermission.getDefault(), true);
+
+ // However, the streams, opened before disconnect, should not be valid.
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Nullable @Override public Object call() throws Exception {
+ s.write("test".getBytes());
+
+ s.flush();
+
+ return null;
+ }
+ }, IOException.class, null);
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.getFileStatus(filePath);
+
+ return null;
+ }
+ }, FileNotFoundException.class, null);
+ }
+ finally {
+ U.closeQuiet(s);
+ }
+ }
+
+ /**
+ * Verifies that client reconnects after connection to the server has been lost (multithreaded mode).
+ *
+ * @throws Exception If error occurs.
+ */
+ public void testClientReconnectMultithreaded() throws Exception {
+ final ConcurrentLinkedQueue<FileSystem> q = new ConcurrentLinkedQueue<>();
+
+ Configuration cfg = new Configuration();
+
+ for (Map.Entry<String, String> entry : primaryFsCfg)
+ cfg.set(entry.getKey(), entry.getValue());
+
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ final int nClients = 16;
+
+ // Initialize clients.
+ for (int i = 0; i < nClients; i++)
+ q.add(FileSystem.get(primaryFsUri, cfg));
+
+ G.stopAll(true); // Stop the server.
+
+ startNodes(); // Start server again.
+
+ GridTestUtils.runMultiThreaded(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ FileSystem fs = q.poll();
+
+ try {
+ // Check that client is again operational.
+ assertTrue(fs.mkdirs(new Path("igfs:///" + Thread.currentThread().getName())));
+
+ return true;
+ }
+ finally {
+ U.closeQuiet(fs);
+ }
+ }
+ }, nClients, "test-client");
+ }
+
+ /**
+ * Checks consistency of create --> open --> append --> open operations with different buffer sizes.
+ *
+ * @param createBufSize Buffer size used for file creation.
+ * @param writeCntsInCreate Count of times to write in file creation.
+ * @param openAfterCreateBufSize Buffer size used for file opening after creation.
+ * @param appendBufSize Buffer size used for file appending.
+ * @param writeCntsInAppend Count of times to write in file appending.
+ * @param openAfterAppendBufSize Buffer size used for file opening after appending.
+ * @throws Exception If failed.
+ */
+ private void checkConsistency(int createBufSize, int writeCntsInCreate, int openAfterCreateBufSize,
+ int appendBufSize, int writeCntsInAppend, int openAfterAppendBufSize) throws Exception {
+ final Path igfsHome = new Path(primaryFsUri);
+
+ Path file = new Path(igfsHome, "/someDir/someInnerDir/someFile");
+
+ if (createBufSize == -1)
+ createBufSize = fs.getServerDefaults().getFileBufferSize();
+
+ if (appendBufSize == -1)
+ appendBufSize = fs.getServerDefaults().getFileBufferSize();
+
+ FSDataOutputStream os = fs.create(file, EnumSet.of(CreateFlag.OVERWRITE),
+ Options.CreateOpts.perms(FsPermission.getDefault()), Options.CreateOpts.bufferSize(createBufSize));
+
+ for (int i = 0; i < writeCntsInCreate; i++)
+ os.writeInt(i);
+
+ os.close();
+
+ FSDataInputStream is = fs.open(file, openAfterCreateBufSize);
+
+ for (int i = 0; i < writeCntsInCreate; i++)
+ assertEquals(i, is.readInt());
+
+ is.close();
+
+ os = fs.create(file, EnumSet.of(CreateFlag.APPEND),
+ Options.CreateOpts.perms(FsPermission.getDefault()), Options.CreateOpts.bufferSize(appendBufSize));
+
+ for (int i = writeCntsInCreate; i < writeCntsInCreate + writeCntsInAppend; i++)
+ os.writeInt(i);
+
+ os.close();
+
+ is = fs.open(file, openAfterAppendBufSize);
+
+ for (int i = 0; i < writeCntsInCreate + writeCntsInAppend; i++)
+ assertEquals(i, is.readInt());
+
+ is.close();
+ }
+
+ /**
+ * Test expected failures for 'close' operation.
+ *
+ * @param fs File system to test.
+ * @param msg Expected exception message.
+ */
+ public void assertCloseFails(final FileSystem fs, String msg) {
+ GridTestUtils.assertThrows(log, new Callable() {
+ @Override public Object call() throws Exception {
+ fs.close();
+
+ return null;
+ }
+ }, IOException.class, msg);
+ }
+
+ /**
+ * Test expected failures for 'get content summary' operation.
+ *
+ * @param fs File system to test.
+ * @param path Path to evaluate content summary for.
+ */
+ private void assertContentSummaryFails(final FileSystem fs, final Path path) {
+ GridTestUtils.assertThrows(log, new Callable<ContentSummary>() {
+ @Override public ContentSummary call() throws Exception {
+ return fs.getContentSummary(path);
+ }
+ }, FileNotFoundException.class, null);
+ }
+
+ /**
+ * Assert that a given path exists in a given FileSystem.
+ *
+ * @param fs FileSystem to check.
+ * @param p Path to check.
+ * @throws IOException if the path does not exist.
+ */
+ private void assertPathExists(AbstractFileSystem fs, Path p) throws IOException {
+ FileStatus fileStatus = fs.getFileStatus(p);
+
+ assertEquals(p, fileStatus.getPath());
+ assertNotSame(0, fileStatus.getModificationTime());
+ }
+
+ /**
+ * Check path does not exist in a given FileSystem.
+ *
+ * @param fs FileSystem to check.
+ * @param path Path to check.
+ */
+ private void assertPathDoesNotExist(final AbstractFileSystem fs, final Path path) {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.getFileStatus(path);
+ }
+ }, FileNotFoundException.class, null);
+ }
+
+ /** Helper class to encapsulate source and destination folders. */
+ @SuppressWarnings({"PublicInnerClass", "PublicField"})
+ public static final class Config {
+ /** Source file system. */
+ public final AbstractFileSystem srcFs;
+
+ /** Source path to work with. */
+ public final Path src;
+
+ /** Destination file system. */
+ public final AbstractFileSystem destFs;
+
+ /** Destination path to work with. */
+ public final Path dest;
+
+ /**
+ * Copying task configuration.
+ *
+ * @param srcFs Source file system.
+ * @param src Source path.
+ * @param destFs Destination file system.
+ * @param dest Destination path.
+ */
+ public Config(AbstractFileSystem srcFs, Path src, AbstractFileSystem destFs, Path dest) {
+ this.srcFs = srcFs;
+ this.src = src;
+ this.destFs = destFs;
+ this.dest = dest;
+ }
+ }
+
+ /**
+ * Convert path for exception message testing purposes.
+ *
+ * @param path Path.
+ * @return Converted path.
+ * @throws Exception If failed.
+ */
+ private Path convertPath(Path path) throws Exception {
+ if (mode != PROXY)
+ return path;
+ else {
+ URI secondaryUri = new URI(secondaryFileSystemUriPath());
+
+ URI pathUri = path.toUri();
+
+ return new Path(new URI(pathUri.getScheme() != null ? secondaryUri.getScheme() : null,
+ pathUri.getAuthority() != null ? secondaryUri.getAuthority() : null, pathUri.getPath(), null, null));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemLoopbackPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemLoopbackPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemLoopbackPrimarySelfTest.java
new file mode 100644
index 0000000..2be65fd
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemLoopbackPrimarySelfTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import java.util.*;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
+
+/**
+ * Tests Hadoop 2.x file system in primary mode.
+ */
+public class HadoopIgfs20FileSystemLoopbackPrimarySelfTest extends HadoopIgfs20FileSystemAbstractSelfTest {
+ /**
+ * Creates test in primary mode.
+ */
+ public HadoopIgfs20FileSystemLoopbackPrimarySelfTest() {
+ super(PRIMARY);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected String primaryFileSystemUriPath() {
+ return "igfs://igfs:" + getTestGridName(0) + "@/";
+ }
+
+ /** {@inheritDoc} */
+ @Override protected String primaryFileSystemConfigPath() {
+ return "/modules/core/src/test/config/hadoop/core-site-loopback.xml";
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Map<String, String> primaryIpcEndpointConfiguration(final String gridName) {
+ return new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", String.valueOf(DFLT_IPC_PORT + getTestGridIndex(gridName)));
+ }};
+ }
+
+ /** {@inheritDoc} */
+ @Override protected String secondaryFileSystemUriPath() {
+ assert false;
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected String secondaryFileSystemConfigPath() {
+ assert false;
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Map<String, String> secondaryIpcEndpointConfiguration() {
+ assert false;
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemShmemPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemShmemPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemShmemPrimarySelfTest.java
new file mode 100644
index 0000000..93f2d4a
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemShmemPrimarySelfTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import java.util.*;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
+
+/**
+ * Tests Hadoop 2.x file system in primary mode.
+ */
+public class HadoopIgfs20FileSystemShmemPrimarySelfTest extends HadoopIgfs20FileSystemAbstractSelfTest {
+ /**
+ * Creates test in primary mode.
+ */
+ public HadoopIgfs20FileSystemShmemPrimarySelfTest() {
+ super(PRIMARY);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected String primaryFileSystemUriPath() {
+ return "igfs://igfs:" + getTestGridName(0) + "@/";
+ }
+
+ /** {@inheritDoc} */
+ @Override protected String primaryFileSystemConfigPath() {
+ return "/modules/core/src/test/config/hadoop/core-site.xml";
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Map<String, String> primaryIpcEndpointConfiguration(final String gridName) {
+ return new HashMap<String, String>() {{
+ put("type", "shmem");
+ put("port", String.valueOf(DFLT_IPC_PORT + getTestGridIndex(gridName)));
+ }};
+ }
+
+ /** {@inheritDoc} */
+ @Override protected String secondaryFileSystemUriPath() {
+ assert false;
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected String secondaryFileSystemConfigPath() {
+ assert false;
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Map<String, String> secondaryIpcEndpointConfiguration() {
+ assert false;
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java
new file mode 100644
index 0000000..e89d015
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.ignite.*;
+import org.apache.ignite.cache.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.hadoop.fs.*;
+import org.apache.ignite.igfs.secondary.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+import org.apache.ignite.testframework.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*;
+import static org.apache.ignite.internal.processors.igfs.IgfsAbstractSelfTest.*;
+
+/**
+ * Tests for IGFS working in mode when remote file system exists: DUAL_SYNC, DUAL_ASYNC.
+ */
+public abstract class HadoopIgfsDualAbstractSelfTest extends IgfsCommonAbstractTest {
+ /** IGFS block size. */
+ protected static final int IGFS_BLOCK_SIZE = 512 * 1024;
+
+ /** Amount of blocks to prefetch. */
+ protected static final int PREFETCH_BLOCKS = 1;
+
+ /** Amount of sequential block reads before prefetch is triggered. */
+ protected static final int SEQ_READS_BEFORE_PREFETCH = 2;
+
+ /** Secondary file system URI. */
+ protected static final String SECONDARY_URI = "igfs://igfs-secondary:grid-secondary@127.0.0.1:11500/";
+
+ /** Secondary file system configuration path. */
+ protected static final String SECONDARY_CFG = "modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml";
+
+ /** Primary file system URI. */
+ protected static final String PRIMARY_URI = "igfs://igfs:grid@/";
+
+ /** Primary file system configuration path. */
+ protected static final String PRIMARY_CFG = "modules/core/src/test/config/hadoop/core-site-loopback.xml";
+
+ /** Primary file system REST endpoint configuration map. */
+ protected static final Map<String, String> PRIMARY_REST_CFG = new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", "10500");
+ }};
+
+ /** Secondary file system REST endpoint configuration map. */
+ protected static final Map<String, String> SECONDARY_REST_CFG = new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", "11500");
+ }};
+
+ /** Directory. */
+ protected static final IgfsPath DIR = new IgfsPath("/dir");
+
+ /** Sub-directory. */
+ protected static final IgfsPath SUBDIR = new IgfsPath(DIR, "subdir");
+
+ /** File. */
+ protected static final IgfsPath FILE = new IgfsPath(SUBDIR, "file");
+
+ /** Default data chunk (128 bytes). */
+ protected static byte[] chunk;
+
+ /** Primary IGFS. */
+ protected static IgfsImpl igfs;
+
+ /** Secondary IGFS. */
+ protected static IgfsImpl igfsSecondary;
+
+ /** IGFS mode. */
+ protected final IgfsMode mode;
+
+ /**
+ * Constructor.
+ *
+ * @param mode IGFS mode.
+ */
+ protected HadoopIgfsDualAbstractSelfTest(IgfsMode mode) {
+ this.mode = mode;
+ assert mode == DUAL_SYNC || mode == DUAL_ASYNC;
+ }
+
+ /**
+ * Start grid with IGFS.
+ *
+ * @param gridName Grid name.
+ * @param igfsName IGFS name
+ * @param mode IGFS mode.
+ * @param secondaryFs Secondary file system (optional).
+ * @param restCfg Rest configuration string (optional).
+ * @return Started grid instance.
+ * @throws Exception If failed.
+ */
+ protected Ignite startGridWithIgfs(String gridName, String igfsName, IgfsMode mode,
+ @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable Map<String, String> restCfg) throws Exception {
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("dataCache");
+ igfsCfg.setMetaCacheName("metaCache");
+ igfsCfg.setName(igfsName);
+ igfsCfg.setBlockSize(IGFS_BLOCK_SIZE);
+ igfsCfg.setDefaultMode(mode);
+ igfsCfg.setIpcEndpointConfiguration(restCfg);
+ igfsCfg.setSecondaryFileSystem(secondaryFs);
+ igfsCfg.setPrefetchBlocks(PREFETCH_BLOCKS);
+ igfsCfg.setSequentialReadsBeforePrefetch(SEQ_READS_BEFORE_PREFETCH);
+
+ CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
+
+ dataCacheCfg.setName("dataCache");
+ dataCacheCfg.setCacheMode(PARTITIONED);
+ dataCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(2));
+ dataCacheCfg.setBackups(0);
+ dataCacheCfg.setQueryIndexEnabled(false);
+ dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
+ dataCacheCfg.setOffHeapMaxMemory(0);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("metaCache");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName(gridName);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ cfg.setLocalHost("127.0.0.1");
+ cfg.setConnectorConfiguration(null);
+
+ return G.start(cfg);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ chunk = new byte[128];
+
+ for (int i = 0; i < chunk.length; i++)
+ chunk[i] = (byte)i;
+
+ Ignite igniteSecondary = startGridWithIgfs("grid-secondary", "igfs-secondary", PRIMARY, null, SECONDARY_REST_CFG);
+
+ IgfsSecondaryFileSystem hadoopFs = new IgniteHadoopIgfsSecondaryFileSystem(SECONDARY_URI, SECONDARY_CFG);
+
+ Ignite ignite = startGridWithIgfs("grid", "igfs", mode, hadoopFs, PRIMARY_REST_CFG);
+
+ igfsSecondary = (IgfsImpl) igniteSecondary.fileSystem("igfs-secondary");
+ igfs = (IgfsImpl) ignite.fileSystem("igfs");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ clear(igfs);
+ clear(igfsSecondary);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ G.stopAll(true);
+ }
+
+ /**
+ * Convenient method to group paths.
+ *
+ * @param paths Paths to group.
+ * @return Paths as array.
+ */
+ protected IgfsPath[] paths(IgfsPath... paths) {
+ return paths;
+ }
+
+ /**
+ * Check how prefetch override works.
+ *
+ * @throws Exception IF failed.
+ */
+ public void testOpenPrefetchOverride() throws Exception {
+ create(igfsSecondary, paths(DIR, SUBDIR), paths(FILE));
+
+ // Write enough data to the secondary file system.
+ final int blockSize = IGFS_BLOCK_SIZE;
+
+ IgfsOutputStream out = igfsSecondary.append(FILE, false);
+
+ int totalWritten = 0;
+
+ while (totalWritten < blockSize * 2 + chunk.length) {
+ out.write(chunk);
+
+ totalWritten += chunk.length;
+ }
+
+ out.close();
+
+ awaitFileClose(igfsSecondary.asSecondary(), FILE);
+
+ // Instantiate file system with overridden "seq reads before prefetch" property.
+ Configuration cfg = new Configuration();
+
+ cfg.addResource(U.resolveIgniteUrl(PRIMARY_CFG));
+
+ int seqReads = SEQ_READS_BEFORE_PREFETCH + 1;
+
+ cfg.setInt(String.format(PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH, "igfs:grid@"), seqReads);
+
+ FileSystem fs = FileSystem.get(new URI(PRIMARY_URI), cfg);
+
+ // Read the first two blocks.
+ Path fsHome = new Path(PRIMARY_URI);
+ Path dir = new Path(fsHome, DIR.name());
+ Path subdir = new Path(dir, SUBDIR.name());
+ Path file = new Path(subdir, FILE.name());
+
+ FSDataInputStream fsIn = fs.open(file);
+
+ final byte[] readBuf = new byte[blockSize * 2];
+
+ fsIn.readFully(0, readBuf, 0, readBuf.length);
+
+ // Wait for a while for prefetch to finish (if any).
+ IgfsMetaManager meta = igfs.context().meta();
+
+ IgfsFileInfo info = meta.info(meta.fileId(FILE));
+
+ IgfsBlockKey key = new IgfsBlockKey(info.id(), info.affinityKey(), info.evictExclude(), 2);
+
+ GridCache<IgfsBlockKey, byte[]> dataCache = igfs.context().kernalContext().cache().cache(
+ igfs.configuration().getDataCacheName());
+
+ for (int i = 0; i < 10; i++) {
+ if (dataCache.containsKey(key))
+ break;
+ else
+ U.sleep(100);
+ }
+
+ fsIn.close();
+
+ // Remove the file from the secondary file system.
+ igfsSecondary.delete(FILE, false);
+
+ // Try reading the third block. Should fail.
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ IgfsInputStream in0 = igfs.open(FILE);
+
+ in0.seek(blockSize * 2);
+
+ try {
+ in0.read(readBuf);
+ }
+ finally {
+ U.closeQuiet(in0);
+ }
+
+ return null;
+ }
+ }, IOException.class,
+ "Failed to read data due to secondary file system exception: /dir/subdir/file");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAsyncSelfTest.java
new file mode 100644
index 0000000..c518b9e
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAsyncSelfTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * Tests for DUAL_ASYNC mode.
+ */
+public class HadoopIgfsDualAsyncSelfTest extends HadoopIgfsDualAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public HadoopIgfsDualAsyncSelfTest() {
+ super(DUAL_ASYNC);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualSyncSelfTest.java
new file mode 100644
index 0000000..6739535
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualSyncSelfTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * Tests for DUAL_SYNC mode.
+ */
+public class HadoopIgfsDualSyncSelfTest extends HadoopIgfsDualAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public HadoopIgfsDualSyncSelfTest() {
+ super(DUAL_SYNC);
+ }
+}
[45/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
IGNITE-386: Squashed changes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/498dcfab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/498dcfab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/498dcfab
Branch: refs/heads/ignite-386-tmp
Commit: 498dcfab57c322a33a4125ac9d0e13c8c98ef571
Parents: 9af2566
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Wed Mar 4 18:33:35 2015 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Wed Mar 4 18:33:40 2015 +0300
----------------------------------------------------------------------
bin/setup-hadoop.bat | 2 +-
bin/setup-hadoop.sh | 2 +-
config/hadoop/default-config.xml | 10 +-
docs/core-site.ignite.xml | 6 +-
examples/config/filesystem/core-site.xml | 4 +-
examples/config/filesystem/example-igfs.xml | 4 +-
.../ignite/examples/igfs/IgfsExample.java | 18 +-
.../examples/igfs/IgfsMapReduceExample.java | 6 +-
.../src/main/java/org/apache/ignite/Ignite.java | 13 +-
.../org/apache/ignite/IgniteFileSystem.java | 457 ++++
.../main/java/org/apache/ignite/IgniteFs.java | 346 ---
.../configuration/FileSystemConfiguration.java | 805 ++++++
.../configuration/HadoopConfiguration.java | 173 ++
.../ignite/configuration/IgfsConfiguration.java | 807 ------
.../configuration/IgniteConfiguration.java | 19 +-
.../main/java/org/apache/ignite/igfs/Igfs.java | 216 --
.../apache/ignite/igfs/IgfsBlockLocation.java | 2 +-
.../java/org/apache/ignite/igfs/IgfsFile.java | 2 +-
.../org/apache/ignite/igfs/IgfsInputStream.java | 6 +-
.../org/apache/ignite/igfs/IgfsMetrics.java | 2 +-
.../java/org/apache/ignite/igfs/IgfsMode.java | 8 +-
.../java/org/apache/ignite/igfs/IgfsReader.java | 38 -
.../mapreduce/IgfsInputStreamJobAdapter.java | 4 +-
.../apache/ignite/igfs/mapreduce/IgfsJob.java | 4 +-
.../igfs/mapreduce/IgfsRecordResolver.java | 2 +-
.../apache/ignite/igfs/mapreduce/IgfsTask.java | 4 +-
.../ignite/igfs/mapreduce/IgfsTaskArgs.java | 2 +-
.../IgfsByteDelimiterRecordResolver.java | 3 +-
.../records/IgfsFixedLengthRecordResolver.java | 2 +-
.../igfs/secondary/IgfsSecondaryFileSystem.java | 201 ++
...fsSecondaryFileSystemPositionedReadable.java | 38 +
.../apache/ignite/igfs/secondary/package.html | 24 +
.../ignite/internal/GridKernalContext.java | 2 +-
.../ignite/internal/GridKernalContextImpl.java | 8 +-
.../ignite/internal/IgniteComponentType.java | 4 +-
.../org/apache/ignite/internal/IgniteEx.java | 4 +-
.../apache/ignite/internal/IgniteKernal.java | 14 +-
.../org/apache/ignite/internal/IgnitionEx.java | 8 +-
.../processors/cache/GridCacheAdapter.java | 4 +-
.../processors/cache/GridCacheProcessor.java | 4 +-
.../processors/cache/GridCacheUtils.java | 4 +-
.../internal/processors/hadoop/GridHadoop.java | 86 -
.../hadoop/GridHadoopConfiguration.java | 172 --
.../processors/hadoop/GridHadoopCounter.java | 44 -
.../hadoop/GridHadoopCounterWriter.java | 36 -
.../processors/hadoop/GridHadoopCounters.java | 49 -
.../processors/hadoop/GridHadoopFileBlock.java | 162 --
.../processors/hadoop/GridHadoopInputSplit.java | 54 -
.../processors/hadoop/GridHadoopJob.java | 102 -
.../processors/hadoop/GridHadoopJobId.java | 103 -
.../processors/hadoop/GridHadoopJobInfo.java | 83 -
.../processors/hadoop/GridHadoopJobPhase.java | 38 -
.../hadoop/GridHadoopJobProperty.java | 138 -
.../processors/hadoop/GridHadoopJobStatus.java | 207 --
.../hadoop/GridHadoopMapReducePlan.java | 80 -
.../hadoop/GridHadoopMapReducePlanner.java | 40 -
.../hadoop/GridHadoopPartitioner.java | 33 -
.../hadoop/GridHadoopSerialization.java | 54 -
.../processors/hadoop/GridHadoopTask.java | 72 -
.../hadoop/GridHadoopTaskContext.java | 189 --
.../processors/hadoop/GridHadoopTaskInfo.java | 153 --
.../processors/hadoop/GridHadoopTaskInput.java | 55 -
.../processors/hadoop/GridHadoopTaskOutput.java | 40 -
.../processors/hadoop/GridHadoopTaskType.java | 56 -
.../internal/processors/hadoop/Hadoop.java | 88 +
.../processors/hadoop/HadoopFileBlock.java | 162 ++
.../processors/hadoop/HadoopInputSplit.java | 54 +
.../internal/processors/hadoop/HadoopJob.java | 102 +
.../internal/processors/hadoop/HadoopJobId.java | 103 +
.../processors/hadoop/HadoopJobInfo.java | 83 +
.../processors/hadoop/HadoopJobPhase.java | 38 +
.../processors/hadoop/HadoopJobProperty.java | 138 +
.../processors/hadoop/HadoopJobStatus.java | 207 ++
.../processors/hadoop/HadoopMapReducePlan.java | 80 +
.../hadoop/HadoopMapReducePlanner.java | 40 +
.../processors/hadoop/HadoopNoopProcessor.java | 76 +
.../processors/hadoop/HadoopPartitioner.java | 33 +
.../hadoop/HadoopProcessorAdapter.java | 96 +
.../processors/hadoop/HadoopSerialization.java | 54 +
.../internal/processors/hadoop/HadoopTask.java | 72 +
.../processors/hadoop/HadoopTaskContext.java | 190 ++
.../processors/hadoop/HadoopTaskInfo.java | 153 ++
.../processors/hadoop/HadoopTaskInput.java | 55 +
.../processors/hadoop/HadoopTaskOutput.java | 40 +
.../processors/hadoop/HadoopTaskType.java | 56 +
.../hadoop/IgniteHadoopNoopProcessor.java | 74 -
.../hadoop/IgniteHadoopProcessorAdapter.java | 94 -
.../hadoop/counter/HadoopCounter.java | 44 +
.../hadoop/counter/HadoopCounterWriter.java | 37 +
.../hadoop/counter/HadoopCounters.java | 49 +
.../internal/processors/igfs/IgfsAsyncImpl.java | 9 +-
.../internal/processors/igfs/IgfsContext.java | 6 +-
.../processors/igfs/IgfsDataManager.java | 5 +-
.../ignite/internal/processors/igfs/IgfsEx.java | 28 +-
.../internal/processors/igfs/IgfsFileInfo.java | 4 +-
.../internal/processors/igfs/IgfsImpl.java | 42 +-
.../processors/igfs/IgfsInputStreamAdapter.java | 4 +-
.../processors/igfs/IgfsInputStreamImpl.java | 5 +-
.../internal/processors/igfs/IgfsJobImpl.java | 2 +-
.../processors/igfs/IgfsMetaManager.java | 27 +-
.../processors/igfs/IgfsNoopProcessor.java | 4 +-
.../processors/igfs/IgfsOutputStreamImpl.java | 2 +-
.../internal/processors/igfs/IgfsProcessor.java | 26 +-
.../processors/igfs/IgfsProcessorAdapter.java | 4 +-
.../igfs/IgfsSecondaryFileSystemImpl.java | 121 +
.../IgfsSecondaryInputStreamDescriptor.java | 8 +-
.../processors/igfs/IgfsServerManager.java | 4 +-
.../ignite/internal/visor/igfs/VisorIgfs.java | 4 +-
.../visor/igfs/VisorIgfsProfilerClearTask.java | 2 +-
.../visor/node/VisorGridConfiguration.java | 2 +-
.../visor/node/VisorIgfsConfiguration.java | 9 +-
.../visor/node/VisorNodeDataCollectorJob.java | 2 +-
.../internal/visor/util/VisorTaskUtils.java | 4 +-
.../hadoop/core-site-loopback-secondary.xml | 4 +-
.../test/config/hadoop/core-site-loopback.xml | 4 +-
.../test/config/hadoop/core-site-secondary.xml | 4 +-
.../core/src/test/config/hadoop/core-site.xml | 4 +-
modules/core/src/test/config/igfs-loopback.xml | 4 +-
modules/core/src/test/config/igfs-shmem.xml | 4 +-
.../ignite/igfs/IgfsEventsAbstractSelfTest.java | 10 +-
.../igfs/IgfsFragmentizerAbstractSelfTest.java | 4 +-
.../ignite/igfs/IgfsFragmentizerSelfTest.java | 8 +-
.../igfs/IgfsFragmentizerTopologySelfTest.java | 2 +-
...heIgfsPerBlockLruEvictionPolicySelfTest.java | 485 ----
.../processors/igfs/IgfsAbstractSelfTest.java | 85 +-
...sCachePerBlockLruEvictionPolicySelfTest.java | 485 ++++
.../processors/igfs/IgfsCacheSelfTest.java | 4 +-
.../igfs/IgfsDataManagerSelfTest.java | 5 +-
.../igfs/IgfsDualAbstractSelfTest.java | 12 +-
.../igfs/IgfsMetaManagerSelfTest.java | 4 +-
.../processors/igfs/IgfsMetricsSelfTest.java | 22 +-
.../processors/igfs/IgfsModesSelfTest.java | 14 +-
.../processors/igfs/IgfsProcessorSelfTest.java | 10 +-
.../igfs/IgfsProcessorValidationSelfTest.java | 50 +-
...IpcEndpointRegistrationAbstractSelfTest.java | 10 +-
...dpointRegistrationOnLinuxAndMacSelfTest.java | 4 +-
...pcEndpointRegistrationOnWindowsSelfTest.java | 2 +-
.../processors/igfs/IgfsSizeSelfTest.java | 4 +-
.../processors/igfs/IgfsStreamsSelfTest.java | 14 +-
.../processors/igfs/IgfsTaskSelfTest.java | 10 +-
.../IgfsAbstractRecordResolverSelfTest.java | 6 +-
.../ipc/shmem/IpcSharedMemoryNodeStartup.java | 4 +-
.../ignite/testframework/junits/IgniteMock.java | 4 +-
.../ignite/testsuites/IgniteIgfsTestSuite.java | 2 +-
.../client/hadoop/GridHadoopClientProtocol.java | 334 ---
.../GridHadoopClientProtocolProvider.java | 137 -
.../counter/GridHadoopClientCounterGroup.java | 121 -
.../counter/GridHadoopClientCounters.java | 217 --
.../apache/ignite/client/hadoop/package.html | 24 -
.../fs/IgniteHadoopFileSystemCounterWriter.java | 93 +
.../fs/IgniteHadoopIgfsSecondaryFileSystem.java | 414 +++
.../org/apache/ignite/hadoop/fs/package.html | 24 +
.../hadoop/fs/v1/IgniteHadoopFileSystem.java | 1237 +++++++++
.../org/apache/ignite/hadoop/fs/v1/package.html | 24 +
.../hadoop/fs/v2/IgniteHadoopFileSystem.java | 982 ++++++++
.../org/apache/ignite/hadoop/fs/v2/package.html | 24 +
.../IgniteHadoopClientProtocolProvider.java | 138 +
.../mapreduce/IgniteHadoopMapReducePlanner.java | 435 ++++
.../apache/ignite/hadoop/mapreduce/package.html | 24 +
.../java/org/apache/ignite/hadoop/package.html | 24 +
.../hadoop/IgfsHadoopFileSystemWrapper.java | 414 ---
.../igfs/hadoop/IgfsHadoopParameters.java | 94 -
.../org/apache/ignite/igfs/hadoop/package.html | 24 -
.../igfs/hadoop/v1/IgfsHadoopFileSystem.java | 1234 ---------
.../apache/ignite/igfs/hadoop/v1/package.html | 24 -
.../igfs/hadoop/v2/IgfsHadoopFileSystem.java | 982 --------
.../apache/ignite/igfs/hadoop/v2/package.html | 24 -
.../java/org/apache/ignite/igfs/package.html | 24 -
.../ignite/internal/igfs/hadoop/IgfsHadoop.java | 198 --
.../IgfsHadoopCommunicationException.java | 57 -
.../igfs/hadoop/IgfsHadoopEndpoint.java | 210 --
.../internal/igfs/hadoop/IgfsHadoopEx.java | 88 -
.../igfs/hadoop/IgfsHadoopFSProperties.java | 88 -
.../internal/igfs/hadoop/IgfsHadoopFuture.java | 94 -
.../internal/igfs/hadoop/IgfsHadoopInProc.java | 409 ---
.../igfs/hadoop/IgfsHadoopInputStream.java | 626 -----
.../internal/igfs/hadoop/IgfsHadoopIo.java | 76 -
.../internal/igfs/hadoop/IgfsHadoopIpcIo.java | 599 -----
.../igfs/hadoop/IgfsHadoopIpcIoListener.java | 36 -
.../igfs/hadoop/IgfsHadoopJclLogger.java | 112 -
.../internal/igfs/hadoop/IgfsHadoopOutProc.java | 466 ----
.../igfs/hadoop/IgfsHadoopOutputStream.java | 201 --
.../igfs/hadoop/IgfsHadoopProxyInputStream.java | 335 ---
.../hadoop/IgfsHadoopProxyOutputStream.java | 165 --
.../internal/igfs/hadoop/IgfsHadoopReader.java | 104 -
.../igfs/hadoop/IgfsHadoopStreamDelegate.java | 96 -
.../hadoop/IgfsHadoopStreamEventListener.java | 39 -
.../internal/igfs/hadoop/IgfsHadoopUtils.java | 131 -
.../internal/igfs/hadoop/IgfsHadoopWrapper.java | 511 ----
.../ignite/internal/igfs/hadoop/package.html | 24 -
.../apache/ignite/internal/igfs/package.html | 24 -
.../hadoop/GridHadoopClassLoader.java | 552 ----
.../processors/hadoop/GridHadoopComponent.java | 61 -
.../processors/hadoop/GridHadoopContext.java | 196 --
.../hadoop/GridHadoopDefaultJobInfo.java | 163 --
.../processors/hadoop/GridHadoopImpl.java | 132 -
.../processors/hadoop/GridHadoopSetup.java | 505 ----
.../GridHadoopTaskCancelledException.java | 35 -
.../processors/hadoop/GridHadoopUtils.java | 308 ---
.../processors/hadoop/HadoopClassLoader.java | 552 ++++
.../processors/hadoop/HadoopComponent.java | 61 +
.../processors/hadoop/HadoopContext.java | 197 ++
.../processors/hadoop/HadoopDefaultJobInfo.java | 163 ++
.../internal/processors/hadoop/HadoopImpl.java | 134 +
.../hadoop/HadoopMapReduceCounterGroup.java | 121 +
.../hadoop/HadoopMapReduceCounters.java | 216 ++
.../processors/hadoop/HadoopProcessor.java | 227 ++
.../internal/processors/hadoop/HadoopSetup.java | 505 ++++
.../hadoop/HadoopTaskCancelledException.java | 35 +
.../internal/processors/hadoop/HadoopUtils.java | 308 +++
.../hadoop/IgniteHadoopProcessor.java | 225 --
.../counter/GridHadoopCounterAdapter.java | 128 -
.../hadoop/counter/GridHadoopCountersImpl.java | 198 --
.../counter/GridHadoopFSCounterWriter.java | 91 -
.../hadoop/counter/GridHadoopLongCounter.java | 92 -
.../counter/GridHadoopPerformanceCounter.java | 279 ---
.../hadoop/counter/HadoopCounterAdapter.java | 127 +
.../hadoop/counter/HadoopCountersImpl.java | 197 ++
.../hadoop/counter/HadoopLongCounter.java | 90 +
.../counter/HadoopPerformanceCounter.java | 279 +++
.../fs/GridHadoopDistributedFileSystem.java | 91 -
.../hadoop/fs/GridHadoopFileSystemsUtils.java | 57 -
.../hadoop/fs/GridHadoopLocalFileSystemV1.java | 39 -
.../hadoop/fs/GridHadoopLocalFileSystemV2.java | 86 -
.../hadoop/fs/GridHadoopRawLocalFileSystem.java | 304 ---
.../hadoop/fs/HadoopDistributedFileSystem.java | 91 +
.../hadoop/fs/HadoopFileSystemsUtils.java | 57 +
.../hadoop/fs/HadoopLocalFileSystemV1.java | 39 +
.../hadoop/fs/HadoopLocalFileSystemV2.java | 86 +
.../processors/hadoop/fs/HadoopParameters.java | 94 +
.../hadoop/fs/HadoopRawLocalFileSystem.java | 304 +++
.../processors/hadoop/igfs/HadoopIgfs.java | 198 ++
.../igfs/HadoopIgfsCommunicationException.java | 57 +
.../hadoop/igfs/HadoopIgfsEndpoint.java | 210 ++
.../processors/hadoop/igfs/HadoopIgfsEx.java | 88 +
.../hadoop/igfs/HadoopIgfsFuture.java | 94 +
.../hadoop/igfs/HadoopIgfsInProc.java | 409 +++
.../hadoop/igfs/HadoopIgfsInputStream.java | 626 +++++
.../processors/hadoop/igfs/HadoopIgfsIo.java | 76 +
.../processors/hadoop/igfs/HadoopIgfsIpcIo.java | 599 +++++
.../hadoop/igfs/HadoopIgfsIpcIoListener.java | 36 +
.../hadoop/igfs/HadoopIgfsJclLogger.java | 115 +
.../hadoop/igfs/HadoopIgfsOutProc.java | 466 ++++
.../hadoop/igfs/HadoopIgfsOutputStream.java | 201 ++
.../hadoop/igfs/HadoopIgfsProperties.java | 88 +
.../hadoop/igfs/HadoopIgfsProxyInputStream.java | 335 +++
.../igfs/HadoopIgfsProxyOutputStream.java | 165 ++
...fsSecondaryFileSystemPositionedReadable.java | 104 +
.../hadoop/igfs/HadoopIgfsStreamDelegate.java | 96 +
.../igfs/HadoopIgfsStreamEventListener.java | 39 +
.../processors/hadoop/igfs/HadoopIgfsUtils.java | 131 +
.../hadoop/igfs/HadoopIgfsWrapper.java | 511 ++++
.../jobtracker/GridHadoopJobMetadata.java | 305 ---
.../hadoop/jobtracker/GridHadoopJobTracker.java | 1625 ------------
.../hadoop/jobtracker/HadoopJobMetadata.java | 306 +++
.../hadoop/jobtracker/HadoopJobTracker.java | 1626 ++++++++++++
.../hadoop/message/GridHadoopMessage.java | 27 -
.../hadoop/message/HadoopMessage.java | 27 +
.../planner/GridHadoopDefaultMapReducePlan.java | 107 -
.../GridHadoopDefaultMapReducePlanner.java | 434 ----
.../planner/HadoopDefaultMapReducePlan.java | 107 +
.../GridHadoopProtocolJobCountersTask.java | 45 -
.../proto/GridHadoopProtocolJobStatusTask.java | 81 -
.../proto/GridHadoopProtocolKillJobTask.java | 46 -
.../proto/GridHadoopProtocolNextTaskIdTask.java | 35 -
.../proto/GridHadoopProtocolSubmitJobTask.java | 57 -
.../proto/GridHadoopProtocolTaskAdapter.java | 113 -
.../proto/GridHadoopProtocolTaskArguments.java | 81 -
.../hadoop/proto/HadoopClientProtocol.java | 333 +++
.../proto/HadoopProtocolJobCountersTask.java | 46 +
.../proto/HadoopProtocolJobStatusTask.java | 81 +
.../hadoop/proto/HadoopProtocolKillJobTask.java | 46 +
.../proto/HadoopProtocolNextTaskIdTask.java | 35 +
.../proto/HadoopProtocolSubmitJobTask.java | 57 +
.../hadoop/proto/HadoopProtocolTaskAdapter.java | 113 +
.../proto/HadoopProtocolTaskArguments.java | 81 +
.../hadoop/shuffle/GridHadoopShuffle.java | 256 --
.../hadoop/shuffle/GridHadoopShuffleAck.java | 91 -
.../hadoop/shuffle/GridHadoopShuffleJob.java | 593 -----
.../shuffle/GridHadoopShuffleMessage.java | 242 --
.../hadoop/shuffle/HadoopShuffle.java | 256 ++
.../hadoop/shuffle/HadoopShuffleAck.java | 91 +
.../hadoop/shuffle/HadoopShuffleJob.java | 593 +++++
.../hadoop/shuffle/HadoopShuffleMessage.java | 241 ++
.../GridHadoopConcurrentHashMultimap.java | 611 -----
.../collections/GridHadoopHashMultimap.java | 174 --
.../collections/GridHadoopHashMultimapBase.java | 208 --
.../shuffle/collections/GridHadoopMultimap.java | 112 -
.../collections/GridHadoopMultimapBase.java | 368 ---
.../shuffle/collections/GridHadoopSkipList.java | 726 ------
.../HadoopConcurrentHashMultimap.java | 611 +++++
.../shuffle/collections/HadoopHashMultimap.java | 174 ++
.../collections/HadoopHashMultimapBase.java | 208 ++
.../shuffle/collections/HadoopMultimap.java | 112 +
.../shuffle/collections/HadoopMultimapBase.java | 368 +++
.../shuffle/collections/HadoopSkipList.java | 726 ++++++
.../shuffle/streams/GridHadoopDataInStream.java | 170 --
.../streams/GridHadoopDataOutStream.java | 131 -
.../streams/GridHadoopOffheapBuffer.java | 122 -
.../shuffle/streams/HadoopDataInStream.java | 170 ++
.../shuffle/streams/HadoopDataOutStream.java | 131 +
.../shuffle/streams/HadoopOffheapBuffer.java | 122 +
.../GridHadoopEmbeddedTaskExecutor.java | 146 --
.../taskexecutor/GridHadoopExecutorService.java | 232 --
.../taskexecutor/GridHadoopRunnableTask.java | 268 --
.../GridHadoopTaskExecutorAdapter.java | 57 -
.../taskexecutor/GridHadoopTaskState.java | 38 -
.../taskexecutor/GridHadoopTaskStatus.java | 114 -
.../HadoopEmbeddedTaskExecutor.java | 146 ++
.../taskexecutor/HadoopExecutorService.java | 231 ++
.../hadoop/taskexecutor/HadoopRunnableTask.java | 268 ++
.../taskexecutor/HadoopTaskExecutorAdapter.java | 57 +
.../hadoop/taskexecutor/HadoopTaskState.java | 38 +
.../hadoop/taskexecutor/HadoopTaskStatus.java | 114 +
.../GridHadoopExternalTaskExecutor.java | 960 -------
.../GridHadoopExternalTaskMetadata.java | 68 -
.../GridHadoopJobInfoUpdateRequest.java | 109 -
.../GridHadoopPrepareForJobRequest.java | 126 -
.../external/GridHadoopProcessDescriptor.java | 150 --
.../external/GridHadoopProcessStartedAck.java | 46 -
.../GridHadoopTaskExecutionRequest.java | 110 -
.../external/GridHadoopTaskFinishedMessage.java | 92 -
.../external/HadoopExternalTaskExecutor.java | 960 +++++++
.../external/HadoopExternalTaskMetadata.java | 68 +
.../external/HadoopJobInfoUpdateRequest.java | 109 +
.../external/HadoopPrepareForJobRequest.java | 126 +
.../external/HadoopProcessDescriptor.java | 150 ++
.../external/HadoopProcessStartedAck.java | 46 +
.../external/HadoopTaskExecutionRequest.java | 110 +
.../external/HadoopTaskFinishedMessage.java | 92 +
.../child/GridHadoopChildProcessRunner.java | 440 ----
.../child/GridHadoopExternalProcessStarter.java | 296 ---
.../child/HadoopChildProcessRunner.java | 440 ++++
.../child/HadoopExternalProcessStarter.java | 296 +++
.../GridHadoopAbstractCommunicationClient.java | 96 -
.../GridHadoopCommunicationClient.java | 72 -
.../GridHadoopExternalCommunication.java | 1431 -----------
.../GridHadoopHandshakeTimeoutException.java | 42 -
.../GridHadoopIpcToNioAdapter.java | 239 --
.../GridHadoopMarshallerFilter.java | 84 -
.../GridHadoopMessageListener.java | 39 -
.../GridHadoopTcpNioCommunicationClient.java | 99 -
.../HadoopAbstractCommunicationClient.java | 96 +
.../HadoopCommunicationClient.java | 72 +
.../HadoopExternalCommunication.java | 1431 +++++++++++
.../HadoopHandshakeTimeoutException.java | 42 +
.../communication/HadoopIpcToNioAdapter.java | 239 ++
.../communication/HadoopMarshallerFilter.java | 84 +
.../communication/HadoopMessageListener.java | 39 +
.../HadoopTcpNioCommunicationClient.java | 99 +
.../hadoop/v1/GridHadoopV1CleanupTask.java | 62 -
.../hadoop/v1/GridHadoopV1Counter.java | 105 -
.../hadoop/v1/GridHadoopV1MapTask.java | 111 -
.../hadoop/v1/GridHadoopV1OutputCollector.java | 130 -
.../hadoop/v1/GridHadoopV1Partitioner.java | 44 -
.../hadoop/v1/GridHadoopV1ReduceTask.java | 92 -
.../hadoop/v1/GridHadoopV1Reporter.java | 79 -
.../hadoop/v1/GridHadoopV1SetupTask.java | 56 -
.../hadoop/v1/GridHadoopV1Splitter.java | 97 -
.../processors/hadoop/v1/GridHadoopV1Task.java | 95 -
.../hadoop/v1/HadoopV1CleanupTask.java | 62 +
.../processors/hadoop/v1/HadoopV1Counter.java | 105 +
.../processors/hadoop/v1/HadoopV1MapTask.java | 111 +
.../hadoop/v1/HadoopV1OutputCollector.java | 130 +
.../hadoop/v1/HadoopV1Partitioner.java | 44 +
.../hadoop/v1/HadoopV1ReduceTask.java | 92 +
.../processors/hadoop/v1/HadoopV1Reporter.java | 79 +
.../processors/hadoop/v1/HadoopV1SetupTask.java | 56 +
.../processors/hadoop/v1/HadoopV1Splitter.java | 97 +
.../processors/hadoop/v1/HadoopV1Task.java | 95 +
.../hadoop/v2/GridHadoopExternalSplit.java | 87 -
.../hadoop/v2/GridHadoopNativeCodeLoader.java | 74 -
.../v2/GridHadoopSerializationWrapper.java | 133 -
.../v2/GridHadoopShutdownHookManager.java | 96 -
.../hadoop/v2/GridHadoopSplitWrapper.java | 118 -
.../hadoop/v2/GridHadoopV2CleanupTask.java | 73 -
.../hadoop/v2/GridHadoopV2Context.java | 230 --
.../hadoop/v2/GridHadoopV2Counter.java | 87 -
.../processors/hadoop/v2/GridHadoopV2Job.java | 280 ---
.../v2/GridHadoopV2JobResourceManager.java | 305 ---
.../hadoop/v2/GridHadoopV2MapTask.java | 109 -
.../hadoop/v2/GridHadoopV2Partitioner.java | 44 -
.../hadoop/v2/GridHadoopV2ReduceTask.java | 88 -
.../hadoop/v2/GridHadoopV2SetupTask.java | 66 -
.../hadoop/v2/GridHadoopV2Splitter.java | 105 -
.../processors/hadoop/v2/GridHadoopV2Task.java | 181 --
.../hadoop/v2/GridHadoopV2TaskContext.java | 443 ----
.../v2/GridHadoopWritableSerialization.java | 74 -
.../hadoop/v2/HadoopExternalSplit.java | 87 +
.../hadoop/v2/HadoopNativeCodeLoader.java | 74 +
.../hadoop/v2/HadoopSerializationWrapper.java | 133 +
.../hadoop/v2/HadoopShutdownHookManager.java | 96 +
.../hadoop/v2/HadoopSplitWrapper.java | 118 +
.../hadoop/v2/HadoopV2CleanupTask.java | 73 +
.../processors/hadoop/v2/HadoopV2Context.java | 230 ++
.../processors/hadoop/v2/HadoopV2Counter.java | 87 +
.../processors/hadoop/v2/HadoopV2Job.java | 280 +++
.../hadoop/v2/HadoopV2JobResourceManager.java | 305 +++
.../processors/hadoop/v2/HadoopV2MapTask.java | 109 +
.../hadoop/v2/HadoopV2Partitioner.java | 44 +
.../hadoop/v2/HadoopV2ReduceTask.java | 88 +
.../processors/hadoop/v2/HadoopV2SetupTask.java | 66 +
.../processors/hadoop/v2/HadoopV2Splitter.java | 105 +
.../processors/hadoop/v2/HadoopV2Task.java | 181 ++
.../hadoop/v2/HadoopV2TaskContext.java | 444 ++++
.../hadoop/v2/HadoopWritableSerialization.java | 74 +
...op.mapreduce.protocol.ClientProtocolProvider | 2 +-
...ridHadoopClientProtocolEmbeddedSelfTest.java | 34 -
.../GridHadoopClientProtocolSelfTest.java | 633 -----
.../HadoopClientProtocolEmbeddedSelfTest.java | 34 +
.../hadoop/HadoopClientProtocolSelfTest.java | 635 +++++
.../HadoopIgfs20FileSystemAbstractSelfTest.java | 1967 +++++++++++++++
...Igfs20FileSystemLoopbackPrimarySelfTest.java | 74 +
...oopIgfs20FileSystemShmemPrimarySelfTest.java | 74 +
.../igfs/HadoopIgfsDualAbstractSelfTest.java | 305 +++
.../igfs/HadoopIgfsDualAsyncSelfTest.java | 32 +
.../ignite/igfs/HadoopIgfsDualSyncSelfTest.java | 32 +
...oopSecondaryFileSystemConfigurationTest.java | 27 +-
.../apache/ignite/igfs/IgfsEventsTestSuite.java | 54 +-
.../IgfsHadoop20FileSystemAbstractSelfTest.java | 1967 ---------------
...doop20FileSystemLoopbackPrimarySelfTest.java | 74 -
...sHadoop20FileSystemShmemPrimarySelfTest.java | 74 -
.../igfs/IgfsHadoopDualAbstractSelfTest.java | 304 ---
.../igfs/IgfsHadoopDualAsyncSelfTest.java | 32 -
.../ignite/igfs/IgfsHadoopDualSyncSelfTest.java | 32 -
.../IgfsHadoopFileSystemAbstractSelfTest.java | 2366 ------------------
.../IgfsHadoopFileSystemClientSelfTest.java | 199 --
.../IgfsHadoopFileSystemHandshakeSelfTest.java | 311 ---
.../IgfsHadoopFileSystemIpcCacheSelfTest.java | 207 --
.../IgfsHadoopFileSystemLoggerSelfTest.java | 287 ---
...IgfsHadoopFileSystemLoggerStateSelfTest.java | 325 ---
...adoopFileSystemLoopbackAbstractSelfTest.java | 46 -
...SystemLoopbackEmbeddedDualAsyncSelfTest.java | 33 -
...eSystemLoopbackEmbeddedDualSyncSelfTest.java | 33 -
...leSystemLoopbackEmbeddedPrimarySelfTest.java | 33 -
...SystemLoopbackEmbeddedSecondarySelfTest.java | 34 -
...SystemLoopbackExternalDualAsyncSelfTest.java | 33 -
...eSystemLoopbackExternalDualSyncSelfTest.java | 33 -
...leSystemLoopbackExternalPrimarySelfTest.java | 33 -
...SystemLoopbackExternalSecondarySelfTest.java | 34 -
...fsHadoopFileSystemSecondaryModeSelfTest.java | 319 ---
...fsHadoopFileSystemShmemAbstractSelfTest.java | 88 -
...ileSystemShmemEmbeddedDualAsyncSelfTest.java | 33 -
...FileSystemShmemEmbeddedDualSyncSelfTest.java | 33 -
...pFileSystemShmemEmbeddedPrimarySelfTest.java | 33 -
...ileSystemShmemEmbeddedSecondarySelfTest.java | 33 -
...ileSystemShmemExternalDualAsyncSelfTest.java | 33 -
...FileSystemShmemExternalDualSyncSelfTest.java | 33 -
...pFileSystemShmemExternalPrimarySelfTest.java | 33 -
...ileSystemShmemExternalSecondarySelfTest.java | 33 -
.../igfs/IgfsNearOnlyMultiNodeSelfTest.java | 4 +-
.../IgniteHadoopFileSystemAbstractSelfTest.java | 2366 ++++++++++++++++++
.../IgniteHadoopFileSystemClientSelfTest.java | 199 ++
...IgniteHadoopFileSystemHandshakeSelfTest.java | 311 +++
.../IgniteHadoopFileSystemIpcCacheSelfTest.java | 207 ++
.../IgniteHadoopFileSystemLoggerSelfTest.java | 287 +++
...niteHadoopFileSystemLoggerStateSelfTest.java | 325 +++
...adoopFileSystemLoopbackAbstractSelfTest.java | 46 +
...SystemLoopbackEmbeddedDualAsyncSelfTest.java | 33 +
...eSystemLoopbackEmbeddedDualSyncSelfTest.java | 33 +
...leSystemLoopbackEmbeddedPrimarySelfTest.java | 33 +
...SystemLoopbackEmbeddedSecondarySelfTest.java | 34 +
...SystemLoopbackExternalDualAsyncSelfTest.java | 33 +
...eSystemLoopbackExternalDualSyncSelfTest.java | 33 +
...leSystemLoopbackExternalPrimarySelfTest.java | 33 +
...SystemLoopbackExternalSecondarySelfTest.java | 34 +
...teHadoopFileSystemSecondaryModeSelfTest.java | 319 +++
...teHadoopFileSystemShmemAbstractSelfTest.java | 88 +
...ileSystemShmemEmbeddedDualAsyncSelfTest.java | 33 +
...FileSystemShmemEmbeddedDualSyncSelfTest.java | 33 +
...pFileSystemShmemEmbeddedPrimarySelfTest.java | 33 +
...ileSystemShmemEmbeddedSecondarySelfTest.java | 33 +
...ileSystemShmemExternalDualAsyncSelfTest.java | 33 +
...FileSystemShmemExternalDualSyncSelfTest.java | 33 +
...pFileSystemShmemExternalPrimarySelfTest.java | 33 +
...ileSystemShmemExternalSecondarySelfTest.java | 33 +
.../hadoop/GridHadoopAbstractSelfTest.java | 222 --
.../hadoop/GridHadoopAbstractWordCountTest.java | 138 -
.../hadoop/GridHadoopClassLoaderTest.java | 69 -
.../hadoop/GridHadoopCommandLineTest.java | 440 ----
...idHadoopDefaultMapReducePlannerSelfTest.java | 1005 --------
.../hadoop/GridHadoopFileSystemsTest.java | 177 --
.../hadoop/GridHadoopGroupingTest.java | 286 ---
.../hadoop/GridHadoopJobTrackerSelfTest.java | 330 ---
.../GridHadoopMapReduceEmbeddedSelfTest.java | 245 --
.../hadoop/GridHadoopMapReduceTest.java | 195 --
.../hadoop/GridHadoopPopularWordsTest.java | 294 ---
.../GridHadoopSerializationWrapperSelfTest.java | 74 -
.../processors/hadoop/GridHadoopSharedMap.java | 67 -
.../hadoop/GridHadoopSortingExternalTest.java | 32 -
.../hadoop/GridHadoopSortingTest.java | 281 ---
.../hadoop/GridHadoopSplitWrapperSelfTest.java | 68 -
.../processors/hadoop/GridHadoopStartup.java | 55 -
.../hadoop/GridHadoopTaskExecutionSelfTest.java | 551 ----
.../hadoop/GridHadoopTasksAllVersionsTest.java | 259 --
.../hadoop/GridHadoopTasksV1Test.java | 57 -
.../hadoop/GridHadoopTasksV2Test.java | 75 -
.../GridHadoopTestRoundRobinMrPlanner.java | 66 -
.../hadoop/GridHadoopTestTaskContext.java | 219 --
.../processors/hadoop/GridHadoopTestUtils.java | 102 -
.../hadoop/GridHadoopV2JobSelfTest.java | 88 -
.../hadoop/GridHadoopValidationSelfTest.java | 53 -
.../hadoop/HadoopAbstractSelfTest.java | 222 ++
.../hadoop/HadoopAbstractWordCountTest.java | 138 +
.../hadoop/HadoopClassLoaderTest.java | 69 +
.../hadoop/HadoopCommandLineTest.java | 440 ++++
.../HadoopDefaultMapReducePlannerSelfTest.java | 1006 ++++++++
.../hadoop/HadoopFileSystemsTest.java | 177 ++
.../processors/hadoop/HadoopGroupingTest.java | 287 +++
.../hadoop/HadoopJobTrackerSelfTest.java | 331 +++
.../hadoop/HadoopMapReduceEmbeddedSelfTest.java | 246 ++
.../processors/hadoop/HadoopMapReduceTest.java | 197 ++
.../hadoop/HadoopPopularWordsTest.java | 294 +++
.../HadoopSerializationWrapperSelfTest.java | 74 +
.../processors/hadoop/HadoopSharedMap.java | 67 +
.../hadoop/HadoopSortingExternalTest.java | 34 +
.../processors/hadoop/HadoopSortingTest.java | 282 +++
.../hadoop/HadoopSplitWrapperSelfTest.java | 68 +
.../processors/hadoop/HadoopStartup.java | 54 +
.../hadoop/HadoopTaskExecutionSelfTest.java | 551 ++++
.../hadoop/HadoopTasksAllVersionsTest.java | 259 ++
.../processors/hadoop/HadoopTasksV1Test.java | 57 +
.../processors/hadoop/HadoopTasksV2Test.java | 75 +
.../hadoop/HadoopTestRoundRobinMrPlanner.java | 66 +
.../hadoop/HadoopTestTaskContext.java | 219 ++
.../processors/hadoop/HadoopTestUtils.java | 102 +
.../processors/hadoop/HadoopV2JobSelfTest.java | 88 +
.../hadoop/HadoopValidationSelfTest.java | 53 +
.../hadoop/examples/GridHadoopWordCount1.java | 88 -
.../examples/GridHadoopWordCount1Map.java | 62 -
.../examples/GridHadoopWordCount1Reduce.java | 51 -
.../hadoop/examples/GridHadoopWordCount2.java | 95 -
.../examples/GridHadoopWordCount2Mapper.java | 72 -
.../examples/GridHadoopWordCount2Reducer.java | 70 -
.../hadoop/examples/HadoopWordCount1.java | 88 +
.../hadoop/examples/HadoopWordCount1Map.java | 62 +
.../hadoop/examples/HadoopWordCount1Reduce.java | 51 +
.../hadoop/examples/HadoopWordCount2.java | 95 +
.../hadoop/examples/HadoopWordCount2Mapper.java | 72 +
.../examples/HadoopWordCount2Reducer.java | 70 +
.../collections/GridHadoopAbstractMapTest.java | 152 --
...ridHadoopConcurrentHashMultimapSelftest.java | 267 --
.../collections/GridHadoopHashMapSelfTest.java | 170 --
.../collections/GridHadoopSkipListSelfTest.java | 303 ---
.../collections/HadoopAbstractMapTest.java | 154 ++
.../HadoopConcurrentHashMultimapSelftest.java | 267 ++
.../collections/HadoopHashMapSelfTest.java | 170 ++
.../collections/HadoopSkipListSelfTest.java | 303 +++
.../streams/GridHadoopDataStreamSelfTest.java | 151 --
.../streams/HadoopDataStreamSelfTest.java | 151 ++
.../GridHadoopExecutorServiceTest.java | 119 -
.../taskexecutor/HadoopExecutorServiceTest.java | 119 +
...GridHadoopExternalTaskExecutionSelfTest.java | 210 --
.../HadoopExternalTaskExecutionSelfTest.java | 211 ++
...GridHadoopExternalCommunicationSelfTest.java | 209 --
.../HadoopExternalCommunicationSelfTest.java | 209 ++
.../testsuites/IgniteHadoopTestSuite.java | 82 +-
.../IgniteIgfsLinuxAndMacOSTestSuite.java | 22 +-
.../org/apache/ignite/IgniteSpringBean.java | 4 +-
.../scala/org/apache/ignite/visor/visor.scala | 4 +-
pom.xml | 12 +-
561 files changed, 47565 insertions(+), 47291 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/bin/setup-hadoop.bat
----------------------------------------------------------------------
diff --git a/bin/setup-hadoop.bat b/bin/setup-hadoop.bat
index c4c73b3..a11ef8c 100644
--- a/bin/setup-hadoop.bat
+++ b/bin/setup-hadoop.bat
@@ -23,6 +23,6 @@
if "%OS%" == "Windows_NT" setlocal
-set MAIN_CLASS=org.apache.ignite.hadoop.GridHadoopSetup
+set MAIN_CLASS=org.apache.ignite.internal.processors.hadoop.HadoopSetup
call "%~dp0\ignite.bat" %*
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/bin/setup-hadoop.sh
----------------------------------------------------------------------
diff --git a/bin/setup-hadoop.sh b/bin/setup-hadoop.sh
index 8969dfa..d66353f 100755
--- a/bin/setup-hadoop.sh
+++ b/bin/setup-hadoop.sh
@@ -54,7 +54,7 @@ setIgniteHome
#
# Set utility environment.
#
-export MAIN_CLASS=org.apache.ignite.internal.processors.hadoop.GridHadoopSetup
+export MAIN_CLASS=org.apache.ignite.internal.processors.hadoop.HadoopSetup
#
# Start utility.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/config/hadoop/default-config.xml
----------------------------------------------------------------------
diff --git a/config/hadoop/default-config.xml b/config/hadoop/default-config.xml
index a264749..65a281e 100644
--- a/config/hadoop/default-config.xml
+++ b/config/hadoop/default-config.xml
@@ -52,7 +52,7 @@
<!--
Abstract IGFS file system configuration to be used as a template.
-->
- <bean id="igfsCfgBase" class="org.apache.ignite.configuration.IgfsConfiguration" abstract="true">
+ <bean id="igfsCfgBase" class="org.apache.ignite.configuration.FileSystemConfiguration" abstract="true">
<!-- Must correlate with cache affinity mapper. -->
<property name="blockSize" value="#{128 * 1024}"/>
<property name="perNodeBatchSize" value="512"/>
@@ -94,7 +94,7 @@
Apache Hadoop Accelerator configuration.
-->
<property name="hadoopConfiguration">
- <bean class="org.apache.ignite.internal.processors.hadoop.GridHadoopConfiguration">
+ <bean class="org.apache.ignite.configuration.HadoopConfiguration">
<!-- Information about finished jobs will be kept for 30 seconds. -->
<property name="finishedJobInfoTtl" value="30000"/>
</bean>
@@ -112,9 +112,9 @@
<!--
Configure one IGFS file system instance named "igfs" on this node.
-->
- <property name="igfsConfiguration">
+ <property name="fileSystemConfiguration">
<list>
- <bean class="org.apache.ignite.configuration.IgfsConfiguration" parent="igfsCfgBase">
+ <bean class="org.apache.ignite.configuration.FileSystemConfiguration" parent="igfsCfgBase">
<property name="name" value="igfs"/>
<!-- Caches with these names must be configured. -->
@@ -135,7 +135,7 @@
<property name="defaultMode" value="PROXY"/>
<property name="secondaryFileSystem">
- <bean class="org.apache.ignite.igfs.hadoop.IgfsHadoopFileSystemWrapper">
+ <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem">
<constructor-arg name="uri" value="hdfs://1.2.3.4:9000"/>
<constructor-arg name="cfgPath" value="/opt/hadoop-server/etc/hadoop/core-site.xml"/>
</bean>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/docs/core-site.ignite.xml
----------------------------------------------------------------------
diff --git a/docs/core-site.ignite.xml b/docs/core-site.ignite.xml
index ed11a79..8b8e634 100644
--- a/docs/core-site.ignite.xml
+++ b/docs/core-site.ignite.xml
@@ -48,7 +48,7 @@
-->
<property>
<name>fs.igfs.impl</name>
- <value>org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem</value>
+ <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value>
</property>
<!--
@@ -56,7 +56,7 @@
-->
<property>
<name>fs.AbstractFileSystem.igfs.impl</name>
- <value>org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem</value>
+ <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value>
</property>
<!--
@@ -73,7 +73,7 @@
<!--
<property>
<name>ignite.counters.writer</name>
- <value>org.apache.ignite.internal.processors.hadoop.counter.GridHadoopFSCounterWriter</value>
+ <value>org.apache.ignite.hadoop.fs.IgniteHadoopFileSystemCounterWriter</value>
</property>
-->
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/examples/config/filesystem/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/config/filesystem/core-site.xml b/examples/config/filesystem/core-site.xml
index 7c6cfaa..b6f0291 100644
--- a/examples/config/filesystem/core-site.xml
+++ b/examples/config/filesystem/core-site.xml
@@ -31,12 +31,12 @@
<property>
<!-- FS driver class for the 'igfs://' URIs. -->
<name>fs.igfs.impl</name>
- <value>org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem</value>
+ <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value>
</property>
<property>
<!-- FS driver class for the 'igfs://' URIs in Hadoop2.x -->
<name>fs.AbstractFileSystem.igfs.impl</name>
- <value>org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem</value>
+ <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/examples/config/filesystem/example-igfs.xml
----------------------------------------------------------------------
diff --git a/examples/config/filesystem/example-igfs.xml b/examples/config/filesystem/example-igfs.xml
index 30cf51e..d8ccd34 100644
--- a/examples/config/filesystem/example-igfs.xml
+++ b/examples/config/filesystem/example-igfs.xml
@@ -63,9 +63,9 @@
</bean>
</property>
- <property name="igfsConfiguration">
+ <property name="fileSystemConfiguration">
<list>
- <bean class="org.apache.ignite.configuration.IgfsConfiguration">
+ <bean class="org.apache.ignite.configuration.FileSystemConfiguration">
<property name="name" value="igfs"/>
<property name="metaCacheName" value="igfs-meta"/>
<property name="dataCacheName" value="igfs-data"/>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java b/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java
index 56f3fa4..8bfd38d 100644
--- a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java
@@ -25,7 +25,7 @@ import java.io.*;
import java.util.*;
/**
- * Example that shows usage of {@link org.apache.ignite.IgniteFs} API. It starts a node with {@code IgniteFs}
+ * Example that shows usage of {@link org.apache.ignite.IgniteFileSystem} API. It starts a node with {@code IgniteFs}
* configured and performs several file system operations (create, write, append, read and delete
* files, create, list and delete directories).
* <p>
@@ -50,7 +50,7 @@ public final class IgfsExample {
try {
// Get an instance of Ignite File System.
- IgniteFs fs = ignite.fileSystem("igfs");
+ IgniteFileSystem fs = ignite.fileSystem("igfs");
// Working directory path.
IgfsPath workDir = new IgfsPath("/examples/fs");
@@ -107,7 +107,7 @@ public final class IgfsExample {
* @param path File or directory path.
* @throws IgniteException In case of error.
*/
- private static void delete(IgniteFs fs, IgfsPath path) throws IgniteException {
+ private static void delete(IgniteFileSystem fs, IgfsPath path) throws IgniteException {
assert fs != null;
assert path != null;
@@ -139,7 +139,7 @@ public final class IgfsExample {
* @param path Directory path.
* @throws IgniteException In case of error.
*/
- private static void mkdirs(IgniteFs fs, IgfsPath path) throws IgniteException {
+ private static void mkdirs(IgniteFileSystem fs, IgfsPath path) throws IgniteException {
assert fs != null;
assert path != null;
@@ -166,7 +166,7 @@ public final class IgfsExample {
* @throws IgniteException If file can't be created.
* @throws IOException If data can't be written.
*/
- private static void create(IgniteFs fs, IgfsPath path, @Nullable byte[] data)
+ private static void create(IgniteFileSystem fs, IgfsPath path, @Nullable byte[] data)
throws IgniteException, IOException {
assert fs != null;
assert path != null;
@@ -195,7 +195,7 @@ public final class IgfsExample {
* @throws IgniteException If file can't be created.
* @throws IOException If data can't be written.
*/
- private static void append(IgniteFs fs, IgfsPath path, byte[] data) throws IgniteException, IOException {
+ private static void append(IgniteFileSystem fs, IgfsPath path, byte[] data) throws IgniteException, IOException {
assert fs != null;
assert path != null;
assert data != null;
@@ -220,7 +220,7 @@ public final class IgfsExample {
* @throws IgniteException If file can't be opened.
* @throws IOException If data can't be read.
*/
- private static void read(IgniteFs fs, IgfsPath path) throws IgniteException, IOException {
+ private static void read(IgniteFileSystem fs, IgfsPath path) throws IgniteException, IOException {
assert fs != null;
assert path != null;
assert fs.info(path).isFile();
@@ -242,7 +242,7 @@ public final class IgfsExample {
* @param path Directory path.
* @throws IgniteException In case of error.
*/
- private static void list(IgniteFs fs, IgfsPath path) throws IgniteException {
+ private static void list(IgniteFileSystem fs, IgfsPath path) throws IgniteException {
assert fs != null;
assert path != null;
assert fs.info(path).isDirectory();
@@ -271,7 +271,7 @@ public final class IgfsExample {
* @param path File or directory path.
* @throws IgniteException In case of error.
*/
- private static void printInfo(IgniteFs fs, IgfsPath path) throws IgniteException {
+ private static void printInfo(IgniteFileSystem fs, IgfsPath path) throws IgniteException {
System.out.println();
System.out.println("Information for " + path + ": " + fs.info(path));
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java b/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java
index 22a66a3..ed0abe4 100644
--- a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java
@@ -61,7 +61,7 @@ public class IgfsMapReduceExample {
String regexStr = args[1];
// Get an instance of Ignite File System.
- IgniteFs fs = ignite.fileSystem("igfs");
+ IgniteFileSystem fs = ignite.fileSystem("igfs");
// Working directory path.
IgfsPath workDir = new IgfsPath("/examples/fs");
@@ -94,7 +94,7 @@ public class IgfsMapReduceExample {
* @param file File to write.
* @throws Exception In case of exception.
*/
- private static void writeFile(IgniteFs fs, IgfsPath fsPath, File file) throws Exception {
+ private static void writeFile(IgniteFileSystem fs, IgfsPath fsPath, File file) throws Exception {
System.out.println();
System.out.println("Copying file to IGFS: " + file);
@@ -173,7 +173,7 @@ public class IgfsMapReduceExample {
}
/** {@inheritDoc} */
- @Override public Object execute(IgniteFs igfs, IgfsRangeInputStream in) throws IgniteException, IOException {
+ @Override public Object execute(IgniteFileSystem igfs, IgfsRangeInputStream in) throws IgniteException, IOException {
Collection<Line> res = null;
long start = in.startOffset();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/Ignite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java
index 8851d8f..31b827e 100644
--- a/modules/core/src/main/java/org/apache/ignite/Ignite.java
+++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java
@@ -43,7 +43,7 @@ import java.util.concurrent.*;
* <ul>
* <li>{@link org.apache.ignite.cache.GridCache} - functionality for in-memory distributed cache.</li>
* <li>{@link IgniteDataLoader} - functionality for loading data large amounts of data into cache.</li>
- * <li>{@link IgniteFs} - functionality for distributed Hadoop-compliant in-memory file system and map-reduce.</li>
+ * <li>{@link IgniteFileSystem} - functionality for distributed Hadoop-compliant in-memory file system and map-reduce.</li>
* <li>{@link IgniteStreamer} - functionality for streaming events workflow with queries and indexes into rolling windows.</li>
* <li>{@link IgniteScheduler} - functionality for scheduling jobs using UNIX Cron syntax.</li>
* <li>{@link IgniteCompute} - functionality for executing tasks and closures on all grid nodes (inherited form {@link ClusterGroup}).</li>
@@ -213,7 +213,7 @@ public interface Ignite extends AutoCloseable {
public <K, V> IgniteDataLoader<K, V> dataLoader(@Nullable String cacheName);
/**
- * Gets an instance of IGFS - Ignite In-Memory File System, if one is not
+ * Gets an instance of IGFS (Ignite In-Memory File System). If one is not
* configured then {@link IllegalArgumentException} will be thrown.
* <p>
* IGFS is fully compliant with Hadoop {@code FileSystem} APIs and can
@@ -222,15 +222,16 @@ public interface Ignite extends AutoCloseable {
*
* @param name IGFS name.
* @return IGFS instance.
+ * @throws IllegalArgumentException If IGFS with such name is not configured.
*/
- public IgniteFs fileSystem(String name);
+ public IgniteFileSystem fileSystem(String name);
/**
- * Gets all instances of the grid file systems.
+ * Gets all instances of IGFS (Ignite In-Memory File System).
*
- * @return Collection of grid file systems instances.
+ * @return Collection of IGFS instances.
*/
- public Collection<IgniteFs> fileSystems();
+ public Collection<IgniteFileSystem> fileSystems();
/**
* Gets an instance of streamer by name, if one does not exist then
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
new file mode 100644
index 0000000..d221ae2
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
@@ -0,0 +1,457 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite;
+
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.mapreduce.*;
+import org.apache.ignite.lang.*;
+import org.jetbrains.annotations.*;
+
+import java.util.*;
+
+/**
+ * <b>IG</b>nite <b>F</b>ile <b>S</b>ystem API. It provides a typical file system "view" on a particular cache:
+ * <ul>
+ * <li>list directories or get information for a single path</li>
+ * <li>create/move/delete files or directories</li>
+ * <li>write/read data streams into/from files</li>
+ * </ul>
+ * The data of each file is split on separate data blocks and stored in the cache.
+ * You can access file's data with a standard Java streaming API. Moreover, for each part
+ * of the file you can calculate an affinity and process file's content on corresponding
+ * nodes to escape unnecessary networking.
+ * <p/>
+ * This API is fully thread-safe and you can use it from several threads.
+ * <h1 class="header">IGFS Configuration</h1>
+ * The simplest way to run a Ignite node with configured file system is to pass
+ * special configuration file included in Ignite distribution to {@code ignite.sh} or
+ * {@code ignite.bat} scripts, like this: {@code ignite.sh config/hadoop/default-config.xml}
+ * <p>
+ * {@code IGFS} can be started as a data node or as a client node. Data node is responsible for
+ * caching data, while client node is responsible for basic file system operations and accessing
+ * data nodes remotely. When used as Hadoop file system, clients nodes usually started together
+ * with {@code job-submitter} or {@code job-scheduler} processes, while data nodes are usually
+ * started together with Hadoop {@code task-tracker} processes.
+ * <h1 class="header">Integration With Hadoop</h1>
+ * In addition to direct file system API, {@code IGFS} can be integrated with {@code Hadoop} by
+ * plugging in as {@code Hadoop FileSystem}. Refer to
+ * {@code org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem} or
+ * {@code org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem} for more information.
+ * <p>
+ * <b>NOTE:</b> integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition.
+ */
+public interface IgniteFileSystem extends IgniteAsyncSupport {
+ /** IGFS scheme name. */
+ public static final String IGFS_SCHEME = "igfs";
+
+ /**
+ * Gets IGFS name.
+ *
+ * @return IGFS name, or {@code null} for default file system.
+ */
+ public String name();
+
+ /**
+ * Gets IGFS configuration.
+ *
+ * @return IGFS configuration.
+ */
+ public FileSystemConfiguration configuration();
+
+ /**
+ * Gets summary (total number of files, total number of directories and total length)
+ * for a given path.
+ *
+ * @param path Path to get information for.
+ * @return Summary object.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path is not found.
+ * @throws IgniteException If failed.
+ */
+ public IgfsPathSummary summary(IgfsPath path) throws IgniteException;
+
+ /**
+ * Opens a file for reading.
+ *
+ * @param path File path to read.
+ * @return File input stream to read data from.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
+ */
+ public IgfsInputStream open(IgfsPath path) throws IgniteException;
+
+ /**
+ * Opens a file for reading.
+ *
+ * @param path File path to read.
+ * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
+ * @return File input stream to read data from.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
+ */
+ public IgfsInputStream open(IgfsPath path, int bufSize) throws IgniteException;
+
+ /**
+ * Opens a file for reading.
+ *
+ * @param path File path to read.
+ * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
+ * @param seqReadsBeforePrefetch Amount of sequential reads before prefetch is started.
+ * @return File input stream to read data from.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
+ */
+ public IgfsInputStream open(IgfsPath path, int bufSize, int seqReadsBeforePrefetch) throws IgniteException;
+
+ /**
+ * Creates a file and opens it for writing.
+ *
+ * @param path File path to create.
+ * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
+ * @return File output stream to write data to.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsOutputStream create(IgfsPath path, boolean overwrite) throws IgniteException;
+
+ /**
+ * Creates a file and opens it for writing.
+ *
+ * @param path File path to create.
+ * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
+ * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
+ * @param replication Replication factor.
+ * @param blockSize Block size.
+ * @param props File properties to set.
+ * @return File output stream to write data to.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
+ long blockSize, @Nullable Map<String, String> props) throws IgniteException;
+
+ /**
+ * Creates a file and opens it for writing.
+ *
+ * @param path File path to create.
+ * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
+ * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
+ * @param affKey Affinity key used to store file blocks. If not {@code null}, the whole file will be
+ * stored on node where {@code affKey} resides.
+ * @param replication Replication factor.
+ * @param blockSize Block size.
+ * @param props File properties to set.
+ * @return File output stream to write data to.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite,
+ @Nullable IgniteUuid affKey, int replication, long blockSize, @Nullable Map<String, String> props)
+ throws IgniteException;
+
+ /**
+ * Opens an output stream to an existing file for appending data.
+ *
+ * @param path File path to append.
+ * @param create Create file if it doesn't exist yet.
+ * @return File output stream to append data to.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
+ */
+ public IgfsOutputStream append(IgfsPath path, boolean create) throws IgniteException;
+
+ /**
+ * Opens an output stream to an existing file for appending data.
+ *
+ * @param path File path to append.
+ * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
+ * @param create Create file if it doesn't exist yet.
+ * @param props File properties to set only in case it file was just created.
+ * @return File output stream to append data to.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
+ */
+ public IgfsOutputStream append(IgfsPath path, int bufSize, boolean create, @Nullable Map<String, String> props)
+ throws IgniteException;
+
+ /**
+ * Sets last access time and last modification time for a given path. If argument is {@code null},
+ * corresponding time will not be changed.
+ *
+ * @param path Path to update.
+ * @param accessTime Optional last access time to set. Value {@code -1} does not update access time.
+ * @param modificationTime Optional last modification time to set. Value {@code -1} does not update
+ * modification time.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If target was not found.
+ * @throws IgniteException If error occurred.
+ */
+ public void setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteException;
+
+ /**
+ * Gets affinity block locations for data blocks of the file, i.e. the nodes, on which the blocks
+ * are stored.
+ *
+ * @param path File path to get affinity for.
+ * @param start Position in the file to start affinity resolution from.
+ * @param len Size of data in the file to resolve affinity for.
+ * @return Affinity block locations.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
+ */
+ public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) throws IgniteException;
+
+ /**
+ * Get affinity block locations for data blocks of the file. In case {@code maxLen} parameter is set and
+ * particular block location length is greater than this value, block locations will be split into smaller
+ * chunks.
+ *
+ * @param path File path to get affinity for.
+ * @param start Position in the file to start affinity resolution from.
+ * @param len Size of data in the file to resolve affinity for.
+ * @param maxLen Maximum length of a single returned block location length.
+ * @return Affinity block locations.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
+ */
+ public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len, long maxLen)
+ throws IgniteException;
+
+ /**
+ * Gets metrics snapshot for this file system.
+ *
+ * @return Metrics.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsMetrics metrics() throws IgniteException;
+
+ /**
+ * Resets metrics for this file system.
+ *
+ * @throws IgniteException In case of error.
+ */
+ public void resetMetrics() throws IgniteException;
+
+ /**
+ * Determines size of the file denoted by provided path. In case if path is a directory, then
+ * total size of all containing entries will be calculated recursively.
+ *
+ * @param path File system path.
+ * @return Total size.
+ * @throws IgniteException In case of error.
+ */
+ public long size(IgfsPath path) throws IgniteException;
+
+ /**
+ * Formats the file system removing all existing entries from it.
+ * <p>
+ * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
+ *
+ * @throws IgniteException In case format has failed.
+ */
+ @IgniteAsyncSupported
+ public void format() throws IgniteException;
+
+ /**
+ * Executes IGFS task.
+ * <p>
+ * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
+ *
+ * @param task Task to execute.
+ * @param rslvr Optional resolver to control split boundaries.
+ * @param paths Collection of paths to be processed within this task.
+ * @param arg Optional task argument.
+ * @return Task result.
+ * @throws IgniteException If execution failed.
+ */
+ @IgniteAsyncSupported
+ public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
+ Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException;
+
+ /**
+ * Executes IGFS task with overridden maximum range length (see
+ * {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()} for more information).
+ * <p>
+ * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
+ *
+ * @param task Task to execute.
+ * @param rslvr Optional resolver to control split boundaries.
+ * @param paths Collection of paths to be processed within this task.
+ * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will
+ * be ignored. Otherwise an exception will be thrown.
+ * @param maxRangeLen Optional maximum range length. If {@code 0}, then by default all consecutive
+ * IGFS blocks will be included.
+ * @param arg Optional task argument.
+ * @return Task result.
+ * @throws IgniteException If execution failed.
+ */
+ @IgniteAsyncSupported
+ public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
+ Collection<IgfsPath> paths, boolean skipNonExistentFiles, long maxRangeLen, @Nullable T arg)
+ throws IgniteException;
+
+ /**
+ * Executes IGFS task.
+ * <p>
+ * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
+ *
+ * @param taskCls Task class to execute.
+ * @param rslvr Optional resolver to control split boundaries.
+ * @param paths Collection of paths to be processed within this task.
+ * @param arg Optional task argument.
+ * @return Task result.
+ * @throws IgniteException If execution failed.
+ */
+ @IgniteAsyncSupported
+ public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
+ @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException;
+
+ /**
+ * Executes IGFS task with overridden maximum range length (see
+ * {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()} for more information).
+ * <p>
+ * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
+ *
+ * @param taskCls Task class to execute.
+ * @param rslvr Optional resolver to control split boundaries.
+ * @param paths Collection of paths to be processed within this task.
+ * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will
+ * be ignored. Otherwise an exception will be thrown.
+ * @param maxRangeLen Maximum range length.
+ * @param arg Optional task argument.
+ * @return Task result.
+ * @throws IgniteException If execution failed.
+ */
+ @IgniteAsyncSupported
+ public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
+ @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles,
+ long maxRangeLen, @Nullable T arg) throws IgniteException;
+
+ /**
+ * Checks if the specified path exists in the file system.
+ *
+ * @param path Path to check for existence in the file system.
+ * @return {@code True} if such file exists, otherwise - {@code false}.
+ * @throws IgniteException In case of error.
+ */
+ public boolean exists(IgfsPath path);
+
+ /**
+ * Updates file information for the specified path. Existent properties, not listed in the passed collection,
+ * will not be affected. Other properties will be added or overwritten. Passed properties with {@code null} values
+ * will be removed from the stored properties or ignored if they don't exist in the file info.
+ * <p>
+ * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties will be propagated
+ * to the secondary file system:
+ * <ul>
+ * <li>{@code usrName} - file owner name;</li>
+ * <li>{@code grpName} - file owner group;</li>
+ * <li>{@code permission} - Unix-style string representing file permissions.</li>
+ * </ul>
+ *
+ * @param path File path to set properties for.
+ * @param props Properties to update.
+ * @return File information for specified path or {@code null} if such path does not exist.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteException;
+
+ /**
+ * Renames/moves a file.
+ * <p>
+ * You are free to rename/move data files as you wish, but directories can be only renamed.
+ * You cannot move the directory between different parent directories.
+ * <p>
+ * Examples:
+ * <ul>
+ * <li>"/work/file.txt" => "/home/project/Presentation Scenario.txt"</li>
+ * <li>"/work" => "/work-2012.bkp"</li>
+ * <li>"/work" => "<strike>/backups/work</strike>" - such operation is restricted for directories.</li>
+ * </ul>
+ *
+ * @param src Source file path to rename.
+ * @param dest Destination file path. If destination path is a directory, then source file will be placed
+ * into destination directory with original name.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If source file doesn't exist.
+ */
+ public void rename(IgfsPath src, IgfsPath dest) throws IgniteException;
+
+ /**
+ * Deletes file.
+ *
+ * @param path File path to delete.
+ * @param recursive Delete non-empty directories recursively.
+ * @return {@code True} in case of success, {@code false} otherwise.
+ * @throws IgniteException In case of error.
+ */
+ public boolean delete(IgfsPath path, boolean recursive) throws IgniteException;
+
+ /**
+ * Creates directories under specified path.
+ *
+ * @param path Path of directories chain to create.
+ * @throws IgniteException In case of error.
+ */
+ public void mkdirs(IgfsPath path) throws IgniteException;
+
+ /**
+ * Creates directories under specified path with the specified properties.
+ *
+ * @param path Path of directories chain to create.
+ * @param props Metadata properties to set on created directories.
+ * @throws IgniteException In case of error.
+ */
+ public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) throws IgniteException;
+
+ /**
+ * Lists file paths under the specified path.
+ *
+ * @param path Path to list files under.
+ * @return List of files under the specified path.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If path doesn't exist.
+ */
+ public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteException;
+
+ /**
+ * Lists files under the specified path.
+ *
+ * @param path Path to list files under.
+ * @return List of files under the specified path.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If path doesn't exist.
+ */
+ public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteException;
+
+ /**
+ * Gets file information for the specified path.
+ *
+ * @param path Path to get information for.
+ * @return File information for specified path or {@code null} if such path does not exist.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsFile info(IgfsPath path) throws IgniteException;
+
+ /**
+ * Gets used space in bytes.
+ *
+ * @return Used space in bytes.
+ * @throws IgniteException In case of error.
+ */
+ public long usedSpaceSize() throws IgniteException;
+
+ /** {@inheritDoc} */
+ @Override public IgniteFileSystem withAsync();
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/IgniteFs.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteFs.java b/modules/core/src/main/java/org/apache/ignite/IgniteFs.java
deleted file mode 100644
index dc11973..0000000
--- a/modules/core/src/main/java/org/apache/ignite/IgniteFs.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite;
-
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.igfs.mapreduce.*;
-import org.apache.ignite.lang.*;
-import org.jetbrains.annotations.*;
-
-import java.util.*;
-
-/**
- * <b>IG</b>nite <b>F</b>ile <b>S</b>ystem API. It provides a typical file system "view" on a particular cache:
- * <ul>
- * <li>list directories or get information for a single path</li>
- * <li>create/move/delete files or directories</li>
- * <li>write/read data streams into/from files</li>
- * </ul>
- * The data of each file is split on separate data blocks and stored in the cache.
- * You can access file's data with a standard Java streaming API. Moreover, for each part
- * of the file you can calculate an affinity and process file's content on corresponding
- * nodes to escape unnecessary networking.
- * <p/>
- * This API is fully thread-safe and you can use it from several threads.
- * <h1 class="header">IGFS Configuration</h1>
- * The simplest way to run a Ignite node with configured file system is to pass
- * special configuration file included in Ignite distribution to {@code ignite.sh} or
- * {@code ignite.bat} scripts, like this: {@code ignite.sh config/hadoop/default-config.xml}
- * <p>
- * {@code IGFS} can be started as a data node or as a client node. Data node is responsible for
- * caching data, while client node is responsible for basic file system operations and accessing
- * data nodes remotely. When used as Hadoop file system, clients nodes usually started together
- * with {@code job-submitter} or {@code job-scheduler} processes, while data nodes are usually
- * started together with Hadoop {@code task-tracker} processes.
- * <h1 class="header">Integration With Hadoop</h1>
- * In addition to direct file system API, {@code IGFS} can be integrated with {@code Hadoop} by
- * plugging in as {@code Hadoop FileSystem}. Refer to
- * {@code org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem} or
- * {@code org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem} for more information.
- * <p>
- * <b>NOTE:</b> integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition.
- */
-public interface IgniteFs extends Igfs, IgniteAsyncSupport {
- /** IGFS scheme name. */
- public static final String IGFS_SCHEME = "igfs";
-
- /** File property: prefer writes to local node. */
- public static final String PROP_PREFER_LOCAL_WRITES = "locWrite";
-
- /**
- * Gets IGFS name.
- *
- * @return IGFS name, or {@code null} for default file system.
- */
- public String name();
-
- /**
- * Gets IGFS configuration.
- *
- * @return IGFS configuration.
- */
- public IgfsConfiguration configuration();
-
- /**
- * Gets summary (total number of files, total number of directories and total length)
- * for a given path.
- *
- * @param path Path to get information for.
- * @return Summary object.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path is not found.
- * @throws IgniteException If failed.
- */
- public IgfsPathSummary summary(IgfsPath path) throws IgniteException;
-
- /**
- * Opens a file for reading.
- *
- * @param path File path to read.
- * @return File input stream to read data from.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public IgfsInputStream open(IgfsPath path) throws IgniteException;
-
- /**
- * Opens a file for reading.
- *
- * @param path File path to read.
- * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
- * @return File input stream to read data from.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- @Override public IgfsInputStream open(IgfsPath path, int bufSize) throws IgniteException;
-
- /**
- * Opens a file for reading.
- *
- * @param path File path to read.
- * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
- * @param seqReadsBeforePrefetch Amount of sequential reads before prefetch is started.
- * @return File input stream to read data from.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public IgfsInputStream open(IgfsPath path, int bufSize, int seqReadsBeforePrefetch) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- @Override public IgfsOutputStream create(IgfsPath path, boolean overwrite) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @param replication Replication factor.
- * @param blockSize Block size.
- * @param props File properties to set.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- @Override public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
- long blockSize, @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @param affKey Affinity key used to store file blocks. If not {@code null}, the whole file will be
- * stored on node where {@code affKey} resides.
- * @param replication Replication factor.
- * @param blockSize Block size.
- * @param props File properties to set.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite,
- @Nullable IgniteUuid affKey, int replication, long blockSize, @Nullable Map<String, String> props)
- throws IgniteException;
-
- /**
- * Opens an output stream to an existing file for appending data.
- *
- * @param path File path to append.
- * @param create Create file if it doesn't exist yet.
- * @return File output stream to append data to.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
- */
- public IgfsOutputStream append(IgfsPath path, boolean create) throws IgniteException;
-
- /**
- * Opens an output stream to an existing file for appending data.
- *
- * @param path File path to append.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param create Create file if it doesn't exist yet.
- * @param props File properties to set only in case it file was just created.
- * @return File output stream to append data to.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
- */
- @Override public IgfsOutputStream append(IgfsPath path, int bufSize, boolean create,
- @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Sets last access time and last modification time for a given path. If argument is {@code null},
- * corresponding time will not be changed.
- *
- * @param path Path to update.
- * @param accessTime Optional last access time to set. Value {@code -1} does not update access time.
- * @param modificationTime Optional last modification time to set. Value {@code -1} does not update
- * modification time.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If target was not found.
- * @throws IgniteException If error occurred.
- */
- public void setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteException;
-
- /**
- * Gets affinity block locations for data blocks of the file, i.e. the nodes, on which the blocks
- * are stored.
- *
- * @param path File path to get affinity for.
- * @param start Position in the file to start affinity resolution from.
- * @param len Size of data in the file to resolve affinity for.
- * @return Affinity block locations.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) throws IgniteException;
-
- /**
- * Get affinity block locations for data blocks of the file. In case {@code maxLen} parameter is set and
- * particular block location length is greater than this value, block locations will be split into smaller
- * chunks.
- *
- * @param path File path to get affinity for.
- * @param start Position in the file to start affinity resolution from.
- * @param len Size of data in the file to resolve affinity for.
- * @param maxLen Maximum length of a single returned block location length.
- * @return Affinity block locations.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len, long maxLen)
- throws IgniteException;
-
- /**
- * Gets metrics snapshot for this file system.
- *
- * @return Metrics.
- * @throws IgniteException In case of error.
- */
- public IgfsMetrics metrics() throws IgniteException;
-
- /**
- * Resets metrics for this file system.
- *
- * @throws IgniteException In case of error.
- */
- public void resetMetrics() throws IgniteException;
-
- /**
- * Determines size of the file denoted by provided path. In case if path is a directory, then
- * total size of all containing entries will be calculated recursively.
- *
- * @param path File system path.
- * @return Total size.
- * @throws IgniteException In case of error.
- */
- public long size(IgfsPath path) throws IgniteException;
-
- /**
- * Formats the file system removing all existing entries from it.
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @throws IgniteException In case format has failed.
- */
- @IgniteAsyncSupported
- public void format() throws IgniteException;
-
- /**
- * Executes IGFS task.
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @param task Task to execute.
- * @param rslvr Optional resolver to control split boundaries.
- * @param paths Collection of paths to be processed within this task.
- * @param arg Optional task argument.
- * @return Task result.
- * @throws IgniteException If execution failed.
- */
- @IgniteAsyncSupported
- public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
- Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException;
-
- /**
- * Executes IGFS task with overridden maximum range length (see
- * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information).
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @param task Task to execute.
- * @param rslvr Optional resolver to control split boundaries.
- * @param paths Collection of paths to be processed within this task.
- * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will
- * be ignored. Otherwise an exception will be thrown.
- * @param maxRangeLen Optional maximum range length. If {@code 0}, then by default all consecutive
- * IGFS blocks will be included.
- * @param arg Optional task argument.
- * @return Task result.
- * @throws IgniteException If execution failed.
- */
- @IgniteAsyncSupported
- public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
- Collection<IgfsPath> paths, boolean skipNonExistentFiles, long maxRangeLen, @Nullable T arg)
- throws IgniteException;
-
- /**
- * Executes IGFS task.
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @param taskCls Task class to execute.
- * @param rslvr Optional resolver to control split boundaries.
- * @param paths Collection of paths to be processed within this task.
- * @param arg Optional task argument.
- * @return Task result.
- * @throws IgniteException If execution failed.
- */
- @IgniteAsyncSupported
- public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
- @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException;
-
- /**
- * Executes IGFS task with overridden maximum range length (see
- * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information).
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @param taskCls Task class to execute.
- * @param rslvr Optional resolver to control split boundaries.
- * @param paths Collection of paths to be processed within this task.
- * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will
- * be ignored. Otherwise an exception will be thrown.
- * @param maxRangeLen Maximum range length.
- * @param arg Optional task argument.
- * @return Task result.
- * @throws IgniteException If execution failed.
- */
- @IgniteAsyncSupported
- public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
- @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles,
- long maxRangeLen, @Nullable T arg) throws IgniteException;
-
- /** {@inheritDoc} */
- @Override public IgniteFs withAsync();
-}
[09/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
new file mode 100644
index 0000000..be25c61
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
@@ -0,0 +1,2366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.*;
+import org.apache.ignite.*;
+import org.apache.ignite.cache.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.hadoop.fs.*;
+import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem;
+import org.apache.ignite.internal.processors.hadoop.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
+import org.apache.ignite.spi.communication.*;
+import org.apache.ignite.spi.communication.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+import org.apache.ignite.testframework.*;
+import org.jdk8.backport.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.lang.reflect.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.events.EventType.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * Test hadoop file system implementation.
+ */
+@SuppressWarnings("all")
+public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonAbstractTest {
+ /** Primary file system authority. */
+ private static final String PRIMARY_AUTHORITY = "igfs:grid0@";
+
+ /** Primary file systme URI. */
+ private static final String PRIMARY_URI = "igfs://" + PRIMARY_AUTHORITY + "/";
+
+ /** Secondary file system authority. */
+ private static final String SECONDARY_AUTHORITY = "igfs_secondary:grid_secondary@127.0.0.1:11500";
+
+ /** Secondary file systme URI. */
+ private static final String SECONDARY_URI = "igfs://" + SECONDARY_AUTHORITY + "/";
+
+ /** Secondary file system configuration path. */
+ private static final String SECONDARY_CFG_PATH = "/work/core-site-test.xml";
+
+ /** Secondary endpoint configuration. */
+ protected static final Map<String, String> SECONDARY_ENDPOINT_CFG = new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", "11500");
+ }};
+
+ /** Group size. */
+ public static final int GRP_SIZE = 128;
+
+ /** Path to the default hadoop configuration. */
+ public static final String HADOOP_FS_CFG = "examples/config/filesystem/core-site.xml";
+
+ /** Thread count for multithreaded tests. */
+ private static final int THREAD_CNT = 8;
+
+ /** IP finder. */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** Barrier for multithreaded tests. */
+ private static CyclicBarrier barrier;
+
+ /** File system. */
+ private static FileSystem fs;
+
+ /** Default IGFS mode. */
+ protected final IgfsMode mode;
+
+ /** Skip embedded mode flag. */
+ private final boolean skipEmbed;
+
+ /** Skip local shmem flag. */
+ private final boolean skipLocShmem;
+
+ /** Endpoint. */
+ private final String endpoint;
+
+ /** Primary file system URI. */
+ protected URI primaryFsUri;
+
+ /** Primary file system configuration. */
+ protected Configuration primaryFsCfg;
+
+ /** File statuses comparator. */
+ private static final Comparator<FileStatus> STATUS_COMPARATOR = new Comparator<FileStatus>() {
+ @SuppressWarnings("deprecation")
+ @Override public int compare(FileStatus o1, FileStatus o2) {
+ if (o1 == null || o2 == null)
+ return o1 == o2 ? 0 : o1 == null ? -1 : 1;
+
+ return o1.isDir() == o2.isDir() ? o1.getPath().compareTo(o2.getPath()) : o1.isDir() ? -1 : 1;
+ }
+ };
+
+ /**
+ * Constructor.
+ *
+ * @param mode Default IGFS mode.
+ * @param skipEmbed Whether to skip embedded mode.
+ * @param skipLocShmem Whether to skip local shmem mode.
+ * @param skipLocTcp Whether to skip local TCP mode.
+ */
+ protected IgniteHadoopFileSystemAbstractSelfTest(IgfsMode mode, boolean skipEmbed, boolean skipLocShmem) {
+ this.mode = mode;
+ this.skipEmbed = skipEmbed;
+ this.skipLocShmem = skipLocShmem;
+
+ endpoint = skipLocShmem ? "127.0.0.1:10500" : "shmem:10500";
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ Configuration secondaryConf = configuration(SECONDARY_AUTHORITY, true, true);
+
+ secondaryConf.setInt("fs.igfs.block.size", 1024);
+
+ String path = U.getIgniteHome() + SECONDARY_CFG_PATH;
+
+ File file = new File(path);
+
+ try (FileOutputStream fos = new FileOutputStream(file)) {
+ secondaryConf.writeXml(fos);
+ }
+
+ startNodes();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 10 * 60 * 1000;
+ }
+
+ /**
+ * Starts the nodes for this test.
+ *
+ * @throws Exception If failed.
+ */
+ private void startNodes() throws Exception {
+ if (mode != PRIMARY) {
+ // Start secondary IGFS.
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName("igfs_secondary");
+ igfsCfg.setIpcEndpointConfiguration(SECONDARY_ENDPOINT_CFG);
+ igfsCfg.setBlockSize(512 * 1024);
+ igfsCfg.setPrefetchBlocks(1);
+
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("grid_secondary");
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
+ cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
+
+ cfg.setCommunicationSpi(communicationSpi());
+
+ G.start(cfg);
+ }
+
+ startGrids(4);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ G.stopAll(true);
+
+ String path = U.getIgniteHome() + SECONDARY_CFG_PATH;
+
+ new File(path).delete();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ primaryFsUri = new URI(PRIMARY_URI);
+
+ primaryFsCfg = configuration(PRIMARY_AUTHORITY, skipEmbed, skipLocShmem);
+
+ fs = FileSystem.get(primaryFsUri, primaryFsCfg);
+
+ barrier = new CyclicBarrier(THREAD_CNT);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ try {
+ fs.delete(new Path("/"), true);
+ }
+ catch (Exception ignore) {
+ // No-op.
+ }
+
+ U.closeQuiet(fs);
+ }
+
+ /**
+ * Get primary IPC endpoint configuration.
+ *
+ * @param gridName Grid name.
+ * @return IPC primary endpoint configuration.
+ */
+ protected abstract Map<String, String> primaryIpcEndpointConfiguration(String gridName);
+
+ /** {@inheritDoc} */
+ @Override public String getTestGridName() {
+ return "grid";
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(cacheConfiguration(gridName));
+ cfg.setFileSystemConfiguration(igfsConfiguration(gridName));
+ cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
+ cfg.setCommunicationSpi(communicationSpi());
+
+ return cfg;
+ }
+
+ /**
+ * Gets cache configuration.
+ *
+ * @param gridName Grid name.
+ * @return Cache configuration.
+ */
+ protected CacheConfiguration[] cacheConfiguration(String gridName) {
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ return new CacheConfiguration[] {metaCacheCfg, cacheCfg};
+ }
+
+ /**
+ * Gets IGFS configuration.
+ *
+ * @param gridName Grid name.
+ * @return IGFS configuration.
+ */
+ protected FileSystemConfiguration igfsConfiguration(String gridName) throws IgniteCheckedException {
+ FileSystemConfiguration cfg = new FileSystemConfiguration();
+
+ cfg.setDataCacheName("partitioned");
+ cfg.setMetaCacheName("replicated");
+ cfg.setName("igfs");
+ cfg.setPrefetchBlocks(1);
+ cfg.setDefaultMode(mode);
+
+ if (mode != PRIMARY)
+ cfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(SECONDARY_URI, SECONDARY_CFG_PATH));
+
+ cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(gridName));
+
+ cfg.setManagementPort(-1);
+ cfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups.
+
+ return cfg;
+ }
+
+ /** @return Communication SPI. */
+ private CommunicationSpi communicationSpi() {
+ TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
+
+ commSpi.setSharedMemoryPort(-1);
+
+ return commSpi;
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetUriIfFSIsNotInitialized() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return new IgniteHadoopFileSystem().getUri();
+ }
+ }, IllegalStateException.class, "URI is null (was IgniteHadoopFileSystem properly initialized?).");
+ }
+
+ /** @throws Exception If failed. */
+ @SuppressWarnings("NullableProblems")
+ public void testInitializeCheckParametersNameIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ new IgniteHadoopFileSystem().initialize(null, new Configuration());
+
+ return null;
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: name");
+ }
+
+ /** @throws Exception If failed. */
+ @SuppressWarnings("NullableProblems")
+ public void testInitializeCheckParametersCfgIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ new IgniteHadoopFileSystem().initialize(new URI(""), null);
+
+ return null;
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: cfg");
+ }
+
+ /** @throws Exception If failed. */
+ public void testInitialize() throws Exception {
+ final IgniteHadoopFileSystem fs = new IgniteHadoopFileSystem();
+
+ fs.initialize(primaryFsUri, primaryFsCfg);
+
+ // Check repeatable initialization.
+ try {
+ fs.initialize(primaryFsUri, primaryFsCfg);
+
+ fail();
+ }
+ catch (IOException e) {
+ assertTrue(e.getMessage().contains("File system is already initialized"));
+ }
+
+ assertEquals(primaryFsUri, fs.getUri());
+
+ assertEquals(0, fs.getUsed());
+
+ fs.close();
+ }
+
+ /**
+ * Test how IPC cache map works.
+ *
+ * @throws Exception If failed.
+ */
+ public void testIpcCache() throws Exception {
+ HadoopIgfsEx hadoop = GridTestUtils.getFieldValue(fs, "rmtClient", "delegateRef", "value", "hadoop");
+
+ if (hadoop instanceof HadoopIgfsOutProc) {
+ FileSystem fsOther = null;
+
+ try {
+ Field field = HadoopIgfsIpcIo.class.getDeclaredField("ipcCache");
+
+ field.setAccessible(true);
+
+ Map<String, HadoopIgfsIpcIo> cache = (Map<String, HadoopIgfsIpcIo>)field.get(null);
+
+ Configuration cfg = configuration(PRIMARY_AUTHORITY, skipEmbed, skipLocShmem);
+
+ // we disable caching in order to obtain new FileSystem instance.
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ // Initial cache size.
+ int initSize = cache.size();
+
+ // Ensure that when IO is used by multiple file systems and one of them is closed, IO is not stopped.
+ fsOther = FileSystem.get(new URI(PRIMARY_URI), cfg);
+
+ assert fs != fsOther;
+
+ assertEquals(initSize, cache.size());
+
+ fsOther.close();
+
+ assertEquals(initSize, cache.size());
+
+ Field stopField = HadoopIgfsIpcIo.class.getDeclaredField("stopping");
+
+ stopField.setAccessible(true);
+
+ HadoopIgfsIpcIo io = null;
+
+ for (Map.Entry<String, HadoopIgfsIpcIo> ioEntry : cache.entrySet()) {
+ if (endpoint.contains(ioEntry.getKey())) {
+ io = ioEntry.getValue();
+
+ break;
+ }
+ }
+
+ assert io != null;
+
+ assert !(Boolean)stopField.get(io);
+
+ // Ensure that IO is stopped when nobody else is need it.
+ fs.close();
+
+ assertEquals(initSize - 1, cache.size());
+
+ assert (Boolean)stopField.get(io);
+ }
+ finally {
+ U.closeQuiet(fsOther);
+ }
+ }
+ }
+
+ /** @throws Exception If failed. */
+ public void testCloseIfNotInitialized() throws Exception {
+ final FileSystem fs = new IgniteHadoopFileSystem();
+
+ // Check close makes nothing harmful.
+ fs.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testClose() throws Exception {
+ final Path path = new Path("dir");
+
+ fs.close();
+
+ // Check double close makes nothing harmful.
+ fs.close();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Nullable @Override public Object call() throws Exception {
+ fs.initialize(primaryFsUri, primaryFsCfg);
+
+ return null;
+ }
+ }, IOException.class, "File system is stopped.");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Nullable @Override public Object call() throws Exception {
+ fs.setPermission(path, FsPermission.createImmutable((short)777));
+
+ return null;
+ }
+ }, IOException.class, "File system is stopped.");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Nullable @Override public Object call() throws Exception {
+ fs.setOwner(path, "user", "group");
+
+ return null;
+ }
+ }, IOException.class, "File system is stopped.");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.open(path, 256);
+ }
+ }, IOException.class, "File system is stopped.");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.create(path);
+ }
+ }, IOException.class, "File system is stopped.");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.append(path);
+ }
+ }, IOException.class, "File system is stopped.");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.rename(path, new Path("newDir"));
+ }
+ }, IOException.class, "File system is stopped.");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.delete(path, true);
+ }
+ }, IOException.class, "File system is stopped.");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.listStatus(path);
+ }
+ }, IOException.class, "File system is stopped.");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.mkdirs(path);
+ }
+ }, IOException.class, "File system is stopped.");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.getFileStatus(path);
+ }
+ }, IOException.class, "File system is stopped.");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.getFileBlockLocations(new FileStatus(1L, false, 1, 1L, 1L, new Path("path")), 0L, 256L);
+ }
+ }, IOException.class, "File system is stopped.");
+ }
+
+ /** @throws Exception If failed. */
+ public void testCreateCheckParameters() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.create(null);
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: f");
+ }
+
+ /** @throws Exception If failed. */
+ @SuppressWarnings("deprecation")
+ public void testCreateBase() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
+ Path file = new Path(dir, "someFile");
+
+ assertPathDoesNotExist(fs, file);
+
+ FsPermission fsPerm = new FsPermission((short)644);
+
+ FSDataOutputStream os = fs.create(file, fsPerm, false, 1, (short)1, 1L, null);
+
+ // Try to write something in file.
+ os.write("abc".getBytes());
+
+ os.close();
+
+ // Check file status.
+ FileStatus fileStatus = fs.getFileStatus(file);
+
+ assertFalse(fileStatus.isDir());
+ assertEquals(file, fileStatus.getPath());
+ assertEquals(fsPerm, fileStatus.getPermission());
+ }
+
+ /** @throws Exception If failed. */
+ @SuppressWarnings("deprecation")
+ public void testCreateCheckOverwrite() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
+ final Path file = new Path(dir, "someFile");
+
+ FSDataOutputStream out = fs.create(file, FsPermission.getDefault(), false, 64 * 1024,
+ fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
+
+ out.close();
+
+ // Check intermediate directory permissions.
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir).getPermission());
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir.getParent()).getPermission());
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir.getParent().getParent()).getPermission());
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.create(file, FsPermission.getDefault(), false, 1024, (short)1, 2048, null);
+ }
+ }, PathExistsException.class, null);
+
+ // Overwrite should be successful.
+ FSDataOutputStream out1 = fs.create(file, true);
+
+ out1.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteIfNoSuchPath() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
+
+ assertPathDoesNotExist(fs, dir);
+
+ assertFalse(fs.delete(dir, true));
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteSuccessfulIfPathIsOpenedToRead() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "myFile");
+
+ FSDataOutputStream os = fs.create(file, false, 128);
+
+ final int cnt = 5 * FileSystemConfiguration.DFLT_BLOCK_SIZE; // Write 5 blocks.
+
+ for (int i = 0; i < cnt; i++)
+ os.writeInt(i);
+
+ os.close();
+
+ final FSDataInputStream is = fs.open(file, -1);
+
+ for (int i = 0; i < cnt / 2; i++)
+ assertEquals(i, is.readInt());
+
+ assert fs.delete(file, false);
+
+ assert !fs.exists(file);
+
+ is.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteIfFilePathExists() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "myFile");
+
+ FSDataOutputStream os = fs.create(file);
+
+ os.close();
+
+ assertTrue(fs.delete(file, false));
+
+ assertPathDoesNotExist(fs, file);
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteIfDirectoryPathExists() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
+
+ FSDataOutputStream os = fs.create(dir);
+
+ os.close();
+
+ assertTrue(fs.delete(dir, false));
+
+ assertPathDoesNotExist(fs, dir);
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteFailsIfNonRecursive() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
+
+ fs.create(someDir3).close();
+
+ Path someDir2 = new Path(fsHome, "/someDir1/someDir2");
+
+ assertFalse(fs.delete(someDir2, false));
+
+ assertPathExists(fs, someDir2);
+ assertPathExists(fs, someDir3);
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteRecursively() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
+
+ FSDataOutputStream os = fs.create(someDir3);
+
+ os.close();
+
+ Path someDir2 = new Path(fsHome, "/someDir1/someDir2");
+
+ assertTrue(fs.delete(someDir2, true));
+
+ assertPathDoesNotExist(fs, someDir2);
+ assertPathDoesNotExist(fs, someDir3);
+ }
+
+ /** @throws Exception If failed. */
+ public void testDeleteRecursivelyFromRoot() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
+
+ FSDataOutputStream os = fs.create(someDir3);
+
+ os.close();
+
+ Path root = new Path(fsHome, "/");
+
+ assertTrue(fs.delete(root, true));
+
+ assertPathDoesNotExist(fs, someDir3);
+ assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));
+ assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1"));
+ assertPathExists(fs, root);
+ }
+
+ /** @throws Exception If failed. */
+ @SuppressWarnings("deprecation")
+ public void testSetPermissionCheckDefaultPermission() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file, FsPermission.getDefault(), false, 64 * 1024,
+ fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
+
+ os.close();
+
+ fs.setPermission(file, null);
+
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(file.getParent()).getPermission());
+ }
+
+ /** @throws Exception If failed. */
+ @SuppressWarnings("deprecation")
+ public void testSetPermissionCheckNonRecursiveness() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file, FsPermission.getDefault(), false, 64 * 1024,
+ fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
+
+ os.close();
+
+ Path tmpDir = new Path(fsHome, "/tmp");
+
+ FsPermission perm = new FsPermission((short)123);
+
+ fs.setPermission(tmpDir, perm);
+
+ assertEquals(perm, fs.getFileStatus(tmpDir).getPermission());
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
+ }
+
+ /** @throws Exception If failed. */
+ @SuppressWarnings("OctalInteger")
+ public void testSetPermission() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file);
+
+ os.close();
+
+ for (short i = 0; i <= 0777; i += 7) {
+ FsPermission perm = new FsPermission(i);
+
+ fs.setPermission(file, perm);
+
+ assertEquals(perm, fs.getFileStatus(file).getPermission());
+ }
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetPermissionIfOutputStreamIsNotClosed() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "myFile");
+
+ FsPermission perm = new FsPermission((short)123);
+
+ FSDataOutputStream os = fs.create(file);
+
+ fs.setPermission(file, perm);
+
+ os.close();
+
+ assertEquals(perm, fs.getFileStatus(file).getPermission());
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetOwnerCheckParametersPathIsNull() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file);
+
+ os.close();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.setOwner(null, "aUser", "aGroup");
+
+ return null;
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: p");
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetOwnerCheckParametersUserIsNull() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file);
+
+ os.close();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fs.setOwner(file, null, "aGroup");
+
+ return null;
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: username");
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetOwnerCheckParametersGroupIsNull() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file);
+
+ os.close();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ fs.setOwner(file, "aUser", null);
+
+ return null;
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: grpName");
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetOwner() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file);
+
+ os.close();
+
+ fs.setOwner(file, "aUser", "aGroup");
+
+ assertEquals("aUser", fs.getFileStatus(file).getOwner());
+ assertEquals("aGroup", fs.getFileStatus(file).getGroup());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSetTimes() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "/heartbeat");
+
+ fs.create(file).close();
+
+ FileStatus status = fs.getFileStatus(file);
+
+ assertTrue(status.getAccessTime() > 0);
+ assertTrue(status.getModificationTime() > 0);
+
+ long mtime = System.currentTimeMillis() - 5000;
+ long atime = System.currentTimeMillis() - 4000;
+
+ fs.setTimes(file, mtime, atime);
+
+ status = fs.getFileStatus(file);
+
+ assertEquals(mtime, status.getModificationTime());
+ assertEquals(atime, status.getAccessTime());
+
+ mtime -= 5000;
+
+ fs.setTimes(file, mtime, -1);
+
+ status = fs.getFileStatus(file);
+
+ assertEquals(mtime, status.getModificationTime());
+ assertEquals(atime, status.getAccessTime());
+
+ atime -= 5000;
+
+ fs.setTimes(file, -1, atime);
+
+ status = fs.getFileStatus(file);
+
+ assertEquals(mtime, status.getModificationTime());
+ assertEquals(atime, status.getAccessTime());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSetOwnerIfOutputStreamIsNotClosed() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "myFile");
+
+ FSDataOutputStream os = fs.create(file);
+
+ fs.setOwner(file, "aUser", "aGroup");
+
+ os.close();
+
+ assertEquals("aUser", fs.getFileStatus(file).getOwner());
+ assertEquals("aGroup", fs.getFileStatus(file).getGroup());
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetOwnerCheckNonRecursiveness() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "/tmp/my");
+
+ FSDataOutputStream os = fs.create(file);
+
+ os.close();
+
+ Path tmpDir = new Path(fsHome, "/tmp");
+
+ fs.setOwner(file, "fUser", "fGroup");
+ fs.setOwner(tmpDir, "dUser", "dGroup");
+
+ assertEquals("dUser", fs.getFileStatus(tmpDir).getOwner());
+ assertEquals("dGroup", fs.getFileStatus(tmpDir).getGroup());
+
+ assertEquals("fUser", fs.getFileStatus(file).getOwner());
+ assertEquals("fGroup", fs.getFileStatus(file).getGroup());
+ }
+
+ /** @throws Exception If failed. */
+ public void testOpenCheckParametersPathIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.open(null, 1024);
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: f");
+ }
+
+ /** @throws Exception If failed. */
+ public void testOpenNoSuchPath() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "someFile");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.open(file, 1024);
+ }
+ }, FileNotFoundException.class, null);
+ }
+
+ /** @throws Exception If failed. */
+ public void testOpenIfPathIsAlreadyOpened() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "someFile");
+
+ FSDataOutputStream os = fs.create(file);
+
+ os.close();
+
+ FSDataInputStream is1 = fs.open(file);
+ FSDataInputStream is2 = fs.open(file);
+
+ is1.close();
+ is2.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testOpen() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "someFile");
+
+ int cnt = 2 * 1024;
+
+ FSDataOutputStream out = fs.create(file, true, 1024);
+
+ for (long i = 0; i < cnt; i++)
+ out.writeLong(i);
+
+ out.close();
+
+ FSDataInputStream in = fs.open(file, 1024);
+
+ for (long i = 0; i < cnt; i++)
+ assertEquals(i, in.readLong());
+
+ in.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testAppendCheckParametersPathIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.append(null);
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: f");
+ }
+
+ /** @throws Exception If failed. */
+ public void testAppendIfPathPointsToDirectory() throws Exception {
+ final Path fsHome = new Path(primaryFsUri);
+ final Path dir = new Path(fsHome, "/tmp");
+ Path file = new Path(dir, "my");
+
+ FSDataOutputStream os = fs.create(file);
+
+ os.close();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.append(new Path(fsHome, dir), 1024);
+ }
+ }, IOException.class, null);
+ }
+
+ /** @throws Exception If failed. */
+ public void testAppendIfFileIsAlreadyBeingOpenedToWrite() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "someFile");
+
+ FSDataOutputStream os = fs.create(file);
+
+ os.close();
+
+ FSDataOutputStream appendOs = fs.append(file);
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ return fs.append(file);
+ }
+ }, IOException.class, null);
+
+ appendOs.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testAppend() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path file = new Path(fsHome, "someFile");
+
+ int cnt = 1024;
+
+ FSDataOutputStream out = fs.create(file, true, 1024);
+
+ for (int i = 0; i < cnt; i++)
+ out.writeLong(i);
+
+ out.close();
+
+ out = fs.append(file);
+
+ for (int i = cnt; i < cnt * 2; i++)
+ out.writeLong(i);
+
+ out.close();
+
+ FSDataInputStream in = fs.open(file, 1024);
+
+ for (int i = 0; i < cnt * 2; i++)
+ assertEquals(i, in.readLong());
+
+ in.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameCheckParametersSrcPathIsNull() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "someFile");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.rename(null, file);
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: src");
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameCheckParametersDstPathIsNull() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ final Path file = new Path(fsHome, "someFile");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ return fs.rename(file, null);
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: dst");
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameIfSrcPathDoesNotExist() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path srcFile = new Path(fsHome, "srcFile");
+ Path dstFile = new Path(fsHome, "dstFile");
+
+ assertPathDoesNotExist(fs, srcFile);
+
+ assertFalse(fs.rename(srcFile, dstFile));
+
+ assertPathDoesNotExist(fs, dstFile);
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameIfSrcPathIsAlreadyBeingOpenedToWrite() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path srcFile = new Path(fsHome, "srcFile");
+ Path dstFile = new Path(fsHome, "dstFile");
+
+ FSDataOutputStream os = fs.create(srcFile);
+
+ os.close();
+
+ os = fs.append(srcFile);
+
+ assertTrue(fs.rename(srcFile, dstFile));
+
+ assertPathExists(fs, dstFile);
+
+ String testStr = "Test";
+
+ try {
+ os.writeBytes(testStr);
+ }
+ finally {
+ os.close();
+ }
+
+ try (FSDataInputStream is = fs.open(dstFile)) {
+ byte[] buf = new byte[testStr.getBytes().length];
+
+ is.readFully(buf);
+
+ assertEquals(testStr, new String(buf));
+ }
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameFileIfDstPathExists() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path srcFile = new Path(fsHome, "srcFile");
+ Path dstFile = new Path(fsHome, "dstFile");
+
+ FSDataOutputStream os = fs.create(srcFile);
+
+ os.close();
+
+ os = fs.create(dstFile);
+
+ os.close();
+
+ assertFalse(fs.rename(srcFile, dstFile));
+
+ assertPathExists(fs, srcFile);
+ assertPathExists(fs, dstFile);
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameFile() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path srcFile = new Path(fsHome, "/tmp/srcFile");
+ Path dstFile = new Path(fsHome, "/tmp/dstFile");
+
+ FSDataOutputStream os = fs.create(srcFile);
+
+ os.close();
+
+ assertTrue(fs.rename(srcFile, dstFile));
+
+ assertPathDoesNotExist(fs, srcFile);
+ assertPathExists(fs, dstFile);
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameIfSrcPathIsAlreadyBeingOpenedToRead() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path srcFile = new Path(fsHome, "srcFile");
+ Path dstFile = new Path(fsHome, "dstFile");
+
+ FSDataOutputStream os = fs.create(srcFile);
+
+ int cnt = 1024;
+
+ for (int i = 0; i < cnt; i++)
+ os.writeInt(i);
+
+ os.close();
+
+ FSDataInputStream is = fs.open(srcFile);
+
+ for (int i = 0; i < cnt; i++) {
+ if (i == 100)
+ // Rename file during the read process.
+ assertTrue(fs.rename(srcFile, dstFile));
+
+ assertEquals(i, is.readInt());
+ }
+
+ assertPathDoesNotExist(fs, srcFile);
+ assertPathExists(fs, dstFile);
+
+ os.close();
+ is.close();
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameDirectoryIfDstPathExists() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path srcDir = new Path(fsHome, "/tmp/");
+ Path dstDir = new Path(fsHome, "/tmpNew/");
+
+ FSDataOutputStream os = fs.create(new Path(srcDir, "file1"));
+
+ os.close();
+
+ os = fs.create(new Path(dstDir, "file2"));
+
+ os.close();
+
+ assertTrue("Rename succeeded [srcDir=" + srcDir + ", dstDir=" + dstDir + ']', fs.rename(srcDir, dstDir));
+
+ assertPathExists(fs, dstDir);
+ assertPathExists(fs, new Path(fsHome, "/tmpNew/tmp"));
+ assertPathExists(fs, new Path(fsHome, "/tmpNew/tmp/file1"));
+ }
+
+ /** @throws Exception If failed. */
+ public void testRenameDirectory() throws Exception {
+ Path fsHome = new Path(primaryFsUri);
+ Path dir = new Path(fsHome, "/tmp/");
+ Path newDir = new Path(fsHome, "/tmpNew/");
+
+ FSDataOutputStream os = fs.create(new Path(dir, "myFile"));
+
+ os.close();
+
+ assertTrue("Rename failed [dir=" + dir + ", newDir=" + newDir + ']', fs.rename(dir, newDir));
+
+ assertPathDoesNotExist(fs, dir);
+ assertPathExists(fs, newDir);
+ }
+
+ /** @throws Exception If failed. */
+ public void testListStatusIfPathIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.listStatus((Path)null);
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: f");
+ }
+
+ /** @throws Exception If failed. */
+ public void testListStatusIfPathDoesNotExist() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.listStatus(new Path("/tmp/some/dir"));
+ }
+ }, FileNotFoundException.class, null);
+ }
+
+ /**
+ * Test directory listing.
+ *
+ * @throws Exception If failed.
+ */
+ public void testListStatus() throws Exception {
+ Path igfsHome = new Path(PRIMARY_URI);
+
+ // Test listing of an empty directory.
+ Path dir = new Path(igfsHome, "dir");
+
+ assert fs.mkdirs(dir);
+
+ FileStatus[] list = fs.listStatus(dir);
+
+ assert list.length == 0;
+
+ // Test listing of a not empty directory.
+ Path subDir = new Path(dir, "subDir");
+
+ assert fs.mkdirs(subDir);
+
+ Path file = new Path(dir, "file");
+
+ FSDataOutputStream fos = fs.create(file);
+
+ fos.close();
+
+ list = fs.listStatus(dir);
+
+ assert list.length == 2;
+
+ String listRes1 = list[0].getPath().getName();
+ String listRes2 = list[1].getPath().getName();
+
+ assert "subDir".equals(listRes1) && "file".equals(listRes2) || "subDir".equals(listRes2) &&
+ "file".equals(listRes1);
+
+ // Test listing of a file.
+ list = fs.listStatus(file);
+
+ assert list.length == 1;
+
+ assert "file".equals(list[0].getPath().getName());
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetWorkingDirectoryIfPathIsNull() throws Exception {
+ fs.setWorkingDirectory(null);
+
+ Path file = new Path("file");
+
+ FSDataOutputStream os = fs.create(file);
+ os.close();
+
+ String path = fs.getFileStatus(file).getPath().toString();
+
+ assertTrue(path.endsWith("/user/" + System.getProperty("user.name", "anonymous") + "/file"));
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetWorkingDirectoryIfPathDoesNotExist() throws Exception {
+ // Should not throw any exceptions.
+ fs.setWorkingDirectory(new Path("/someDir"));
+ }
+
+ /** @throws Exception If failed. */
+ public void testSetWorkingDirectory() throws Exception {
+ Path dir = new Path("/tmp/nested/dir");
+ Path file = new Path("file");
+
+ fs.mkdirs(dir);
+
+ fs.setWorkingDirectory(dir);
+
+ FSDataOutputStream os = fs.create(file);
+ os.close();
+
+ String filePath = fs.getFileStatus(new Path(dir, file)).getPath().toString();
+
+ assertTrue(filePath.contains("/tmp/nested/dir/file"));
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetWorkingDirectoryIfDefault() throws Exception {
+ String path = fs.getWorkingDirectory().toString();
+
+ assertTrue(path.endsWith("/user/" + System.getProperty("user.name", "anonymous")));
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetWorkingDirectory() throws Exception {
+ Path dir = new Path("/tmp/some/dir");
+
+ fs.mkdirs(dir);
+
+ fs.setWorkingDirectory(dir);
+
+ String path = fs.getWorkingDirectory().toString();
+
+ assertTrue(path.endsWith("/tmp/some/dir"));
+ }
+
+ /** @throws Exception If failed. */
+ public void testMkdirsIfPathIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.mkdirs(null);
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: f");
+ }
+
+ /** @throws Exception If failed. */
+ public void testMkdirsIfPermissionIsNull() throws Exception {
+ Path dir = new Path("/tmp");
+
+ assertTrue(fs.mkdirs(dir, null));
+
+ assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir).getPermission());
+ }
+
+ /** @throws Exception If failed. */
+ @SuppressWarnings("OctalInteger")
+ public void testMkdirs() throws Exception {
+ Path fsHome = new Path(PRIMARY_URI);
+ Path dir = new Path(fsHome, "/tmp/staging");
+ Path nestedDir = new Path(dir, "nested");
+
+ FsPermission dirPerm = FsPermission.createImmutable((short)0700);
+ FsPermission nestedDirPerm = FsPermission.createImmutable((short)111);
+
+ assertTrue(fs.mkdirs(dir, dirPerm));
+ assertTrue(fs.mkdirs(nestedDir, nestedDirPerm));
+
+ assertEquals(dirPerm, fs.getFileStatus(dir).getPermission());
+ assertEquals(nestedDirPerm, fs.getFileStatus(nestedDir).getPermission());
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetFileStatusIfPathIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.getFileStatus(null);
+ }
+ }, NullPointerException.class, "Ouch! Argument cannot be null: f");
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetFileStatusIfPathDoesNotExist() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.getFileStatus(new Path("someDir"));
+ }
+ }, FileNotFoundException.class, "File not found: someDir");
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetFileBlockLocationsIfFileStatusIsNull() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ // Argument is checked by Hadoop.
+ return fs.getFileBlockLocations((Path)null, 1, 2);
+ }
+ }, NullPointerException.class, null);
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetFileBlockLocationsIfFileStatusReferenceNotExistingPath() throws Exception {
+ Path path = new Path("someFile");
+
+ fs.create(path).close();
+
+ final FileStatus status = fs.getFileStatus(path);
+
+ fs.delete(path, true);
+
+ BlockLocation[] locations = fs.getFileBlockLocations(status, 1, 2);
+
+ assertEquals(0, locations.length);
+ }
+
+ /** @throws Exception If failed. */
+ public void testGetFileBlockLocations() throws Exception {
+ Path igfsHome = new Path(PRIMARY_URI);
+
+ Path file = new Path(igfsHome, "someFile");
+
+ try (OutputStream out = new BufferedOutputStream(fs.create(file, true, 1024 * 1024))) {
+ byte[] data = new byte[128 * 1024];
+
+ for (int i = 0; i < 100; i++)
+ out.write(data);
+
+ out.flush();
+ }
+
+ try (FSDataInputStream in = fs.open(file, 1024 * 1024)) {
+ byte[] data = new byte[128 * 1024];
+
+ int read;
+
+ do {
+ read = in.read(data);
+ }
+ while (read > 0);
+ }
+
+ FileStatus status = fs.getFileStatus(file);
+
+ int grpLen = 128 * 512 * 1024;
+
+ int grpCnt = (int)((status.getLen() + grpLen - 1) / grpLen);
+
+ BlockLocation[] locations = fs.getFileBlockLocations(status, 0, status.getLen());
+
+ assertEquals(grpCnt, locations.length);
+ }
+
+ /** @throws Exception If failed. */
+ @SuppressWarnings("deprecation")
+ public void testGetDefaultBlockSize() throws Exception {
+ assertEquals(1L << 26, fs.getDefaultBlockSize());
+ }
+
+ /** @throws Exception If failed. */
+ public void testZeroReplicationFactor() throws Exception {
+ // This test doesn't make sense for any mode except of PRIMARY.
+ if (mode == PRIMARY) {
+ Path igfsHome = new Path(PRIMARY_URI);
+
+ Path file = new Path(igfsHome, "someFile");
+
+ try (FSDataOutputStream out = fs.create(file, (short)0)) {
+ out.write(new byte[1024 * 1024]);
+ }
+
+ IgniteFileSystem igfs = grid(0).fileSystem("igfs");
+
+ IgfsPath filePath = new IgfsPath("/someFile");
+
+ IgfsFile fileInfo = igfs.info(filePath);
+
+ Collection<IgfsBlockLocation> locations = igfs.affinity(filePath, 0, fileInfo.length());
+
+ assertEquals(1, locations.size());
+
+ IgfsBlockLocation location = F.first(locations);
+
+ assertEquals(1, location.nodeIds().size());
+ }
+ }
+
+ /**
+ * Ensure that when running in multithreaded mode only one create() operation succeed.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMultithreadedCreate() throws Exception {
+ Path dir = new Path(new Path(PRIMARY_URI), "/dir");
+
+ assert fs.mkdirs(dir);
+
+ final Path file = new Path(dir, "file");
+
+ fs.create(file).close();
+
+ final AtomicInteger cnt = new AtomicInteger();
+
+ final Collection<Integer> errs = new GridConcurrentHashSet<>(THREAD_CNT, 1.0f, THREAD_CNT);
+
+ final AtomicBoolean err = new AtomicBoolean();
+
+ multithreaded(new Runnable() {
+ @Override
+ public void run() {
+ int idx = cnt.getAndIncrement();
+
+ byte[] data = new byte[256];
+
+ Arrays.fill(data, (byte)idx);
+
+ FSDataOutputStream os = null;
+
+ try {
+ os = fs.create(file, true);
+ }
+ catch (IOException ignore) {
+ errs.add(idx);
+ }
+
+ U.awaitQuiet(barrier);
+
+ try {
+ if (os != null)
+ os.write(data);
+ }
+ catch (IOException ignore) {
+ err.set(true);
+ }
+ finally {
+ U.closeQuiet(os);
+ }
+ }
+ }, THREAD_CNT);
+
+ assert !err.get();
+
+ // Only one thread could obtain write lock on the file.
+ assert errs.size() == THREAD_CNT - 1;
+
+ int idx = -1;
+
+ for (int i = 0; i < THREAD_CNT; i++) {
+ if (!errs.remove(i)) {
+ idx = i;
+
+ break;
+ }
+ }
+
+ byte[] expData = new byte[256];
+
+ Arrays.fill(expData, (byte)idx);
+
+ FSDataInputStream is = fs.open(file);
+
+ byte[] data = new byte[256];
+
+ is.read(data);
+
+ is.close();
+
+ assert Arrays.equals(expData, data) : "Expected=" + Arrays.toString(expData) + ", actual=" +
+ Arrays.toString(data);
+ }
+
+ /**
+ * Ensure that when running in multithreaded mode only one append() operation succeed.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMultithreadedAppend() throws Exception {
+ Path dir = new Path(new Path(PRIMARY_URI), "/dir");
+
+ assert fs.mkdirs(dir);
+
+ final Path file = new Path(dir, "file");
+
+ fs.create(file).close();
+
+ final AtomicInteger cnt = new AtomicInteger();
+
+ final Collection<Integer> errs = new GridConcurrentHashSet<>(THREAD_CNT, 1.0f, THREAD_CNT);
+
+ final AtomicBoolean err = new AtomicBoolean();
+
+ multithreaded(new Runnable() {
+ @Override public void run() {
+ int idx = cnt.getAndIncrement();
+
+ byte[] data = new byte[256];
+
+ Arrays.fill(data, (byte)idx);
+
+ U.awaitQuiet(barrier);
+
+ FSDataOutputStream os = null;
+
+ try {
+ os = fs.append(file);
+ }
+ catch (IOException ignore) {
+ errs.add(idx);
+ }
+
+ U.awaitQuiet(barrier);
+
+ try {
+ if (os != null)
+ os.write(data);
+ }
+ catch (IOException ignore) {
+ err.set(true);
+ }
+ finally {
+ U.closeQuiet(os);
+ }
+ }
+ }, THREAD_CNT);
+
+ assert !err.get();
+
+ // Only one thread could obtain write lock on the file.
+ assert errs.size() == THREAD_CNT - 1;
+
+ int idx = -1;
+
+ for (int i = 0; i < THREAD_CNT; i++) {
+ if (!errs.remove(i)) {
+ idx = i;
+
+ break;
+ }
+ }
+
+ byte[] expData = new byte[256];
+
+ Arrays.fill(expData, (byte)idx);
+
+ FSDataInputStream is = fs.open(file);
+
+ byte[] data = new byte[256];
+
+ is.read(data);
+
+ is.close();
+
+ assert Arrays.equals(expData, data);
+ }
+
+ /**
+ * Test concurrent reads within the file.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMultithreadedOpen() throws Exception {
+ final byte[] dataChunk = new byte[256];
+
+ for (int i = 0; i < dataChunk.length; i++)
+ dataChunk[i] = (byte)i;
+
+ Path dir = new Path(new Path(PRIMARY_URI), "/dir");
+
+ assert fs.mkdirs(dir);
+
+ final Path file = new Path(dir, "file");
+
+ FSDataOutputStream os = fs.create(file);
+
+ // Write 256 * 2048 = 512Kb of data.
+ for (int i = 0; i < 2048; i++)
+ os.write(dataChunk);
+
+ os.close();
+
+ final AtomicBoolean err = new AtomicBoolean();
+
+ multithreaded(new Runnable() {
+ @Override
+ public void run() {
+ FSDataInputStream is = null;
+
+ try {
+ int pos = ThreadLocalRandom8.current().nextInt(2048);
+
+ try {
+ is = fs.open(file);
+ }
+ finally {
+ U.awaitQuiet(barrier);
+ }
+
+ is.seek(256 * pos);
+
+ byte[] buf = new byte[256];
+
+ for (int i = pos; i < 2048; i++) {
+ // First perform normal read.
+ int read = is.read(buf);
+
+ assert read == 256;
+
+ Arrays.equals(dataChunk, buf);
+ }
+
+ int res = is.read(buf);
+
+ assert res == -1;
+ }
+ catch (IOException ignore) {
+ err.set(true);
+ }
+ finally {
+ U.closeQuiet(is);
+ }
+ }
+ }, THREAD_CNT);
+
+ assert !err.get();
+ }
+
+ /**
+ * Test concurrent creation of multiple directories.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMultithreadedMkdirs() throws Exception {
+ final Path dir = new Path(new Path(PRIMARY_URI), "/dir");
+
+ assert fs.mkdirs(dir);
+
+ final int depth = 3;
+ final int entryCnt = 5;
+
+ final AtomicReference<IOException> err = new AtomicReference();
+
+ multithreaded(new Runnable() {
+ @Override public void run() {
+ Deque<IgniteBiTuple<Integer, Path>> queue = new ArrayDeque<>();
+
+ queue.add(F.t(0, dir));
+
+ U.awaitQuiet(barrier);
+
+ while (!queue.isEmpty()) {
+ IgniteBiTuple<Integer, Path> t = queue.pollFirst();
+
+ int curDepth = t.getKey();
+ Path curPath = t.getValue();
+
+ if (curDepth <= depth) {
+ int newDepth = curDepth + 1;
+
+ // Create directories.
+ for (int i = 0; i < entryCnt; i++) {
+ Path subDir = new Path(curPath, "dir-" + newDepth + "-" + i);
+
+ try {
+ if (fs.mkdirs(subDir))
+ queue.addLast(F.t(newDepth, subDir));
+ }
+ catch (IOException e) {
+ err.compareAndSet(null, e);
+ }
+ }
+ }
+ }
+ }
+ }, THREAD_CNT);
+
+ // Ensure there were no errors.
+ assert err.get() == null : err.get();
+
+ // Ensure correct folders structure.
+ Deque<IgniteBiTuple<Integer, Path>> queue = new ArrayDeque<>();
+
+ queue.add(F.t(0, dir));
+
+ while (!queue.isEmpty()) {
+ IgniteBiTuple<Integer, Path> t = queue.pollFirst();
+
+ int curDepth = t.getKey();
+ Path curPath = t.getValue();
+
+ if (curDepth <= depth) {
+ int newDepth = curDepth + 1;
+
+ // Create directories.
+ for (int i = 0; i < entryCnt; i++) {
+ Path subDir = new Path(curPath, "dir-" + newDepth + "-" + i);
+
+ assert fs.exists(subDir) : "Expected directory doesn't exist: " + subDir;
+
+ queue.add(F.t(newDepth, subDir));
+ }
+ }
+ }
+ }
+
+ /**
+ * Test concurrent deletion of the same directory with advanced structure.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("TooBroadScope")
+ public void testMultithreadedDelete() throws Exception {
+ final Path dir = new Path(new Path(PRIMARY_URI), "/dir");
+
+ assert fs.mkdirs(dir);
+
+ int depth = 3;
+ int entryCnt = 5;
+
+ Deque<IgniteBiTuple<Integer, Path>> queue = new ArrayDeque<>();
+
+ queue.add(F.t(0, dir));
+
+ while (!queue.isEmpty()) {
+ IgniteBiTuple<Integer, Path> t = queue.pollFirst();
+
+ int curDepth = t.getKey();
+ Path curPath = t.getValue();
+
+ if (curDepth < depth) {
+ int newDepth = curDepth + 1;
+
+ // Create directories.
+ for (int i = 0; i < entryCnt; i++) {
+ Path subDir = new Path(curPath, "dir-" + newDepth + "-" + i);
+
+ fs.mkdirs(subDir);
+
+ queue.addLast(F.t(newDepth, subDir));
+ }
+ }
+ else {
+ // Create files.
+ for (int i = 0; i < entryCnt; i++) {
+ Path file = new Path(curPath, "file " + i);
+
+ fs.create(file).close();
+ }
+ }
+ }
+
+ final AtomicBoolean err = new AtomicBoolean();
+
+ multithreaded(new Runnable() {
+ @Override public void run() {
+ try {
+ U.awaitQuiet(barrier);
+
+ fs.delete(dir, true);
+ }
+ catch (IOException ignore) {
+ err.set(true);
+ }
+ }
+ }, THREAD_CNT);
+
+ // Ensure there were no errors.
+ assert !err.get();
+
+ // Ensure the directory was actually deleted.
+
+ assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ try {
+ return !fs.exists(dir);
+ }
+ catch (IOException e) {
+ throw new AssertionError(e);
+ }
+ }
+ }, 5000L);
+ }
+
+ /** @throws Exception If failed. */
+ public void testConsistency() throws Exception {
+ // Default buffers values
+ checkConsistency(-1, 1, -1, -1, 1, -1);
+ checkConsistency(-1, 10, -1, -1, 10, -1);
+ checkConsistency(-1, 100, -1, -1, 100, -1);
+ checkConsistency(-1, 1000, -1, -1, 1000, -1);
+ checkConsistency(-1, 10000, -1, -1, 10000, -1);
+ checkConsistency(-1, 100000, -1, -1, 100000, -1);
+
+ checkConsistency(65 * 1024 + 13, 100000, -1, -1, 100000, -1);
+
+ checkConsistency(-1, 100000, 2 * 4 * 1024 + 17, -1, 100000, -1);
+
+ checkConsistency(-1, 100000, -1, 65 * 1024 + 13, 100000, -1);
+
+ checkConsistency(-1, 100000, -1, -1, 100000, 2 * 4 * 1024 + 17);
+
+ checkConsistency(65 * 1024 + 13, 100000, 2 * 4 * 1024 + 13, 65 * 1024 + 149, 100000, 2 * 4 * 1024 + 157);
+ }
+
+ /**
+ * Verifies that client reconnects after connection to the server has been lost.
+ *
+ * @throws Exception If error occurs.
+ */
+ public void testClientReconnect() throws Exception {
+ Path filePath = new Path(PRIMARY_URI, "file1");
+
+ final FSDataOutputStream s = fs.create(filePath); // Open the stream before stopping IGFS.
+
+ try {
+ G.stopAll(true); // Stop the server.
+
+ startNodes(); // Start server again.
+
+ // Check that client is again operational.
+ assertTrue(fs.mkdirs(new Path(PRIMARY_URI, "dir1/dir2")));
+
+ // However, the streams, opened before disconnect, should not be valid.
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Nullable @Override public Object call() throws Exception {
+ s.write("test".getBytes());
+
+ s.flush(); // Flush data to the broken output stream.
+
+ return null;
+ }
+ }, IOException.class, null);
+
+ assertFalse(fs.exists(filePath));
+ }
+ finally {
+ U.closeQuiet(s); // Safety.
+ }
+ }
+
+ /**
+ * Verifies that client reconnects after connection to the server has been lost (multithreaded mode).
+ *
+ * @throws Exception If error occurs.
+ */
+ public void testClientReconnectMultithreaded() throws Exception {
+ final ConcurrentLinkedQueue<FileSystem> q = new ConcurrentLinkedQueue<>();
+
+ Configuration cfg = new Configuration();
+
+ for (Map.Entry<String, String> entry : primaryFsCfg)
+ cfg.set(entry.getKey(), entry.getValue());
+
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ final int nClients = 1;
+
+ // Initialize clients.
+ for (int i = 0; i < nClients; i++)
+ q.add(FileSystem.get(primaryFsUri, cfg));
+
+ G.stopAll(true); // Stop the server.
+
+ startNodes(); // Start server again.
+
+ GridTestUtils.runMultiThreaded(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ FileSystem fs = q.poll();
+
+ try {
+ // Check that client is again operational.
+ assertTrue(fs.mkdirs(new Path("/" + Thread.currentThread().getName())));
+
+ return true;
+ }
+ finally {
+ U.closeQuiet(fs);
+ }
+ }
+ }, nClients, "test-client");
+ }
+
+ /**
+ * Checks consistency of create --> open --> append --> open operations with different buffer sizes.
+ *
+ * @param createBufSize Buffer size used for file creation.
+ * @param writeCntsInCreate Count of times to write in file creation.
+ * @param openAfterCreateBufSize Buffer size used for file opening after creation.
+ * @param appendBufSize Buffer size used for file appending.
+ * @param writeCntsInAppend Count of times to write in file appending.
+ * @param openAfterAppendBufSize Buffer size used for file opening after appending.
+ * @throws Exception If failed.
+ */
+ private void checkConsistency(int createBufSize, int writeCntsInCreate, int openAfterCreateBufSize,
+ int appendBufSize, int writeCntsInAppend, int openAfterAppendBufSize) throws Exception {
+ final Path igfsHome = new Path(PRIMARY_URI);
+
+ Path file = new Path(igfsHome, "/someDir/someInnerDir/someFile");
+
+ FSDataOutputStream os = fs.create(file, true, createBufSize);
+
+ for (int i = 0; i < writeCntsInCreate; i++)
+ os.writeInt(i);
+
+ os.close();
+
+ FSDataInputStream is = fs.open(file, openAfterCreateBufSize);
+
+ for (int i = 0; i < writeCntsInCreate; i++)
+ assertEquals(i, is.readInt());
+
+ is.close();
+
+ os = fs.append(file, appendBufSize);
+
+ for (int i = writeCntsInCreate; i < writeCntsInCreate + writeCntsInAppend; i++)
+ os.writeInt(i);
+
+ os.close();
+
+ is = fs.open(file, openAfterAppendBufSize);
+
+ for (int i = 0; i < writeCntsInCreate + writeCntsInAppend; i++)
+ assertEquals(i, is.readInt());
+
+ is.close();
+ }
+
+ /**
+ * Gets instance of Hadoop local file system.
+ *
+ * @param home File system home.
+ * @return File system.
+ * @throws IOException If failed.
+ */
+ private FileSystem local(Path home) throws IOException {
+ Configuration cfg = new Configuration();
+
+ cfg.addResource(U.resolveIgniteUrl(HADOOP_FS_CFG));
+
+ return FileSystem.get(home.toUri(), cfg);
+ }
+
+ /**
+ * Copy files from one FS to another.
+ *
+ * @param msg Info message to display after copying finishes.
+ * @param srcFs Source file system.
+ * @param src Source path to copy from.
+ * @param destFs Destination file system.
+ * @param dest Destination path to copy to.
+ * @throws IOException If failed.
+ */
+ private void copy(String msg, FileSystem srcFs, Path src, FileSystem destFs, Path dest) throws IOException {
+ assert destFs.delete(dest, true) || !destFs.exists(dest) : "Failed to remove: " + dest;
+
+ destFs.mkdirs(dest);
+
+ Configuration conf = new Configuration(true);
+
+ long time = System.currentTimeMillis();
+
+ FileUtil.copy(srcFs, src, destFs, dest, false, true, conf);
+
+ time = System.currentTimeMillis() - time;
+
+ info("Copying finished, " + msg + " [time=" + time + "ms, src=" + src + ", dest=" + dest + ']');
+ }
+
+ /**
+ * Compare content of two folders.
+ *
+ * @param cfg Paths configuration to compare.
+ * @throws IOException If failed.
+ */
+ @SuppressWarnings("deprecation")
+ private void compareContent(Config cfg) throws IOException {
+ Deque<Config> queue = new LinkedList<>();
+
+ queue.add(cfg);
+
+ for (Config c = queue.poll(); c != null; c = queue.poll()) {
+ boolean exists;
+
+ assertEquals("Check existence [src=" + c.src + ", dest=" + c.dest + ']',
+ exists = c.srcFs.exists(c.src), c.destFs.exists(c.dest));
+
+ assertEquals("Check types (files?) [src=" + c.src + ", dest=" + c.dest + ']',
+ c.srcFs.isFile(c.src), c.destFs.isFile(c.dest));
+
+ if (exists) {
+ ContentSummary srcSummary = c.srcFs.getContentSummary(c.src);
+ ContentSummary dstSummary = c.destFs.getContentSummary(c.dest);
+
+ assertEquals("Directories number comparison failed",
+ srcSummary.getDirectoryCount(), dstSummary.getDirectoryCount());
+
+ assertEquals("Files number comparison failed",
+ srcSummary.getFileCount(), dstSummary.getFileCount());
+
+ assertEquals("Space consumed comparison failed",
+ srcSummary.getSpaceConsumed(), dstSummary.getSpaceConsumed());
+
+ assertEquals("Length comparison failed",
+ srcSummary.getLength(), dstSummary.getLength());
+
+ // Intentionally skipping quotas checks as they can vary.
+ }
+ else {
+ assertContentSummaryFails(c.srcFs, c.src);
+ assertContentSummaryFails(c.destFs, c.dest);
+ }
+
+ if (!exists)
+ continue;
+
+ FileStatus[] srcSt = c.srcFs.listStatus(c.src);
+ FileStatus[] destSt = c.destFs.listStatus(c.dest);
+
+ assert srcSt != null && destSt != null : "Both not null" +
+ " [srcSt=" + Arrays.toString(srcSt) + ", destSt=" + Arrays.toString(destSt) + ']';
+
+ assertEquals("Check listing [src=" + c.src + ", dest=" + c.dest + ']', srcSt.length, destSt.length);
+
+ // Listing of the file returns the only element with this file.
+ if (srcSt.length == 1 && c.src.equals(srcSt[0].getPath())) {
+ assertEquals(c.dest, destSt[0].getPath());
+
+ assertTrue("Expects file [src=" + c.src + ", srcSt[0]=" + srcSt[0] + ']', !srcSt[0].isDir());
+ assertTrue("Expects file [dest=" + c.dest + ", destSt[0]=" + destSt[0] + ']', !destSt[0].isDir());
+
+ FSDataInputStream srcIn = null;
+ FSDataInputStream destIn = null;
+
+ try {
+ srcIn = c.srcFs.open(c.src);
+ destIn = c.destFs.open(c.dest);
+
+ GridTestIoUtils.assertEqualStreams(srcIn, destIn, srcSt[0].getLen());
+ }
+ finally {
+ U.closeQuiet(srcIn);
+ U.closeQuiet(destIn);
+ }
+
+ continue; // Skip the following directories validations.
+ }
+
+ // Sort both arrays.
+ Arrays.sort(srcSt, STATUS_COMPARATOR);
+ Arrays.sort(destSt, STATUS_COMPARATOR);
+
+ for (int i = 0; i < srcSt.length; i++)
+ // Dig in deep to the last leaf, instead of collecting full tree in memory.
+ queue.addFirst(new Config(c.srcFs, srcSt[i].getPath(), c.destFs, destSt[i].getPath()));
+
+ // Add non-existent file to check in the current folder.
+ String rndFile = "Non-existent file #" + UUID.randomUUID().toString();
+
+ queue.addFirst(new Config(c.srcFs, new Path(c.src, rndFile), c.destFs, new Path(c.dest, rndFile)));
+ }
+ }
+
+ /**
+ * Test expected failures for 'close' operation.
+ *
+ * @param fs File system to test.
+ * @param msg Expected exception message.
+ */
+ public void assertCloseFails(final FileSystem fs, String msg) {
+ GridTestUtils.assertThrows(log, new Callable() {
+ @Override public Object call() throws Exception {
+ fs.close();
+
+ return null;
+ }
+ }, IOException.class, msg);
+ }
+
+ /**
+ * Test expected failures for 'get content summary' operation.
+ *
+ * @param fs File system to test.
+ * @param path Path to evaluate content summary for.
+ */
+ private void assertContentSummaryFails(final FileSystem fs, final Path path) {
+ GridTestUtils.assertThrows(log, new Callable<ContentSummary>() {
+ @Override public ContentSummary call() throws Exception {
+ return fs.getContentSummary(path);
+ }
+ }, FileNotFoundException.class, null);
+ }
+
+ /**
+ * Assert that a given path exists in a given FileSystem.
+ *
+ * @param fs FileSystem to check.
+ * @param p Path to check.
+ * @throws IOException if the path does not exist.
+ */
+ private void assertPathExists(FileSystem fs, Path p) throws IOException {
+ FileStatus fileStatus = fs.getFileStatus(p);
+
+ assertEquals(p, fileStatus.getPath());
+ assertNotSame(0, fileStatus.getModificationTime());
+ }
+
+ /**
+ * Check path does not exist in a given FileSystem.
+ *
+ * @param fs FileSystem to check.
+ * @param path Path to check.
+ */
+ private void assertPathDoesNotExist(final FileSystem fs, final Path path) {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return fs.getFileStatus(path);
+ }
+ }, FileNotFoundException.class, null);
+ }
+
+ /** Helper class to encapsulate source and destination folders. */
+ @SuppressWarnings({"PublicInnerClass", "PublicField"})
+ public static final class Config {
+ /** Source file system. */
+ public final FileSystem srcFs;
+
+ /** Source path to work with. */
+ public final Path src;
+
+ /** Destination file system. */
+ public final FileSystem destFs;
+
+ /** Destination path to work with. */
+ public final Path dest;
+
+ /**
+ * Copying task configuration.
+ *
+ * @param srcFs Source file system.
+ * @param src Source path.
+ * @param destFs Destination file system.
+ * @param dest Destination path.
+ */
+ public Config(FileSystem srcFs, Path src, FileSystem destFs, Path dest) {
+ this.srcFs = srcFs;
+ this.src = src;
+ this.destFs = destFs;
+ this.dest = dest;
+ }
+ }
+
+ /**
+ * Convert path for exception message testing purposes.
+ *
+ * @param path Path.
+ * @return Converted path.
+ * @throws Exception If failed.
+ */
+ private Path convertPath(Path path) throws Exception {
+ if (mode != PROXY)
+ return path;
+ else {
+ URI secondaryUri = new URI(SECONDARY_URI);
+
+ URI pathUri = path.toUri();
+
+ return new Path(new URI(pathUri.getScheme() != null ? secondaryUri.getScheme() : null,
+ pathUri.getAuthority() != null ? secondaryUri.getAuthority() : null, pathUri.getPath(), null, null));
+ }
+ }
+
+ /**
+ * Create configuration for test.
+ *
+ * @param authority Authority.
+ * @param skipEmbed Whether to skip embedded mode.
+ * @param skipLocShmem Whether to skip local shmem mode.
+ * @return Configuration.
+ */
+ private static Configuration configuration(String authority, boolean skipEmbed, boolean skipLocShmem) {
+ Configuration cfg = new Configuration();
+
+ cfg.set("fs.defaultFS", "igfs://" + authority + "/");
+ cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
+ cfg.set("fs.AbstractFileSystem.igfs.impl",
+ org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName());
+
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ if (skipEmbed)
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);
+
+ if (skipLocShmem)
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);
+
+ return cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemClientSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemClientSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemClientSelfTest.java
new file mode 100644
index 0000000..29dd996
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemClientSelfTest.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.commons.logging.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.igfs.common.*;
+import org.apache.ignite.internal.processors.hadoop.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+import org.apache.ignite.testframework.*;
+
+import java.io.*;
+import java.lang.reflect.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheDistributionMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*;
+import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
+
+/**
+ * Test interaction between a IGFS client and a IGFS server.
+ */
+public class IgniteHadoopFileSystemClientSelfTest extends IgfsCommonAbstractTest {
+ /** Logger. */
+ private static final Log LOG = LogFactory.getLog(IgniteHadoopFileSystemClientSelfTest.class);
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ startGrids(1);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ G.stopAll(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName("igfs");
+ igfsCfg.setBlockSize(512 * 1024);
+ igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", String.valueOf(DFLT_IPC_PORT));
+ }});
+
+ cfg.setCacheConfiguration(cacheConfiguration());
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ return cfg;
+ }
+
+ /**
+ * Gets cache configuration.
+ *
+ * @return Cache configuration.
+ */
+ protected CacheConfiguration[] cacheConfiguration() {
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(FULL_SYNC);
+ cacheCfg.setEvictionPolicy(null);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(FULL_SYNC);
+ metaCacheCfg.setEvictionPolicy(null);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ return new CacheConfiguration[] {metaCacheCfg, cacheCfg};
+ }
+
+ /**
+ * Test output stream deferred exception (GG-4440).
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testOutputStreamDeferredException() throws Exception {
+ final byte[] data = "test".getBytes();
+
+ try {
+ switchHandlerErrorFlag(true);
+
+ HadoopIgfs client = new HadoopIgfsOutProc("127.0.0.1", 10500, getTestGridName(0), "igfs", LOG);
+
+ client.handshake(null);
+
+ IgfsPath path = new IgfsPath("/test1.file");
+
+ HadoopIgfsStreamDelegate delegate = client.create(path, true, false, 1, 1024, null);
+
+ final HadoopIgfsOutputStream igfsOut = new HadoopIgfsOutputStream(delegate, LOG,
+ IgfsLogger.disabledLogger(), 0);
+
+ // This call should return fine as exception is thrown for the first time.
+ igfsOut.write(data);
+
+ U.sleep(500);
+
+ // This call should throw an IO exception.
+ GridTestUtils.assertThrows(null, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ igfsOut.write(data);
+
+ return null;
+ }
+ }, IOException.class, "Failed to write data to server (test).");
+ }
+ finally {
+ switchHandlerErrorFlag(false);
+ }
+ }
+
+ /**
+ * Set IGFS REST handler error flag to the given state.
+ *
+ * @param flag Flag state.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ConstantConditions")
+ private void switchHandlerErrorFlag(boolean flag) throws Exception {
+ IgfsProcessorAdapter igfsProc = ((IgniteKernal)grid(0)).context().igfs();
+
+ Map<String, IgfsContext> igfsMap = getField(igfsProc, "igfsCache");
+
+ IgfsServerManager srvMgr = F.first(igfsMap.values()).server();
+
+ Collection<IgfsServer> srvrs = getField(srvMgr, "srvrs");
+
+ IgfsServerHandler igfsHnd = getField(F.first(srvrs), "hnd");
+
+ Field field = igfsHnd.getClass().getDeclaredField("errWrite");
+
+ field.setAccessible(true);
+
+ field.set(null, flag);
+ }
+
+ /**
+ * Get value of the field with the given name of the given object.
+ *
+ * @param obj Object.
+ * @param fieldName Field name.
+ * @return Value of the field.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ private <T> T getField(Object obj, String fieldName) throws Exception {
+ Field field = obj.getClass().getDeclaredField(fieldName);
+
+ field.setAccessible(true);
+
+ return (T)field.get(obj);
+ }
+}
[04/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java
new file mode 100644
index 0000000..76988a3
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java
@@ -0,0 +1,1006 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.apache.ignite.cache.*;
+import org.apache.ignite.cluster.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.hadoop.mapreduce.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.mapreduce.*;
+import org.apache.ignite.igfs.secondary.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.cluster.*;
+import org.apache.ignite.internal.processors.cache.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.lang.*;
+import org.apache.ignite.testframework.*;
+import org.jetbrains.annotations.*;
+
+import java.net.*;
+import java.util.*;
+
+/**
+ *
+ */
+public class HadoopDefaultMapReducePlannerSelfTest extends HadoopAbstractSelfTest {
+ /** */
+ private static final UUID ID_1 = new UUID(0, 1);
+
+ /** */
+ private static final UUID ID_2 = new UUID(0, 2);
+
+ /** */
+ private static final UUID ID_3 = new UUID(0, 3);
+
+ /** */
+ private static final String HOST_1 = "host1";
+
+ /** */
+ private static final String HOST_2 = "host2";
+
+ /** */
+ private static final String HOST_3 = "host3";
+
+ /** */
+ private static final String INVALID_HOST_1 = "invalid_host1";
+
+ /** */
+ private static final String INVALID_HOST_2 = "invalid_host2";
+
+ /** */
+ private static final String INVALID_HOST_3 = "invalid_host3";
+
+ /** Mocked Grid. */
+ private static final MockIgnite GRID = new MockIgnite();
+
+ /** Mocked IGFS. */
+ private static final IgniteFileSystem IGFS = new MockIgfs();
+
+ /** Planner. */
+ private static final HadoopMapReducePlanner PLANNER = new IgniteHadoopMapReducePlanner();
+
+ /** Block locations. */
+ private static final Map<Block, Collection<IgfsBlockLocation>> BLOCK_MAP = new HashMap<>();
+
+ /** Proxy map. */
+ private static final Map<URI, Boolean> PROXY_MAP = new HashMap<>();
+
+ /** Last created plan. */
+ private static final ThreadLocal<HadoopMapReducePlan> PLAN = new ThreadLocal<>();
+
+ /**
+ *
+ */
+ static {
+ GridTestUtils.setFieldValue(PLANNER, "ignite", GRID);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ GridTestUtils.setFieldValue(PLANNER, "log", log());
+
+ BLOCK_MAP.clear();
+ PROXY_MAP.clear();
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testIgfsOneBlockPerNode() throws IgniteCheckedException {
+ HadoopFileBlock split1 = split(true, "/file1", 0, 100, HOST_1);
+ HadoopFileBlock split2 = split(true, "/file2", 0, 100, HOST_2);
+ HadoopFileBlock split3 = split(true, "/file3", 0, 100, HOST_3);
+
+ mapIgfsBlock(split1.file(), 0, 100, location(0, 100, ID_1));
+ mapIgfsBlock(split2.file(), 0, 100, location(0, 100, ID_2));
+ mapIgfsBlock(split3.file(), 0, 100, location(0, 100, ID_3));
+
+ plan(1, split1);
+ assert ensureMappers(ID_1, split1);
+ assert ensureReducers(ID_1, 1);
+ assert ensureEmpty(ID_2);
+ assert ensureEmpty(ID_3);
+
+ plan(2, split1);
+ assert ensureMappers(ID_1, split1);
+ assert ensureReducers(ID_1, 2);
+ assert ensureEmpty(ID_2);
+ assert ensureEmpty(ID_3);
+
+ plan(1, split1, split2);
+ assert ensureMappers(ID_1, split1);
+ assert ensureMappers(ID_2, split2);
+ assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 0) || ensureReducers(ID_1, 0) && ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+
+ plan(2, split1, split2);
+ assert ensureMappers(ID_1, split1);
+ assert ensureMappers(ID_2, split2);
+ assert ensureReducers(ID_1, 1);
+ assert ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+
+ plan(3, split1, split2);
+ assert ensureMappers(ID_1, split1);
+ assert ensureMappers(ID_2, split2);
+ assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) || ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+
+ plan(3, split1, split2, split3);
+ assert ensureMappers(ID_1, split1);
+ assert ensureMappers(ID_2, split2);
+ assert ensureMappers(ID_3, split3);
+ assert ensureReducers(ID_1, 1);
+ assert ensureReducers(ID_2, 1);
+ assert ensureReducers(ID_3, 1);
+
+ plan(5, split1, split2, split3);
+ assert ensureMappers(ID_1, split1);
+ assert ensureMappers(ID_2, split2);
+ assert ensureMappers(ID_3, split3);
+ assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 2) ||
+ ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1) && ensureReducers(ID_3, 2) ||
+ ensureReducers(ID_1, 2) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 1);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testNonIgfsOneBlockPerNode() throws IgniteCheckedException {
+ HadoopFileBlock split1 = split(false, "/file1", 0, 100, HOST_1);
+ HadoopFileBlock split2 = split(false, "/file2", 0, 100, HOST_2);
+ HadoopFileBlock split3 = split(false, "/file3", 0, 100, HOST_3);
+
+ plan(1, split1);
+ assert ensureMappers(ID_1, split1);
+ assert ensureReducers(ID_1, 1);
+ assert ensureEmpty(ID_2);
+ assert ensureEmpty(ID_3);
+
+ plan(2, split1);
+ assert ensureMappers(ID_1, split1);
+ assert ensureReducers(ID_1, 2);
+ assert ensureEmpty(ID_2);
+ assert ensureEmpty(ID_3);
+
+ plan(1, split1, split2);
+ assert ensureMappers(ID_1, split1);
+ assert ensureMappers(ID_2, split2);
+ assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 0) || ensureReducers(ID_1, 0) && ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+
+ plan(2, split1, split2);
+ assert ensureMappers(ID_1, split1);
+ assert ensureMappers(ID_2, split2);
+ assert ensureReducers(ID_1, 1);
+ assert ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+
+ plan(3, split1, split2);
+ assert ensureMappers(ID_1, split1);
+ assert ensureMappers(ID_2, split2);
+ assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) || ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+
+ plan(3, split1, split2, split3);
+ assert ensureMappers(ID_1, split1);
+ assert ensureMappers(ID_2, split2);
+ assert ensureMappers(ID_3, split3);
+ assert ensureReducers(ID_1, 1);
+ assert ensureReducers(ID_2, 1);
+ assert ensureReducers(ID_3, 1);
+
+ plan(5, split1, split2, split3);
+ assert ensureMappers(ID_1, split1);
+ assert ensureMappers(ID_2, split2);
+ assert ensureMappers(ID_3, split3);
+ assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 2) ||
+ ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1) && ensureReducers(ID_3, 2) ||
+ ensureReducers(ID_1, 2) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 1);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testIgfsSeveralBlocksPerNode() throws IgniteCheckedException {
+ HadoopFileBlock split1 = split(true, "/file1", 0, 100, HOST_1, HOST_2);
+ HadoopFileBlock split2 = split(true, "/file2", 0, 100, HOST_1, HOST_2);
+ HadoopFileBlock split3 = split(true, "/file3", 0, 100, HOST_1, HOST_3);
+
+ mapIgfsBlock(split1.file(), 0, 100, location(0, 100, ID_1, ID_2));
+ mapIgfsBlock(split2.file(), 0, 100, location(0, 100, ID_1, ID_2));
+ mapIgfsBlock(split3.file(), 0, 100, location(0, 100, ID_1, ID_3));
+
+ plan(1, split1);
+ assert ensureMappers(ID_1, split1) && ensureReducers(ID_1, 1) && ensureEmpty(ID_2) ||
+ ensureEmpty(ID_1) && ensureMappers(ID_2, split1) && ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+
+ plan(2, split1);
+ assert ensureMappers(ID_1, split1) && ensureReducers(ID_1, 2) && ensureEmpty(ID_2) ||
+ ensureEmpty(ID_1) && ensureMappers(ID_2, split1) && ensureReducers(ID_2, 2);
+ assert ensureEmpty(ID_3);
+
+ plan(1, split1, split2);
+ assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) ||
+ ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1);
+ assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 0) || ensureReducers(ID_1, 0) && ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+
+ plan(2, split1, split2);
+ assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) ||
+ ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1);
+ assert ensureReducers(ID_1, 1);
+ assert ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+
+ plan(3, split1, split2, split3);
+ assert ensureReducers(ID_1, 1);
+ assert ensureReducers(ID_2, 1);
+ assert ensureReducers(ID_3, 1);
+
+ plan(5, split1, split2, split3);
+ assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 2) ||
+ ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1) && ensureReducers(ID_3, 2) ||
+ ensureReducers(ID_1, 2) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 1);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testNonIgfsSeveralBlocksPerNode() throws IgniteCheckedException {
+ HadoopFileBlock split1 = split(false, "/file1", 0, 100, HOST_1, HOST_2);
+ HadoopFileBlock split2 = split(false, "/file2", 0, 100, HOST_1, HOST_2);
+ HadoopFileBlock split3 = split(false, "/file3", 0, 100, HOST_1, HOST_3);
+
+ plan(1, split1);
+ assert ensureMappers(ID_1, split1) && ensureReducers(ID_1, 1) && ensureEmpty(ID_2) ||
+ ensureEmpty(ID_1) && ensureMappers(ID_2, split1) && ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+
+ plan(2, split1);
+ assert ensureMappers(ID_1, split1) && ensureReducers(ID_1, 2) && ensureEmpty(ID_2) ||
+ ensureEmpty(ID_1) && ensureMappers(ID_2, split1) && ensureReducers(ID_2, 2);
+ assert ensureEmpty(ID_3);
+
+ plan(1, split1, split2);
+ assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) ||
+ ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1);
+ assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 0) || ensureReducers(ID_1, 0) && ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+
+ plan(2, split1, split2);
+ assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) ||
+ ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1);
+ assert ensureReducers(ID_1, 1);
+ assert ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+
+ plan(3, split1, split2, split3);
+ assert ensureReducers(ID_1, 1);
+ assert ensureReducers(ID_2, 1);
+ assert ensureReducers(ID_3, 1);
+
+ plan(5, split1, split2, split3);
+ assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 2) ||
+ ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1) && ensureReducers(ID_3, 2) ||
+ ensureReducers(ID_1, 2) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 1);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testIgfsSeveralComplexBlocksPerNode() throws IgniteCheckedException {
+ HadoopFileBlock split1 = split(true, "/file1", 0, 100, HOST_1, HOST_2, HOST_3);
+ HadoopFileBlock split2 = split(true, "/file2", 0, 100, HOST_1, HOST_2, HOST_3);
+
+ mapIgfsBlock(split1.file(), 0, 100, location(0, 50, ID_1, ID_2), location(51, 100, ID_1, ID_3));
+ mapIgfsBlock(split2.file(), 0, 100, location(0, 50, ID_1, ID_2), location(51, 100, ID_2, ID_3));
+
+ plan(1, split1);
+ assert ensureMappers(ID_1, split1);
+ assert ensureReducers(ID_1, 1);
+ assert ensureEmpty(ID_2);
+ assert ensureEmpty(ID_3);
+
+ plan(1, split2);
+ assert ensureMappers(ID_2, split2);
+ assert ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_1);
+ assert ensureEmpty(ID_3);
+
+ plan(1, split1, split2);
+ assert ensureMappers(ID_1, split1);
+ assert ensureMappers(ID_2, split2);
+ assert ensureReducers(ID_1, 0) && ensureReducers(ID_2, 1) || ensureReducers(ID_1, 1) && ensureReducers(ID_2, 0);
+ assert ensureEmpty(ID_3);
+
+ plan(2, split1, split2);
+ assert ensureMappers(ID_1, split1);
+ assert ensureMappers(ID_2, split2);
+ assert ensureReducers(ID_1, 1);
+ assert ensureReducers(ID_2, 1);
+ assert ensureEmpty(ID_3);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testNonIgfsOrphans() throws IgniteCheckedException {
+ HadoopFileBlock split1 = split(false, "/file1", 0, 100, INVALID_HOST_1, INVALID_HOST_2);
+ HadoopFileBlock split2 = split(false, "/file2", 0, 100, INVALID_HOST_1, INVALID_HOST_3);
+ HadoopFileBlock split3 = split(false, "/file3", 0, 100, INVALID_HOST_2, INVALID_HOST_3);
+
+ plan(1, split1);
+ assert ensureMappers(ID_1, split1) && ensureReducers(ID_1, 1) && ensureEmpty(ID_2) && ensureEmpty(ID_3) ||
+ ensureEmpty(ID_1) && ensureMappers(ID_2, split1) && ensureReducers(ID_2, 1) && ensureEmpty(ID_3) ||
+ ensureEmpty(ID_1) && ensureEmpty(ID_2) && ensureMappers(ID_3, split1) && ensureReducers(ID_3, 1);
+
+ plan(2, split1);
+ assert ensureMappers(ID_1, split1) && ensureReducers(ID_1, 2) && ensureEmpty(ID_2) && ensureEmpty(ID_3) ||
+ ensureEmpty(ID_1) && ensureMappers(ID_2, split1) && ensureReducers(ID_2, 2) && ensureEmpty(ID_3) ||
+ ensureEmpty(ID_1) && ensureEmpty(ID_2) && ensureMappers(ID_3, split1) && ensureReducers(ID_3, 2);
+
+ plan(1, split1, split2, split3);
+ assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) && ensureMappers(ID_3, split3) ||
+ ensureMappers(ID_1, split1) && ensureMappers(ID_2, split3) && ensureMappers(ID_3, split2) ||
+ ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1) && ensureMappers(ID_3, split3) ||
+ ensureMappers(ID_1, split2) && ensureMappers(ID_2, split3) && ensureMappers(ID_3, split1) ||
+ ensureMappers(ID_1, split3) && ensureMappers(ID_2, split1) && ensureMappers(ID_3, split2) ||
+ ensureMappers(ID_1, split3) && ensureMappers(ID_2, split2) && ensureMappers(ID_3, split1);
+ assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 0) && ensureReducers(ID_3, 0) ||
+ ensureReducers(ID_1, 0) && ensureReducers(ID_2, 1) && ensureReducers(ID_3, 0) ||
+ ensureReducers(ID_1, 0) && ensureReducers(ID_2, 0) && ensureReducers(ID_3, 1);
+
+ plan(3, split1, split2, split3);
+ assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) && ensureMappers(ID_3, split3) ||
+ ensureMappers(ID_1, split1) && ensureMappers(ID_2, split3) && ensureMappers(ID_3, split2) ||
+ ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1) && ensureMappers(ID_3, split3) ||
+ ensureMappers(ID_1, split2) && ensureMappers(ID_2, split3) && ensureMappers(ID_3, split1) ||
+ ensureMappers(ID_1, split3) && ensureMappers(ID_2, split1) && ensureMappers(ID_3, split2) ||
+ ensureMappers(ID_1, split3) && ensureMappers(ID_2, split2) && ensureMappers(ID_3, split1);
+ assert ensureReducers(ID_1, 1);
+ assert ensureReducers(ID_2, 1);
+ assert ensureReducers(ID_3, 1);
+
+ plan(5, split1, split2, split3);
+ assert ensureMappers(ID_1, split1) && ensureMappers(ID_2, split2) && ensureMappers(ID_3, split3) ||
+ ensureMappers(ID_1, split1) && ensureMappers(ID_2, split3) && ensureMappers(ID_3, split2) ||
+ ensureMappers(ID_1, split2) && ensureMappers(ID_2, split1) && ensureMappers(ID_3, split3) ||
+ ensureMappers(ID_1, split2) && ensureMappers(ID_2, split3) && ensureMappers(ID_3, split1) ||
+ ensureMappers(ID_1, split3) && ensureMappers(ID_2, split1) && ensureMappers(ID_3, split2) ||
+ ensureMappers(ID_1, split3) && ensureMappers(ID_2, split2) && ensureMappers(ID_3, split1);
+ assert ensureReducers(ID_1, 1) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 2) ||
+ ensureReducers(ID_1, 2) && ensureReducers(ID_2, 1) && ensureReducers(ID_3, 2) ||
+ ensureReducers(ID_1, 2) && ensureReducers(ID_2, 2) && ensureReducers(ID_3, 1);
+ }
+
+ /**
+ * Create plan.
+ *
+ * @param reducers Reducers count.
+ * @param splits Splits.
+ * @return Plan.
+ * @throws IgniteCheckedException If failed.
+ */
+ private static HadoopMapReducePlan plan(int reducers, HadoopInputSplit... splits) throws IgniteCheckedException {
+ assert reducers > 0;
+ assert splits != null && splits.length > 0;
+
+ Collection<HadoopInputSplit> splitList = new ArrayList<>(splits.length);
+
+ Collections.addAll(splitList, splits);
+
+ Collection<ClusterNode> top = new ArrayList<>();
+
+ GridTestNode node1 = new GridTestNode(ID_1);
+ GridTestNode node2 = new GridTestNode(ID_2);
+ GridTestNode node3 = new GridTestNode(ID_3);
+
+ node1.setHostName(HOST_1);
+ node2.setHostName(HOST_2);
+ node3.setHostName(HOST_3);
+
+ top.add(node1);
+ top.add(node2);
+ top.add(node3);
+
+ HadoopMapReducePlan plan = PLANNER.preparePlan(new MockJob(reducers, splitList), top, null);
+
+ PLAN.set(plan);
+
+ return plan;
+ }
+
+ /**
+ * Ensure that node contains the given mappers.
+ *
+ * @param nodeId Node ID.
+ * @param expSplits Expected splits.
+ * @return {@code True} if this assumption is valid.
+ */
+ private static boolean ensureMappers(UUID nodeId, HadoopInputSplit... expSplits) {
+ Collection<HadoopInputSplit> expSplitsCol = new ArrayList<>();
+
+ Collections.addAll(expSplitsCol, expSplits);
+
+ Collection<HadoopInputSplit> splits = PLAN.get().mappers(nodeId);
+
+ return F.eq(expSplitsCol, splits);
+ }
+
+ /**
+ * Ensure that node contains the given amount of reducers.
+ *
+ * @param nodeId Node ID.
+ * @param reducers Reducers.
+ * @return {@code True} if this assumption is valid.
+ */
+ private static boolean ensureReducers(UUID nodeId, int reducers) {
+ int[] reducersArr = PLAN.get().reducers(nodeId);
+
+ return reducers == 0 ? F.isEmpty(reducersArr) : (reducersArr != null && reducersArr.length == reducers);
+ }
+
+ /**
+ * Ensure that no mappers and reducers is located on this node.
+ *
+ * @param nodeId Node ID.
+ * @return {@code True} if this assumption is valid.
+ */
+ private static boolean ensureEmpty(UUID nodeId) {
+ return F.isEmpty(PLAN.get().mappers(nodeId)) && F.isEmpty(PLAN.get().reducers(nodeId));
+ }
+
+ /**
+ * Create split.
+ *
+ * @param igfs IGFS flag.
+ * @param file File.
+ * @param start Start.
+ * @param len Length.
+ * @param hosts Hosts.
+ * @return Split.
+ */
+ private static HadoopFileBlock split(boolean igfs, String file, long start, long len, String... hosts) {
+ URI uri = URI.create((igfs ? "igfs://igfs@" : "hdfs://") + file);
+
+ return new HadoopFileBlock(hosts, uri, start, len);
+ }
+
+ /**
+ * Create block location.
+ *
+ * @param start Start.
+ * @param len Length.
+ * @param nodeIds Node IDs.
+ * @return Block location.
+ */
+ private static IgfsBlockLocation location(long start, long len, UUID... nodeIds) {
+ assert nodeIds != null && nodeIds.length > 0;
+
+ Collection<ClusterNode> nodes = new ArrayList<>(nodeIds.length);
+
+ for (UUID id : nodeIds)
+ nodes.add(new GridTestNode(id));
+
+ return new IgfsBlockLocationImpl(start, len, nodes);
+ }
+
+ /**
+ * Map IGFS block to nodes.
+ *
+ * @param file File.
+ * @param start Start.
+ * @param len Length.
+ * @param locations Locations.
+ */
+ private static void mapIgfsBlock(URI file, long start, long len, IgfsBlockLocation... locations) {
+ assert locations != null && locations.length > 0;
+
+ IgfsPath path = new IgfsPath(file);
+
+ Block block = new Block(path, start, len);
+
+ Collection<IgfsBlockLocation> locationsList = new ArrayList<>();
+
+ Collections.addAll(locationsList, locations);
+
+ BLOCK_MAP.put(block, locationsList);
+ }
+
+ /**
+ * Block.
+ */
+ private static class Block {
+ /** */
+ private final IgfsPath path;
+
+ /** */
+ private final long start;
+
+ /** */
+ private final long len;
+
+ /**
+ * Constructor.
+ *
+ * @param path Path.
+ * @param start Start.
+ * @param len Length.
+ */
+ private Block(IgfsPath path, long start, long len) {
+ this.path = path;
+ this.start = start;
+ this.len = len;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("RedundantIfStatement")
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof Block)) return false;
+
+ Block block = (Block) o;
+
+ if (len != block.len)
+ return false;
+
+ if (start != block.start)
+ return false;
+
+ if (!path.equals(block.path))
+ return false;
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = path.hashCode();
+
+ res = 31 * res + (int) (start ^ (start >>> 32));
+ res = 31 * res + (int) (len ^ (len >>> 32));
+
+ return res;
+ }
+ }
+
+ /**
+ * Mocked job.
+ */
+ private static class MockJob implements HadoopJob {
+ /** Reducers count. */
+ private final int reducers;
+
+ /** */
+ private Collection<HadoopInputSplit> splitList;
+
+ /**
+ * Constructor.
+ *
+ * @param reducers Reducers count.
+ * @param splitList Splits.
+ */
+ private MockJob(int reducers, Collection<HadoopInputSplit> splitList) {
+ this.reducers = reducers;
+ this.splitList = splitList;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopJobId id() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopJobInfo info() {
+ return new HadoopDefaultJobInfo() {
+ @Override public int reducers() {
+ return reducers;
+ }
+ };
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<HadoopInputSplit> input() throws IgniteCheckedException {
+ return splitList;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopTaskContext getTaskContext(HadoopTaskInfo info) throws IgniteCheckedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initialize(boolean external, UUID nodeId) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void dispose(boolean external) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void prepareTaskEnvironment(HadoopTaskInfo info) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cleanupTaskEnvironment(HadoopTaskInfo info) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cleanupStagingDirectory() {
+ // No-op.
+ }
+ }
+
+ /**
+ * Mocked IGFS.
+ */
+ private static class MockIgfs implements IgfsEx {
+ /** {@inheritDoc} */
+ @Override public boolean isProxy(URI path) {
+ return PROXY_MAP.containsKey(path) && PROXY_MAP.get(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) {
+ return BLOCK_MAP.get(new Block(path, start, len));
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len,
+ long maxLen) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsContext context() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsPaths proxyPaths() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsInputStreamAdapter open(IgfsPath path, int bufSize, int seqReadsBeforePrefetch) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsInputStreamAdapter open(IgfsPath path) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsInputStreamAdapter open(IgfsPath path, int bufSize) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsStatus globalSpace() throws IgniteCheckedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void globalSampling(@Nullable Boolean val) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Boolean globalSampling() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsLocalMetrics localMetrics() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long groupBlockSize() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteInternalFuture<?> awaitDeletesAsync() throws IgniteCheckedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public String clientLogDirectory() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void clientLogDirectory(String logDir) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean evictExclude(IgfsPath path, boolean primary) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public String name() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileSystemConfiguration configuration() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean exists(IgfsPath path) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public IgfsFile info(IgfsPath path) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsPathSummary summary(IgfsPath path) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public IgfsFile update(IgfsPath path, Map<String, String> props) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void rename(IgfsPath src, IgfsPath dest) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean delete(IgfsPath path, boolean recursive) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdirs(IgfsPath path) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsPath> listPaths(IgfsPath path) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsFile> listFiles(IgfsPath path) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long usedSpaceSize() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsOutputStream create(IgfsPath path, boolean overwrite) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
+ long blockSize, @Nullable Map<String, String> props) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite,
+ @Nullable IgniteUuid affKey, int replication, long blockSize, @Nullable Map<String, String> props) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsOutputStream append(IgfsPath path, boolean create) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsOutputStream append(IgfsPath path, int bufSize, boolean create,
+ @Nullable Map<String, String> props) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setTimes(IgfsPath path, long accessTime, long modificationTime) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsMetrics metrics() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resetMetrics() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public long size(IgfsPath path) {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void format() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
+ Collection<IgfsPath> paths, @Nullable T arg) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
+ Collection<IgfsPath> paths, boolean skipNonExistentFiles, long maxRangeLen, @Nullable T arg) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
+ @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, @Nullable T arg) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
+ @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles,
+ long maxRangeLen, @Nullable T arg) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid nextAffinityKey() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteFileSystem withAsync() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isAsync() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public <R> IgniteFuture<R> future() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsSecondaryFileSystem asSecondary() {
+ return null;
+ }
+ }
+
+ /**
+ * Mocked Grid.
+ */
+ @SuppressWarnings("ExternalizableWithoutPublicNoArgConstructor")
+ private static class MockIgnite extends IgniteSpringBean implements IgniteEx {
+ /** {@inheritDoc} */
+ @Override public IgniteClusterEx cluster() {
+ return (IgniteClusterEx)super.cluster();
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteFileSystem igfsx(String name) {
+ assert F.eq("igfs", name);
+
+ return IGFS;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Hadoop hadoop() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String name() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K extends GridCacheUtilityKey, V> GridCacheProjectionEx<K, V> utilityCache(Class<K> keyCls,
+ Class<V> valCls) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public <K, V> GridCache<K, V> cachex(@Nullable String name) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public <K, V> GridCache<K, V> cachex() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public Collection<GridCache<?, ?>> cachesx(@Nullable IgnitePredicate<? super GridCache<?, ?>>... p) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean eventUserRecordable(int type) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean allEventsUserRecordable(int[] types) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<String> compatibleVersions() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isJmxRemoteEnabled() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isRestartEnabled() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public ClusterNode localNode() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String latestVersion() {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopFileSystemsTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopFileSystemsTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopFileSystemsTest.java
new file mode 100644
index 0000000..8cf31a2
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopFileSystemsTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.ignite.internal.processors.hadoop.fs.*;
+import org.apache.ignite.testframework.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+
+/**
+ * Test file systems for the working directory multi-threading support.
+ */
+public class HadoopFileSystemsTest extends HadoopAbstractSelfTest {
+ private static final int THREAD_COUNT = 3;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ startGrids(gridCount());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean igfsEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int gridCount() {
+ return 1;
+ }
+
+
+ /**
+ * Test the file system with specified URI for the multi-thread working directory support.
+ *
+ * @param uri Base URI of the file system (scheme and authority).
+ * @throws Exception If fails.
+ */
+ private void testFileSystem(final URI uri) throws Exception {
+ final Configuration cfg = new Configuration();
+
+ setupFileSystems(cfg);
+
+ cfg.set(HadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP,
+ new Path(new Path(uri), "user/" + System.getProperty("user.name")).toString());
+
+ final CountDownLatch changeUserPhase = new CountDownLatch(THREAD_COUNT);
+ final CountDownLatch changeDirPhase = new CountDownLatch(THREAD_COUNT);
+ final CountDownLatch changeAbsDirPhase = new CountDownLatch(THREAD_COUNT);
+ final CountDownLatch finishPhase = new CountDownLatch(THREAD_COUNT);
+
+ final Path[] newUserInitWorkDir = new Path[THREAD_COUNT];
+ final Path[] newWorkDir = new Path[THREAD_COUNT];
+ final Path[] newAbsWorkDir = new Path[THREAD_COUNT];
+ final Path[] newInstanceWorkDir = new Path[THREAD_COUNT];
+
+ final AtomicInteger threadNum = new AtomicInteger(0);
+
+ GridTestUtils.runMultiThreadedAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ int curThreadNum = threadNum.getAndIncrement();
+
+ FileSystem fs = FileSystem.get(uri, cfg);
+
+ HadoopFileSystemsUtils.setUser(fs, "user" + curThreadNum);
+
+ if ("file".equals(uri.getScheme()))
+ FileSystem.get(uri, cfg).setWorkingDirectory(new Path("file:///user/user" + curThreadNum));
+
+ changeUserPhase.countDown();
+ changeUserPhase.await();
+
+ newUserInitWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory();
+
+ FileSystem.get(uri, cfg).setWorkingDirectory(new Path("folder" + curThreadNum));
+
+ changeDirPhase.countDown();
+ changeDirPhase.await();
+
+ newWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory();
+
+ FileSystem.get(uri, cfg).setWorkingDirectory(new Path("/folder" + curThreadNum));
+
+ changeAbsDirPhase.countDown();
+ changeAbsDirPhase.await();
+
+ newAbsWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory();
+
+ newInstanceWorkDir[curThreadNum] = FileSystem.newInstance(uri, cfg).getWorkingDirectory();
+
+ finishPhase.countDown();
+ }
+ catch (InterruptedException | IOException e) {
+ error("Failed to execute test thread.", e);
+
+ fail();
+ }
+ }
+ }, THREAD_COUNT, "filesystems-test");
+
+ finishPhase.await();
+
+ for (int i = 0; i < THREAD_COUNT; i ++) {
+ cfg.set(MRJobConfig.USER_NAME, "user" + i);
+
+ Path workDir = new Path(new Path(uri), "user/user" + i);
+
+ cfg.set(HadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP, workDir.toString());
+
+ assertEquals(workDir, FileSystem.newInstance(uri, cfg).getWorkingDirectory());
+
+ assertEquals(workDir, newUserInitWorkDir[i]);
+
+ assertEquals(new Path(new Path(uri), "user/user" + i + "/folder" + i), newWorkDir[i]);
+
+ assertEquals(new Path("/folder" + i), newAbsWorkDir[i]);
+
+ assertEquals(new Path(new Path(uri), "user/" + System.getProperty("user.name")), newInstanceWorkDir[i]);
+ }
+
+ System.out.println(System.getProperty("user.dir"));
+ }
+
+ /**
+ * Test IGFS multi-thread working directory.
+ *
+ * @throws Exception If fails.
+ */
+ public void testIgfs() throws Exception {
+ testFileSystem(URI.create(igfsScheme()));
+ }
+
+ /**
+ * Test HDFS multi-thread working directory.
+ *
+ * @throws Exception If fails.
+ */
+ public void testHdfs() throws Exception {
+ testFileSystem(URI.create("hdfs://localhost/"));
+ }
+
+ /**
+ * Test LocalFS multi-thread working directory.
+ *
+ * @throws Exception If fails.
+ */
+ public void testLocal() throws Exception {
+ testFileSystem(URI.create("file:///"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopGroupingTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopGroupingTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopGroupingTest.java
new file mode 100644
index 0000000..e385ca7
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopGroupingTest.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Grouping test.
+ */
+public class HadoopGroupingTest extends HadoopAbstractSelfTest {
+ /** */
+ private static final String PATH_OUTPUT = "/test-out";
+
+ /** */
+ private static final GridConcurrentHashSet<UUID> vals = HadoopSharedMap.map(HadoopGroupingTest.class)
+ .put("vals", new GridConcurrentHashSet<UUID>());
+
+ /** {@inheritDoc} */
+ @Override protected int gridCount() {
+ return 3;
+ }
+
+ /** {@inheritDoc} */
+ protected boolean igfsEnabled() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ startGrids(gridCount());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopConfiguration hadoopConfiguration(String gridName) {
+ HadoopConfiguration cfg = super.hadoopConfiguration(gridName);
+
+ cfg.setExternalExecution(false);
+
+ return cfg;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testGroupingReducer() throws Exception {
+ doTestGrouping(false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testGroupingCombiner() throws Exception {
+ doTestGrouping(true);
+ }
+
+ /**
+ * @param combiner With combiner.
+ * @throws Exception If failed.
+ */
+ public void doTestGrouping(boolean combiner) throws Exception {
+ vals.clear();
+
+ Job job = Job.getInstance();
+
+ job.setInputFormatClass(InFormat.class);
+ job.setOutputFormatClass(OutFormat.class);
+
+ job.setOutputKeyClass(YearTemperature.class);
+ job.setOutputValueClass(Text.class);
+
+ job.setMapperClass(Mapper.class);
+
+ if (combiner) {
+ job.setCombinerClass(MyReducer.class);
+ job.setNumReduceTasks(0);
+ job.setCombinerKeyGroupingComparatorClass(YearComparator.class);
+ }
+ else {
+ job.setReducerClass(MyReducer.class);
+ job.setNumReduceTasks(4);
+ job.setGroupingComparatorClass(YearComparator.class);
+ }
+
+ grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 2),
+ createJobInfo(job.getConfiguration())).get(30000);
+
+ assertTrue(vals.isEmpty());
+ }
+
+ public static class MyReducer extends Reducer<YearTemperature, Text, Text, Object> {
+ /** */
+ int lastYear;
+
+ @Override protected void reduce(YearTemperature key, Iterable<Text> vals0, Context context)
+ throws IOException, InterruptedException {
+ X.println("___ : " + context.getTaskAttemptID() + " --> " + key);
+
+ Set<UUID> ids = new HashSet<>();
+
+ for (Text val : vals0)
+ assertTrue(ids.add(UUID.fromString(val.toString())));
+
+ for (Text val : vals0)
+ assertTrue(ids.remove(UUID.fromString(val.toString())));
+
+ assertTrue(ids.isEmpty());
+
+ assertTrue(key.year > lastYear);
+
+ lastYear = key.year;
+
+ for (Text val : vals0)
+ assertTrue(vals.remove(UUID.fromString(val.toString())));
+ }
+ }
+
+ public static class YearComparator implements RawComparator<YearTemperature> { // Grouping comparator.
+ /** {@inheritDoc} */
+ @Override public int compare(YearTemperature o1, YearTemperature o2) {
+ return Integer.compare(o1.year, o2.year);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ throw new IllegalStateException();
+ }
+ }
+
+ public static class YearTemperature implements WritableComparable<YearTemperature>, Cloneable {
+ /** */
+ private int year;
+
+ /** */
+ private int temperature;
+
+ /** {@inheritDoc} */
+ @Override public void write(DataOutput out) throws IOException {
+ out.writeInt(year);
+ out.writeInt(temperature);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFields(DataInput in) throws IOException {
+ year = in.readInt();
+ temperature = in.readInt();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ throw new IllegalStateException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() { // To be partitioned by year.
+ return year;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compareTo(YearTemperature o) {
+ int res = Integer.compare(year, o.year);
+
+ if (res != 0)
+ return res;
+
+ // Sort comparator by year and temperature, to find max for year.
+ return Integer.compare(o.temperature, temperature);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(YearTemperature.class, this);
+ }
+ }
+
+ public static class InFormat extends InputFormat<YearTemperature, Text> {
+ /** {@inheritDoc} */
+ @Override public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
+ ArrayList<InputSplit> list = new ArrayList<>();
+
+ for (int i = 0; i < 10; i++)
+ list.add(new HadoopSortingTest.FakeSplit(20));
+
+ return list;
+ }
+
+ /** {@inheritDoc} */
+ @Override public RecordReader<YearTemperature, Text> createRecordReader(final InputSplit split,
+ TaskAttemptContext context) throws IOException, InterruptedException {
+ return new RecordReader<YearTemperature, Text>() {
+ /** */
+ int cnt;
+
+ /** */
+ Random rnd = new GridRandom();
+
+ /** */
+ YearTemperature key = new YearTemperature();
+
+ /** */
+ Text val = new Text();
+
+ @Override public void initialize(InputSplit split, TaskAttemptContext context) {
+ // No-op.
+ }
+
+ @Override public boolean nextKeyValue() throws IOException, InterruptedException {
+ return cnt++ < split.getLength();
+ }
+
+ @Override public YearTemperature getCurrentKey() {
+ key.year = 1990 + rnd.nextInt(10);
+ key.temperature = 10 + rnd.nextInt(20);
+
+ return key;
+ }
+
+ @Override public Text getCurrentValue() {
+ UUID id = UUID.randomUUID();
+
+ assertTrue(vals.add(id));
+
+ val.set(id.toString());
+
+ return val;
+ }
+
+ @Override public float getProgress() {
+ return 0;
+ }
+
+ @Override public void close() {
+ // No-op.
+ }
+ };
+ }
+ }
+
+ /**
+ *
+ */
+ public static class OutFormat extends OutputFormat {
+ /** {@inheritDoc} */
+ @Override public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopJobTrackerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopJobTrackerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopJobTrackerSelfTest.java
new file mode 100644
index 0000000..943d89f
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopJobTrackerSelfTest.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.*;
+import org.apache.hadoop.mapreduce.lib.output.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Job tracker self test.
+ */
+public class HadoopJobTrackerSelfTest extends HadoopAbstractSelfTest {
+ /** */
+ private static final String PATH_OUTPUT = "/test-out";
+
+ /** Test block count parameter name. */
+ private static final int BLOCK_CNT = 10;
+
+ /** */
+ private static HadoopSharedMap m = HadoopSharedMap.map(HadoopJobTrackerSelfTest.class);
+
+ /** Map task execution count. */
+ private static final AtomicInteger mapExecCnt = m.put("mapExecCnt", new AtomicInteger());
+
+ /** Reduce task execution count. */
+ private static final AtomicInteger reduceExecCnt = m.put("reduceExecCnt", new AtomicInteger());
+
+ /** Reduce task execution count. */
+ private static final AtomicInteger combineExecCnt = m.put("combineExecCnt", new AtomicInteger());
+
+ /** */
+ private static final Map<String, CountDownLatch> latch = m.put("latch", new HashMap<String, CountDownLatch>());
+
+ /** {@inheritDoc} */
+ @Override protected boolean igfsEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ startGrids(gridCount());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ super.afterTestsStopped();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ latch.put("mapAwaitLatch", new CountDownLatch(1));
+ latch.put("reduceAwaitLatch", new CountDownLatch(1));
+ latch.put("combineAwaitLatch", new CountDownLatch(1));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ mapExecCnt.set(0);
+ combineExecCnt.set(0);
+ reduceExecCnt.set(0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopConfiguration hadoopConfiguration(String gridName) {
+ HadoopConfiguration cfg = super.hadoopConfiguration(gridName);
+
+ cfg.setMapReducePlanner(new HadoopTestRoundRobinMrPlanner());
+ cfg.setExternalExecution(false);
+
+ return cfg;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSimpleTaskSubmit() throws Exception {
+ try {
+ UUID globalId = UUID.randomUUID();
+
+ Job job = Job.getInstance();
+ setupFileSystems(job.getConfiguration());
+
+ job.setMapperClass(TestMapper.class);
+ job.setReducerClass(TestReducer.class);
+ job.setInputFormatClass(InFormat.class);
+
+ FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT + "1"));
+
+ HadoopJobId jobId = new HadoopJobId(globalId, 1);
+
+ grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));
+
+ checkStatus(jobId, false);
+
+ info("Releasing map latch.");
+
+ latch.get("mapAwaitLatch").countDown();
+
+ checkStatus(jobId, false);
+
+ info("Releasing reduce latch.");
+
+ latch.get("reduceAwaitLatch").countDown();
+
+ checkStatus(jobId, true);
+
+ assertEquals(10, mapExecCnt.get());
+ assertEquals(0, combineExecCnt.get());
+ assertEquals(1, reduceExecCnt.get());
+ }
+ finally {
+ // Safety.
+ latch.get("mapAwaitLatch").countDown();
+ latch.get("combineAwaitLatch").countDown();
+ latch.get("reduceAwaitLatch").countDown();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testTaskWithCombinerPerMap() throws Exception {
+ try {
+ UUID globalId = UUID.randomUUID();
+
+ Job job = Job.getInstance();
+ setupFileSystems(job.getConfiguration());
+
+ job.setMapperClass(TestMapper.class);
+ job.setReducerClass(TestReducer.class);
+ job.setCombinerClass(TestCombiner.class);
+ job.setInputFormatClass(InFormat.class);
+
+ FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT + "2"));
+
+ HadoopJobId jobId = new HadoopJobId(globalId, 1);
+
+ grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));
+
+ checkStatus(jobId, false);
+
+ info("Releasing map latch.");
+
+ latch.get("mapAwaitLatch").countDown();
+
+ checkStatus(jobId, false);
+
+ // All maps are completed. We have a combiner, so no reducers should be executed
+ // before combiner latch is released.
+
+ U.sleep(50);
+
+ assertEquals(0, reduceExecCnt.get());
+
+ info("Releasing combiner latch.");
+
+ latch.get("combineAwaitLatch").countDown();
+
+ checkStatus(jobId, false);
+
+ info("Releasing reduce latch.");
+
+ latch.get("reduceAwaitLatch").countDown();
+
+ checkStatus(jobId, true);
+
+ assertEquals(10, mapExecCnt.get());
+ assertEquals(10, combineExecCnt.get());
+ assertEquals(1, reduceExecCnt.get());
+ }
+ finally {
+ // Safety.
+ latch.get("mapAwaitLatch").countDown();
+ latch.get("combineAwaitLatch").countDown();
+ latch.get("reduceAwaitLatch").countDown();
+ }
+ }
+
+ /**
+ * Checks job execution status.
+ *
+ * @param jobId Job ID.
+ * @param complete Completion status.
+ * @throws Exception If failed.
+ */
+ private void checkStatus(HadoopJobId jobId, boolean complete) throws Exception {
+ for (int i = 0; i < gridCount(); i++) {
+ IgniteKernal kernal = (IgniteKernal)grid(i);
+
+ Hadoop hadoop = kernal.hadoop();
+
+ HadoopJobStatus stat = hadoop.status(jobId);
+
+ assert stat != null;
+
+ IgniteInternalFuture<?> fut = hadoop.finishFuture(jobId);
+
+ if (!complete)
+ assertFalse(fut.isDone());
+ else {
+ info("Waiting for status future completion on node [idx=" + i + ", nodeId=" +
+ kernal.getLocalNodeId() + ']');
+
+ fut.get();
+ }
+ }
+ }
+
+ /**
+ * Test input format
+ */
+ public static class InFormat extends InputFormat {
+
+ @Override public List<InputSplit> getSplits(JobContext ctx) throws IOException, InterruptedException {
+ List<InputSplit> res = new ArrayList<>(BLOCK_CNT);
+
+ for (int i = 0; i < BLOCK_CNT; i++)
+ try {
+ res.add(new FileSplit(new Path(new URI("someFile")), i, i + 1, new String[] {"localhost"}));
+ }
+ catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+
+ return res;
+ }
+
+ @Override public RecordReader createRecordReader(InputSplit split, TaskAttemptContext ctx) throws IOException, InterruptedException {
+ return new RecordReader() {
+ @Override public void initialize(InputSplit split, TaskAttemptContext ctx) {
+ }
+
+ @Override public boolean nextKeyValue() {
+ return false;
+ }
+
+ @Override public Object getCurrentKey() {
+ return null;
+ }
+
+ @Override public Object getCurrentValue() {
+ return null;
+ }
+
+ @Override public float getProgress() {
+ return 0;
+ }
+
+ @Override public void close() {
+
+ }
+ };
+ }
+ }
+
+ /**
+ * Test mapper.
+ */
+ private static class TestMapper extends Mapper {
+ @Override public void run(Context ctx) throws IOException, InterruptedException {
+ System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId());
+
+ latch.get("mapAwaitLatch").await();
+
+ mapExecCnt.incrementAndGet();
+
+ System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId());
+ }
+ }
+
+ /**
+ * Test reducer.
+ */
+ private static class TestReducer extends Reducer {
+ @Override public void run(Context ctx) throws IOException, InterruptedException {
+ System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId());
+
+ latch.get("reduceAwaitLatch").await();
+
+ reduceExecCnt.incrementAndGet();
+
+ System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId());
+ }
+ }
+
+ /**
+ * Test combiner.
+ */
+ private static class TestCombiner extends Reducer {
+ @Override public void run(Context ctx) throws IOException, InterruptedException {
+ System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId());
+
+ latch.get("combineAwaitLatch").await();
+
+ combineExecCnt.incrementAndGet();
+
+ System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceEmbeddedSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceEmbeddedSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceEmbeddedSelfTest.java
new file mode 100644
index 0000000..4a6e1ef
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceEmbeddedSelfTest.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.io.serializer.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.examples.*;
+
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Tests map-reduce execution with embedded mode.
+ */
+public class HadoopMapReduceEmbeddedSelfTest extends HadoopMapReduceTest {
+ /** */
+ private static Map<String, Boolean> flags = HadoopSharedMap.map(HadoopMapReduceEmbeddedSelfTest.class)
+ .put("flags", new HashMap<String, Boolean>());
+
+ /** {@inheritDoc} */
+ @Override public HadoopConfiguration hadoopConfiguration(String gridName) {
+ HadoopConfiguration cfg = super.hadoopConfiguration(gridName);
+
+ cfg.setExternalExecution(false);
+
+ return cfg;
+ }
+
+ /**
+ * Tests whole job execution with all phases in old and new versions of API with definition of custom
+ * Serialization, Partitioner and IO formats.
+ * @throws Exception If fails.
+ */
+ public void testMultiReducerWholeMapReduceExecution() throws Exception {
+ IgfsPath inDir = new IgfsPath(PATH_INPUT);
+
+ igfs.mkdirs(inDir);
+
+ IgfsPath inFile = new IgfsPath(inDir, HadoopWordCount2.class.getSimpleName() + "-input");
+
+ generateTestFile(inFile.toString(), "key1", 10000, "key2", 20000, "key3", 15000, "key4", 7000, "key5", 12000,
+ "key6", 18000 );
+
+ for (int i = 0; i < 2; i++) {
+ boolean useNewAPI = i == 1;
+
+ igfs.delete(new IgfsPath(PATH_OUTPUT), true);
+
+ flags.put("serializationWasConfigured", false);
+ flags.put("partitionerWasConfigured", false);
+ flags.put("inputFormatWasConfigured", false);
+ flags.put("outputFormatWasConfigured", false);
+
+ JobConf jobConf = new JobConf();
+
+ jobConf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, CustomSerialization.class.getName());
+
+ //To split into about 6-7 items for v2
+ jobConf.setInt(FileInputFormat.SPLIT_MAXSIZE, 65000);
+
+ //For v1
+ jobConf.setInt("fs.local.block.size", 65000);
+
+ // File system coordinates.
+ setupFileSystems(jobConf);
+
+ HadoopWordCount1.setTasksClasses(jobConf, !useNewAPI, !useNewAPI, !useNewAPI);
+
+ if (!useNewAPI) {
+ jobConf.setPartitionerClass(CustomV1Partitioner.class);
+ jobConf.setInputFormat(CustomV1InputFormat.class);
+ jobConf.setOutputFormat(CustomV1OutputFormat.class);
+ }
+
+ Job job = Job.getInstance(jobConf);
+
+ HadoopWordCount2.setTasksClasses(job, useNewAPI, useNewAPI, useNewAPI);
+
+ if (useNewAPI) {
+ job.setPartitionerClass(CustomV2Partitioner.class);
+ job.setInputFormatClass(CustomV2InputFormat.class);
+ job.setOutputFormatClass(CustomV2OutputFormat.class);
+ }
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ FileInputFormat.setInputPaths(job, new Path(igfsScheme() + inFile.toString()));
+ FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT));
+
+ job.setNumReduceTasks(3);
+
+ job.setJarByClass(HadoopWordCount2.class);
+
+ IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1),
+ createJobInfo(job.getConfiguration()));
+
+ fut.get();
+
+ assertTrue("Serialization was configured (new API is " + useNewAPI + ")",
+ flags.get("serializationWasConfigured"));
+
+ assertTrue("Partitioner was configured (new API is = " + useNewAPI + ")",
+ flags.get("partitionerWasConfigured"));
+
+ assertTrue("Input format was configured (new API is = " + useNewAPI + ")",
+ flags.get("inputFormatWasConfigured"));
+
+ assertTrue("Output format was configured (new API is = " + useNewAPI + ")",
+ flags.get("outputFormatWasConfigured"));
+
+ assertEquals("Use new API = " + useNewAPI,
+ "key3\t15000\n" +
+ "key6\t18000\n",
+ readAndSortFile(PATH_OUTPUT + "/" + (useNewAPI ? "part-r-" : "part-") + "00000")
+ );
+
+ assertEquals("Use new API = " + useNewAPI,
+ "key1\t10000\n" +
+ "key4\t7000\n",
+ readAndSortFile(PATH_OUTPUT + "/" + (useNewAPI ? "part-r-" : "part-") + "00001")
+ );
+
+ assertEquals("Use new API = " + useNewAPI,
+ "key2\t20000\n" +
+ "key5\t12000\n",
+ readAndSortFile(PATH_OUTPUT + "/" + (useNewAPI ? "part-r-" : "part-") + "00002")
+ );
+
+ }
+ }
+
+ /**
+ * Custom serialization class that inherits behaviour of native {@link WritableSerialization}.
+ */
+ protected static class CustomSerialization extends WritableSerialization {
+ @Override public void setConf(Configuration conf) {
+ super.setConf(conf);
+
+ flags.put("serializationWasConfigured", true);
+ }
+ }
+
+ /**
+ * Custom implementation of Partitioner in v1 API.
+ */
+ private static class CustomV1Partitioner extends org.apache.hadoop.mapred.lib.HashPartitioner {
+ /** {@inheritDoc} */
+ @Override public void configure(JobConf job) {
+ flags.put("partitionerWasConfigured", true);
+ }
+ }
+
+ /**
+ * Custom implementation of Partitioner in v2 API.
+ */
+ private static class CustomV2Partitioner extends org.apache.hadoop.mapreduce.lib.partition.HashPartitioner
+ implements Configurable {
+ /** {@inheritDoc} */
+ @Override public void setConf(Configuration conf) {
+ flags.put("partitionerWasConfigured", true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Configuration getConf() {
+ return null;
+ }
+ }
+
+ /**
+ * Custom implementation of InputFormat in v2 API.
+ */
+ private static class CustomV2InputFormat extends org.apache.hadoop.mapreduce.lib.input.TextInputFormat implements Configurable {
+ /** {@inheritDoc} */
+ @Override public void setConf(Configuration conf) {
+ flags.put("inputFormatWasConfigured", true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Configuration getConf() {
+ return null;
+ }
+ }
+
+ /**
+ * Custom implementation of OutputFormat in v2 API.
+ */
+ private static class CustomV2OutputFormat extends org.apache.hadoop.mapreduce.lib.output.TextOutputFormat implements Configurable {
+ /** {@inheritDoc} */
+ @Override public void setConf(Configuration conf) {
+ flags.put("outputFormatWasConfigured", true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Configuration getConf() {
+ return null;
+ }
+ }
+
+ /**
+ * Custom implementation of InputFormat in v1 API.
+ */
+ private static class CustomV1InputFormat extends org.apache.hadoop.mapred.TextInputFormat {
+ /** {@inheritDoc} */
+ @Override public void configure(JobConf job) {
+ super.configure(job);
+
+ flags.put("inputFormatWasConfigured", true);
+ }
+ }
+
+ /**
+ * Custom implementation of OutputFormat in v1 API.
+ */
+ private static class CustomV1OutputFormat extends org.apache.hadoop.mapred.TextOutputFormat implements JobConfigurable {
+ /** {@inheritDoc} */
+ @Override public void configure(JobConf job) {
+ flags.put("outputFormatWasConfigured", true);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java
new file mode 100644
index 0000000..6242ecc
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.ignite.*;
+import org.apache.ignite.hadoop.fs.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
+import org.apache.ignite.internal.processors.hadoop.examples.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.testframework.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Test of whole cycle of map-reduce processing via Job tracker.
+ */
+public class HadoopMapReduceTest extends HadoopAbstractWordCountTest {
+ /** {@inheritDoc} */
+ @Override protected int gridCount() {
+ return 3;
+ }
+
+ /**
+ * Tests whole job execution with all phases in all combination of new and old versions of API.
+ * @throws Exception If fails.
+ */
+ public void testWholeMapReduceExecution() throws Exception {
+ IgfsPath inDir = new IgfsPath(PATH_INPUT);
+
+ igfs.mkdirs(inDir);
+
+ IgfsPath inFile = new IgfsPath(inDir, HadoopWordCount2.class.getSimpleName() + "-input");
+
+ generateTestFile(inFile.toString(), "red", 100000, "blue", 200000, "green", 150000, "yellow", 70000 );
+
+ for (int i = 0; i < 8; i++) {
+ igfs.delete(new IgfsPath(PATH_OUTPUT), true);
+
+ boolean useNewMapper = (i & 1) == 0;
+ boolean useNewCombiner = (i & 2) == 0;
+ boolean useNewReducer = (i & 4) == 0;
+
+ JobConf jobConf = new JobConf();
+
+ jobConf.set(JOB_COUNTER_WRITER_PROPERTY, IgniteHadoopFileSystemCounterWriter.class.getName());
+ jobConf.setUser("yyy");
+ jobConf.set(IgniteHadoopFileSystemCounterWriter.COUNTER_WRITER_DIR_PROPERTY, "/xxx/${USER}/zzz");
+
+ //To split into about 40 items for v2
+ jobConf.setInt(FileInputFormat.SPLIT_MAXSIZE, 65000);
+
+ //For v1
+ jobConf.setInt("fs.local.block.size", 65000);
+
+ // File system coordinates.
+ setupFileSystems(jobConf);
+
+ HadoopWordCount1.setTasksClasses(jobConf, !useNewMapper, !useNewCombiner, !useNewReducer);
+
+ Job job = Job.getInstance(jobConf);
+
+ HadoopWordCount2.setTasksClasses(job, useNewMapper, useNewCombiner, useNewReducer);
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ FileInputFormat.setInputPaths(job, new Path(igfsScheme() + inFile.toString()));
+ FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT));
+
+ job.setJarByClass(HadoopWordCount2.class);
+
+ HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 1);
+
+ IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));
+
+ fut.get();
+
+ checkJobStatistics(jobId);
+
+ assertEquals("Use new mapper: " + useNewMapper + ", new combiner: " + useNewCombiner + ", new reducer: " +
+ useNewReducer,
+ "blue\t200000\n" +
+ "green\t150000\n" +
+ "red\t100000\n" +
+ "yellow\t70000\n",
+ readAndSortFile(PATH_OUTPUT + "/" + (useNewReducer ? "part-r-" : "part-") + "00000")
+ );
+ }
+ }
+
+ /**
+ * Simple test job statistics.
+ *
+ * @param jobId Job id.
+ * @throws IgniteCheckedException
+ */
+ private void checkJobStatistics(HadoopJobId jobId) throws IgniteCheckedException, IOException {
+ HadoopCounters cntrs = grid(0).hadoop().counters(jobId);
+
+ HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(cntrs, null);
+
+ Map<String, SortedMap<Integer,Long>> tasks = new TreeMap<>();
+
+ Map<String, Integer> phaseOrders = new HashMap<>();
+ phaseOrders.put("submit", 0);
+ phaseOrders.put("prepare", 1);
+ phaseOrders.put("start", 2);
+ phaseOrders.put("Cstart", 3);
+ phaseOrders.put("finish", 4);
+
+ String prevTaskId = null;
+
+ long apiEvtCnt = 0;
+
+ for (T2<String, Long> evt : perfCntr.evts()) {
+ //We expect string pattern: COMBINE 1 run 7fa86a14-5a08-40e3-a7cb-98109b52a706
+ String[] parsedEvt = evt.get1().split(" ");
+
+ String taskId;
+ String taskPhase;
+
+ if ("JOB".equals(parsedEvt[0])) {
+ taskId = parsedEvt[0];
+ taskPhase = parsedEvt[1];
+ }
+ else {
+ taskId = ("COMBINE".equals(parsedEvt[0]) ? "MAP" : parsedEvt[0].substring(0, 3)) + parsedEvt[1];
+ taskPhase = ("COMBINE".equals(parsedEvt[0]) ? "C" : "") + parsedEvt[2];
+ }
+
+ if (!taskId.equals(prevTaskId))
+ tasks.put(taskId, new TreeMap<Integer,Long>());
+
+ Integer pos = phaseOrders.get(taskPhase);
+
+ assertNotNull("Invalid phase " + taskPhase, pos);
+
+ tasks.get(taskId).put(pos, evt.get2());
+
+ prevTaskId = taskId;
+
+ apiEvtCnt++;
+ }
+
+ for (Map.Entry<String ,SortedMap<Integer,Long>> task : tasks.entrySet()) {
+ Map<Integer, Long> order = task.getValue();
+
+ long prev = 0;
+
+ for (Map.Entry<Integer, Long> phase : order.entrySet()) {
+ assertTrue("Phase order of " + task.getKey() + " is invalid", phase.getValue() >= prev);
+
+ prev = phase.getValue();
+ }
+ }
+
+ final IgfsPath statPath = new IgfsPath("/xxx/yyy/zzz/" + jobId + "/performance");
+
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return igfs.exists(statPath);
+ }
+ }, 10000);
+
+ BufferedReader reader = new BufferedReader(new InputStreamReader(igfs.open(statPath)));
+
+ assertEquals(apiEvtCnt, HadoopTestUtils.simpleCheckJobStatFile(reader));
+ }
+}
[20/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
new file mode 100644
index 0000000..e95b8cb
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
@@ -0,0 +1,440 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.child;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.processors.hadoop.shuffle.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.*;
+import org.apache.ignite.internal.util.future.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopTaskType.*;
+
+/**
+ * Hadoop process base.
+ */
+@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
+public class HadoopChildProcessRunner {
+ /** Node process descriptor. */
+ private HadoopProcessDescriptor nodeDesc;
+
+ /** Message processing executor service. */
+ private ExecutorService msgExecSvc;
+
+ /** Task executor service. */
+ private HadoopExecutorService execSvc;
+
+ /** */
+ protected GridUnsafeMemory mem = new GridUnsafeMemory(0);
+
+ /** External communication. */
+ private HadoopExternalCommunication comm;
+
+ /** Logger. */
+ private IgniteLogger log;
+
+ /** Init guard. */
+ private final AtomicBoolean initGuard = new AtomicBoolean();
+
+ /** Start time. */
+ private long startTime;
+
+ /** Init future. */
+ private final GridFutureAdapterEx<?> initFut = new GridFutureAdapterEx<>();
+
+ /** Job instance. */
+ private HadoopJob job;
+
+ /** Number of uncompleted tasks. */
+ private final AtomicInteger pendingTasks = new AtomicInteger();
+
+ /** Shuffle job. */
+ private HadoopShuffleJob<HadoopProcessDescriptor> shuffleJob;
+
+ /** Concurrent mappers. */
+ private int concMappers;
+
+ /** Concurrent reducers. */
+ private int concReducers;
+
+ /**
+ * Starts child process runner.
+ */
+ public void start(HadoopExternalCommunication comm, HadoopProcessDescriptor nodeDesc,
+ ExecutorService msgExecSvc, IgniteLogger parentLog)
+ throws IgniteCheckedException {
+ this.comm = comm;
+ this.nodeDesc = nodeDesc;
+ this.msgExecSvc = msgExecSvc;
+
+ comm.setListener(new MessageListener());
+ log = parentLog.getLogger(HadoopChildProcessRunner.class);
+
+ startTime = U.currentTimeMillis();
+
+ // At this point node knows that this process has started.
+ comm.sendMessage(this.nodeDesc, new HadoopProcessStartedAck());
+ }
+
+ /**
+ * Initializes process for task execution.
+ *
+ * @param req Initialization request.
+ */
+ private void prepareProcess(HadoopPrepareForJobRequest req) {
+ if (initGuard.compareAndSet(false, true)) {
+ try {
+ if (log.isDebugEnabled())
+ log.debug("Initializing external hadoop task: " + req);
+
+ assert job == null;
+
+ job = req.jobInfo().createJob(req.jobId(), log);
+
+ job.initialize(true, nodeDesc.processId());
+
+ shuffleJob = new HadoopShuffleJob<>(comm.localProcessDescriptor(), log, job, mem,
+ req.totalReducerCount(), req.localReducers());
+
+ initializeExecutors(req);
+
+ if (log.isDebugEnabled())
+ log.debug("External process initialized [initWaitTime=" +
+ (U.currentTimeMillis() - startTime) + ']');
+
+ initFut.onDone(null, null);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to initialize process: " + req, e);
+
+ initFut.onDone(e);
+ }
+ }
+ else
+ log.warning("Duplicate initialize process request received (will ignore): " + req);
+ }
+
+ /**
+ * @param req Task execution request.
+ */
+ private void runTasks(final HadoopTaskExecutionRequest req) {
+ if (!initFut.isDone() && log.isDebugEnabled())
+ log.debug("Will wait for process initialization future completion: " + req);
+
+ initFut.listenAsync(new CI1<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> f) {
+ try {
+ // Make sure init was successful.
+ f.get();
+
+ boolean set = pendingTasks.compareAndSet(0, req.tasks().size());
+
+ assert set;
+
+ HadoopTaskInfo info = F.first(req.tasks());
+
+ assert info != null;
+
+ int size = info.type() == MAP ? concMappers : concReducers;
+
+// execSvc.setCorePoolSize(size);
+// execSvc.setMaximumPoolSize(size);
+
+ if (log.isDebugEnabled())
+ log.debug("Set executor service size for task type [type=" + info.type() +
+ ", size=" + size + ']');
+
+ for (HadoopTaskInfo taskInfo : req.tasks()) {
+ if (log.isDebugEnabled())
+ log.debug("Submitted task for external execution: " + taskInfo);
+
+ execSvc.submit(new HadoopRunnableTask(log, job, mem, taskInfo, nodeDesc.parentNodeId()) {
+ @Override protected void onTaskFinished(HadoopTaskStatus status) {
+ onTaskFinished0(this, status);
+ }
+
+ @Override protected HadoopTaskInput createInput(HadoopTaskContext ctx)
+ throws IgniteCheckedException {
+ return shuffleJob.input(ctx);
+ }
+
+ @Override protected HadoopTaskOutput createOutput(HadoopTaskContext ctx)
+ throws IgniteCheckedException {
+ return shuffleJob.output(ctx);
+ }
+ });
+ }
+ }
+ catch (IgniteCheckedException e) {
+ for (HadoopTaskInfo info : req.tasks())
+ notifyTaskFinished(info, new HadoopTaskStatus(HadoopTaskState.FAILED, e), false);
+ }
+ }
+ });
+ }
+
+ /**
+ * Creates executor services.
+ *
+ * @param req Init child process request.
+ */
+ private void initializeExecutors(HadoopPrepareForJobRequest req) {
+ int cpus = Runtime.getRuntime().availableProcessors();
+//
+// concMappers = get(req.jobInfo(), EXTERNAL_CONCURRENT_MAPPERS, cpus);
+// concReducers = get(req.jobInfo(), EXTERNAL_CONCURRENT_REDUCERS, cpus);
+
+ execSvc = new HadoopExecutorService(log, "", cpus * 2, 1024);
+ }
+
+ /**
+ * Updates external process map so that shuffle can proceed with sending messages to reducers.
+ *
+ * @param req Update request.
+ */
+ private void updateTasks(final HadoopJobInfoUpdateRequest req) {
+ initFut.listenAsync(new CI1<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> gridFut) {
+ assert initGuard.get();
+
+ assert req.jobId().equals(job.id());
+
+ if (req.reducersAddresses() != null) {
+ if (shuffleJob.initializeReduceAddresses(req.reducersAddresses())) {
+ shuffleJob.startSending("external",
+ new IgniteInClosure2X<HadoopProcessDescriptor, HadoopShuffleMessage>() {
+ @Override public void applyx(HadoopProcessDescriptor dest,
+ HadoopShuffleMessage msg) throws IgniteCheckedException {
+ comm.sendMessage(dest, msg);
+ }
+ });
+ }
+ }
+ }
+ });
+ }
+
+ /**
+ * Stops all executors and running tasks.
+ */
+ private void shutdown() {
+ if (execSvc != null)
+ execSvc.shutdown(5000);
+
+ if (msgExecSvc != null)
+ msgExecSvc.shutdownNow();
+
+ try {
+ job.dispose(true);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to dispose job.", e);
+ }
+ }
+
+ /**
+ * Notifies node about task finish.
+ *
+ * @param run Finished task runnable.
+ * @param status Task status.
+ */
+ private void onTaskFinished0(HadoopRunnableTask run, HadoopTaskStatus status) {
+ HadoopTaskInfo info = run.taskInfo();
+
+ int pendingTasks0 = pendingTasks.decrementAndGet();
+
+ if (log.isDebugEnabled())
+ log.debug("Hadoop task execution finished [info=" + info
+ + ", state=" + status.state() + ", waitTime=" + run.waitTime() + ", execTime=" + run.executionTime() +
+ ", pendingTasks=" + pendingTasks0 +
+ ", err=" + status.failCause() + ']');
+
+ assert info.type() == MAP || info.type() == REDUCE : "Only MAP or REDUCE tasks are supported.";
+
+ boolean flush = pendingTasks0 == 0 && info.type() == MAP;
+
+ notifyTaskFinished(info, status, flush);
+ }
+
+ /**
+ * @param taskInfo Finished task info.
+ * @param status Task status.
+ */
+ private void notifyTaskFinished(final HadoopTaskInfo taskInfo, final HadoopTaskStatus status,
+ boolean flush) {
+
+ final HadoopTaskState state = status.state();
+ final Throwable err = status.failCause();
+
+ if (!flush) {
+ try {
+ if (log.isDebugEnabled())
+ log.debug("Sending notification to parent node [taskInfo=" + taskInfo + ", state=" + state +
+ ", err=" + err + ']');
+
+ comm.sendMessage(nodeDesc, new HadoopTaskFinishedMessage(taskInfo, status));
+ }
+ catch (IgniteCheckedException e) {
+ log.error("Failed to send message to parent node (will terminate child process).", e);
+
+ shutdown();
+
+ terminate();
+ }
+ }
+ else {
+ if (log.isDebugEnabled())
+ log.debug("Flushing shuffle messages before sending last task completion notification [taskInfo=" +
+ taskInfo + ", state=" + state + ", err=" + err + ']');
+
+ final long start = U.currentTimeMillis();
+
+ try {
+ shuffleJob.flush().listenAsync(new CI1<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> f) {
+ long end = U.currentTimeMillis();
+
+ if (log.isDebugEnabled())
+ log.debug("Finished flushing shuffle messages [taskInfo=" + taskInfo +
+ ", flushTime=" + (end - start) + ']');
+
+ try {
+ // Check for errors on shuffle.
+ f.get();
+
+ notifyTaskFinished(taskInfo, status, false);
+ }
+ catch (IgniteCheckedException e) {
+ log.error("Failed to flush shuffle messages (will fail the task) [taskInfo=" + taskInfo +
+ ", state=" + state + ", err=" + err + ']', e);
+
+ notifyTaskFinished(taskInfo,
+ new HadoopTaskStatus(HadoopTaskState.FAILED, e), false);
+ }
+ }
+ });
+ }
+ catch (IgniteCheckedException e) {
+ log.error("Failed to flush shuffle messages (will fail the task) [taskInfo=" + taskInfo +
+ ", state=" + state + ", err=" + err + ']', e);
+
+ notifyTaskFinished(taskInfo, new HadoopTaskStatus(HadoopTaskState.FAILED, e), false);
+ }
+ }
+ }
+
+ /**
+ * Checks if message was received from parent node and prints warning if not.
+ *
+ * @param desc Sender process ID.
+ * @param msg Received message.
+ * @return {@code True} if received from parent node.
+ */
+ private boolean validateNodeMessage(HadoopProcessDescriptor desc, HadoopMessage msg) {
+ if (!nodeDesc.processId().equals(desc.processId())) {
+ log.warning("Received process control request from unknown process (will ignore) [desc=" + desc +
+ ", msg=" + msg + ']');
+
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Stops execution of this process.
+ */
+ private void terminate() {
+ System.exit(1);
+ }
+
+ /**
+ * Message listener.
+ */
+ private class MessageListener implements HadoopMessageListener {
+ /** {@inheritDoc} */
+ @Override public void onMessageReceived(final HadoopProcessDescriptor desc, final HadoopMessage msg) {
+ if (msg instanceof HadoopTaskExecutionRequest) {
+ if (validateNodeMessage(desc, msg))
+ runTasks((HadoopTaskExecutionRequest)msg);
+ }
+ else if (msg instanceof HadoopJobInfoUpdateRequest) {
+ if (validateNodeMessage(desc, msg))
+ updateTasks((HadoopJobInfoUpdateRequest)msg);
+ }
+ else if (msg instanceof HadoopPrepareForJobRequest) {
+ if (validateNodeMessage(desc, msg))
+ prepareProcess((HadoopPrepareForJobRequest)msg);
+ }
+ else if (msg instanceof HadoopShuffleMessage) {
+ if (log.isTraceEnabled())
+ log.trace("Received shuffle message [desc=" + desc + ", msg=" + msg + ']');
+
+ initFut.listenAsync(new CI1<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> f) {
+ try {
+ HadoopShuffleMessage m = (HadoopShuffleMessage)msg;
+
+ shuffleJob.onShuffleMessage(m);
+
+ comm.sendMessage(desc, new HadoopShuffleAck(m.id(), m.jobId()));
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to process hadoop shuffle message [desc=" + desc + ", msg=" + msg + ']', e);
+ }
+ }
+ });
+ }
+ else if (msg instanceof HadoopShuffleAck) {
+ if (log.isTraceEnabled())
+ log.trace("Received shuffle ack [desc=" + desc + ", msg=" + msg + ']');
+
+ shuffleJob.onShuffleAck((HadoopShuffleAck)msg);
+ }
+ else
+ log.warning("Unknown message received (will ignore) [desc=" + desc + ", msg=" + msg + ']');
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onConnectionLost(HadoopProcessDescriptor desc) {
+ if (log.isDebugEnabled())
+ log.debug("Lost connection with remote process: " + desc);
+
+ if (desc == null)
+ U.warn(log, "Handshake failed.");
+ else if (desc.processId().equals(nodeDesc.processId())) {
+ log.warning("Child process lost connection with parent node (will terminate child process).");
+
+ shutdown();
+
+ terminate();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopExternalProcessStarter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopExternalProcessStarter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopExternalProcessStarter.java
new file mode 100644
index 0000000..3a94d43
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopExternalProcessStarter.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.child;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
+import org.apache.ignite.logger.log4j.*;
+import org.apache.ignite.marshaller.optimized.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+/**
+ * Hadoop external process base class.
+ */
+public class HadoopExternalProcessStarter {
+ /** Path to Log4j configuration file. */
+ public static final String DFLT_LOG4J_CONFIG = "config/ignite-log4j.xml";
+
+ /** Arguments. */
+ private Args args;
+
+ /** System out. */
+ private OutputStream out;
+
+ /** System err. */
+ private OutputStream err;
+
+ /**
+ * @param args Parsed arguments.
+ */
+ public HadoopExternalProcessStarter(Args args) {
+ this.args = args;
+ }
+
+ /**
+ * @param cmdArgs Process arguments.
+ */
+ public static void main(String[] cmdArgs) {
+ try {
+ Args args = arguments(cmdArgs);
+
+ new HadoopExternalProcessStarter(args).run();
+ }
+ catch (Exception e) {
+ System.err.println("Failed");
+
+ System.err.println(e.getMessage());
+
+ e.printStackTrace(System.err);
+ }
+ }
+
+ /**
+ *
+ * @throws Exception
+ */
+ public void run() throws Exception {
+ U.setWorkDirectory(args.workDir, U.getIgniteHome());
+
+ File outputDir = outputDirectory();
+
+ initializeStreams(outputDir);
+
+ ExecutorService msgExecSvc = Executors.newFixedThreadPool(
+ Integer.getInteger("MSG_THREAD_POOL_SIZE", Runtime.getRuntime().availableProcessors() * 2));
+
+ IgniteLogger log = logger(outputDir);
+
+ HadoopExternalCommunication comm = new HadoopExternalCommunication(
+ args.nodeId,
+ args.childProcId,
+ new OptimizedMarshaller(),
+ log,
+ msgExecSvc,
+ "external"
+ );
+
+ comm.start();
+
+ HadoopProcessDescriptor nodeDesc = new HadoopProcessDescriptor(args.nodeId, args.parentProcId);
+ nodeDesc.address(args.addr);
+ nodeDesc.tcpPort(args.tcpPort);
+ nodeDesc.sharedMemoryPort(args.shmemPort);
+
+ HadoopChildProcessRunner runner = new HadoopChildProcessRunner();
+
+ runner.start(comm, nodeDesc, msgExecSvc, log);
+
+ System.err.println("Started");
+ System.err.flush();
+
+ System.setOut(new PrintStream(out));
+ System.setErr(new PrintStream(err));
+ }
+
+ /**
+ * @param outputDir Directory for process output.
+ * @throws Exception
+ */
+ private void initializeStreams(File outputDir) throws Exception {
+ out = new FileOutputStream(new File(outputDir, args.childProcId + ".out"));
+ err = new FileOutputStream(new File(outputDir, args.childProcId + ".err"));
+ }
+
+ /**
+ * @return Path to output directory.
+ * @throws IOException If failed.
+ */
+ private File outputDirectory() throws IOException {
+ File f = new File(args.out);
+
+ if (!f.exists()) {
+ if (!f.mkdirs())
+ throw new IOException("Failed to create output directory: " + args.out);
+ }
+ else {
+ if (f.isFile())
+ throw new IOException("Output directory is a file: " + args.out);
+ }
+
+ return f;
+ }
+
+ /**
+ * @param outputDir Directory for process output.
+ * @return Logger.
+ */
+ private IgniteLogger logger(final File outputDir) {
+ final URL url = U.resolveIgniteUrl(DFLT_LOG4J_CONFIG);
+
+ Log4JLogger logger;
+
+ try {
+ logger = url != null ? new Log4JLogger(url) : new Log4JLogger(true);
+ }
+ catch (IgniteCheckedException e) {
+ System.err.println("Failed to create URL-based logger. Will use default one.");
+
+ e.printStackTrace();
+
+ logger = new Log4JLogger(true);
+ }
+
+ logger.updateFilePath(new IgniteClosure<String, String>() {
+ @Override public String apply(String s) {
+ return new File(outputDir, args.childProcId + ".log").getAbsolutePath();
+ }
+ });
+
+ return logger;
+ }
+
+ /**
+ * @param processArgs Process arguments.
+ * @return Child process instance.
+ */
+ private static Args arguments(String[] processArgs) throws Exception {
+ Args args = new Args();
+
+ for (int i = 0; i < processArgs.length; i++) {
+ String arg = processArgs[i];
+
+ switch (arg) {
+ case "-cpid": {
+ if (i == processArgs.length - 1)
+ throw new Exception("Missing process ID for '-cpid' parameter");
+
+ String procIdStr = processArgs[++i];
+
+ args.childProcId = UUID.fromString(procIdStr);
+
+ break;
+ }
+
+ case "-ppid": {
+ if (i == processArgs.length - 1)
+ throw new Exception("Missing process ID for '-ppid' parameter");
+
+ String procIdStr = processArgs[++i];
+
+ args.parentProcId = UUID.fromString(procIdStr);
+
+ break;
+ }
+
+ case "-nid": {
+ if (i == processArgs.length - 1)
+ throw new Exception("Missing node ID for '-nid' parameter");
+
+ String nodeIdStr = processArgs[++i];
+
+ args.nodeId = UUID.fromString(nodeIdStr);
+
+ break;
+ }
+
+ case "-addr": {
+ if (i == processArgs.length - 1)
+ throw new Exception("Missing node address for '-addr' parameter");
+
+ args.addr = processArgs[++i];
+
+ break;
+ }
+
+ case "-tport": {
+ if (i == processArgs.length - 1)
+ throw new Exception("Missing tcp port for '-tport' parameter");
+
+ args.tcpPort = Integer.parseInt(processArgs[++i]);
+
+ break;
+ }
+
+ case "-sport": {
+ if (i == processArgs.length - 1)
+ throw new Exception("Missing shared memory port for '-sport' parameter");
+
+ args.shmemPort = Integer.parseInt(processArgs[++i]);
+
+ break;
+ }
+
+ case "-out": {
+ if (i == processArgs.length - 1)
+ throw new Exception("Missing output folder name for '-out' parameter");
+
+ args.out = processArgs[++i];
+
+ break;
+ }
+
+ case "-wd": {
+ if (i == processArgs.length - 1)
+ throw new Exception("Missing work folder name for '-wd' parameter");
+
+ args.workDir = processArgs[++i];
+
+ break;
+ }
+ }
+ }
+
+ return args;
+ }
+
+ /**
+ * Execution arguments.
+ */
+ private static class Args {
+ /** Process ID. */
+ private UUID childProcId;
+
+ /** Process ID. */
+ private UUID parentProcId;
+
+ /** Process ID. */
+ private UUID nodeId;
+
+ /** Node address. */
+ private String addr;
+
+ /** TCP port */
+ private int tcpPort;
+
+ /** Shmem port. */
+ private int shmemPort = -1;
+
+ /** Output folder. */
+ private String out;
+
+ /** Work directory. */
+ private String workDir;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopAbstractCommunicationClient.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopAbstractCommunicationClient.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopAbstractCommunicationClient.java
deleted file mode 100644
index 5dee79b..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopAbstractCommunicationClient.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
-
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.util.concurrent.atomic.*;
-
-/**
- * Implements basic lifecycle for communication clients.
- */
-public abstract class GridHadoopAbstractCommunicationClient implements GridHadoopCommunicationClient {
- /** Time when this client was last used. */
- private volatile long lastUsed = U.currentTimeMillis();
-
- /** Reservations. */
- private final AtomicInteger reserves = new AtomicInteger();
-
- /** {@inheritDoc} */
- @Override public boolean close() {
- return reserves.compareAndSet(0, -1);
- }
-
- /** {@inheritDoc} */
- @Override public void forceClose() {
- reserves.set(-1);
- }
-
- /** {@inheritDoc} */
- @Override public boolean closed() {
- return reserves.get() == -1;
- }
-
- /** {@inheritDoc} */
- @Override public boolean reserve() {
- while (true) {
- int r = reserves.get();
-
- if (r == -1)
- return false;
-
- if (reserves.compareAndSet(r, r + 1))
- return true;
- }
- }
-
- /** {@inheritDoc} */
- @Override public void release() {
- while (true) {
- int r = reserves.get();
-
- if (r == -1)
- return;
-
- if (reserves.compareAndSet(r, r - 1))
- return;
- }
- }
-
- /** {@inheritDoc} */
- @Override public boolean reserved() {
- return reserves.get() > 0;
- }
-
- /** {@inheritDoc} */
- @Override public long getIdleTime() {
- return U.currentTimeMillis() - lastUsed;
- }
-
- /**
- * Updates used time.
- */
- protected void markUsed() {
- lastUsed = U.currentTimeMillis();
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopAbstractCommunicationClient.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopCommunicationClient.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopCommunicationClient.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopCommunicationClient.java
deleted file mode 100644
index b375b55..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopCommunicationClient.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
-
-/**
- *
- */
-public interface GridHadoopCommunicationClient {
- /**
- * @return {@code True} if client has been closed by this call,
- * {@code false} if failed to close client (due to concurrent reservation or concurrent close).
- */
- public boolean close();
-
- /**
- * Forces client close.
- */
- public void forceClose();
-
- /**
- * @return {@code True} if client is closed;
- */
- public boolean closed();
-
- /**
- * @return {@code True} if client was reserved, {@code false} otherwise.
- */
- public boolean reserve();
-
- /**
- * Releases this client by decreasing reservations.
- */
- public void release();
-
- /**
- * @return {@code True} if client was reserved.
- */
- public boolean reserved();
-
- /**
- * Gets idle time of this client.
- *
- * @return Idle time of this client.
- */
- public long getIdleTime();
-
- /**
- * @param desc Process descriptor.
- * @param msg Message to send.
- * @throws IgniteCheckedException If failed.
- */
- public void sendMessage(GridHadoopProcessDescriptor desc, GridHadoopMessage msg) throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopExternalCommunication.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopExternalCommunication.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopExternalCommunication.java
deleted file mode 100644
index f5ddced..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopExternalCommunication.java
+++ /dev/null
@@ -1,1431 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.ipc.*;
-import org.apache.ignite.internal.util.ipc.shmem.*;
-import org.apache.ignite.internal.util.nio.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.internal.util.worker.*;
-import org.apache.ignite.marshaller.*;
-import org.apache.ignite.thread.*;
-import org.jdk8.backport.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.net.*;
-import java.nio.*;
-import java.nio.channels.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-/**
- * Hadoop external communication class.
- */
-public class GridHadoopExternalCommunication {
- /** IPC error message. */
- public static final String OUT_OF_RESOURCES_TCP_MSG = "Failed to allocate shared memory segment " +
- "(switching to TCP, may be slower)."; // TODO IGNITE-70 Add link to documentation
-
- /** Default port which node sets listener to (value is <tt>47100</tt>). */
- public static final int DFLT_PORT = 27100;
-
- /** Default connection timeout (value is <tt>1000</tt>ms). */
- public static final long DFLT_CONN_TIMEOUT = 1000;
-
- /** Default Maximum connection timeout (value is <tt>600,000</tt>ms). */
- public static final long DFLT_MAX_CONN_TIMEOUT = 10 * 60 * 1000;
-
- /** Default reconnect attempts count (value is <tt>10</tt>). */
- public static final int DFLT_RECONNECT_CNT = 10;
-
- /** Default message queue limit per connection (for incoming and outgoing . */
- public static final int DFLT_MSG_QUEUE_LIMIT = GridNioServer.DFLT_SEND_QUEUE_LIMIT;
-
- /**
- * Default count of selectors for TCP server equals to
- * {@code "Math.min(4, Runtime.getRuntime().availableProcessors())"}.
- */
- public static final int DFLT_SELECTORS_CNT = 1;
-
- /** Node ID meta for session. */
- private static final int PROCESS_META = GridNioSessionMetaKey.nextUniqueKey();
-
- /** Handshake timeout meta for session. */
- private static final int HANDSHAKE_FINISH_META = GridNioSessionMetaKey.nextUniqueKey();
-
- /** Message tracker meta for session. */
- private static final int TRACKER_META = GridNioSessionMetaKey.nextUniqueKey();
-
- /**
- * Default local port range (value is <tt>100</tt>).
- * See {@link #setLocalPortRange(int)} for details.
- */
- public static final int DFLT_PORT_RANGE = 100;
-
- /** Default value for {@code TCP_NODELAY} socket option (value is <tt>true</tt>). */
- public static final boolean DFLT_TCP_NODELAY = true;
-
- /** Server listener. */
- private final GridNioServerListener<GridHadoopMessage> srvLsnr =
- new GridNioServerListenerAdapter<GridHadoopMessage>() {
- @Override public void onConnected(GridNioSession ses) {
- GridHadoopProcessDescriptor desc = ses.meta(PROCESS_META);
-
- assert desc != null : "Received connected notification without finished handshake: " + ses;
- }
-
- /** {@inheritDoc} */
- @Override public void onDisconnected(GridNioSession ses, @Nullable Exception e) {
- if (log.isDebugEnabled())
- log.debug("Closed connection for session: " + ses);
-
- if (e != null)
- U.error(log, "Session disconnected due to exception: " + ses, e);
-
- GridHadoopProcessDescriptor desc = ses.meta(PROCESS_META);
-
- if (desc != null) {
- GridHadoopCommunicationClient rmv = clients.remove(desc.processId());
-
- if (rmv != null)
- rmv.forceClose();
- }
-
- GridHadoopMessageListener lsnr0 = lsnr;
-
- if (lsnr0 != null)
- // Notify listener about connection close.
- lsnr0.onConnectionLost(desc);
- }
-
- /** {@inheritDoc} */
- @Override public void onMessage(GridNioSession ses, GridHadoopMessage msg) {
- notifyListener(ses.<GridHadoopProcessDescriptor>meta(PROCESS_META), msg);
-
- if (msgQueueLimit > 0) {
- GridNioMessageTracker tracker = ses.meta(TRACKER_META);
-
- assert tracker != null : "Missing tracker for limited message queue: " + ses;
-
- tracker.run();
- }
- }
- };
-
- /** Logger. */
- private IgniteLogger log;
-
- /** Local process descriptor. */
- private GridHadoopProcessDescriptor locProcDesc;
-
- /** Marshaller. */
- private Marshaller marsh;
-
- /** Message notification executor service. */
- private ExecutorService execSvc;
-
- /** Grid name. */
- private String gridName;
-
- /** Complex variable that represents this node IP address. */
- private volatile InetAddress locHost;
-
- /** Local port which node uses. */
- private int locPort = DFLT_PORT;
-
- /** Local port range. */
- private int locPortRange = DFLT_PORT_RANGE;
-
- /** Local port which node uses to accept shared memory connections. */
- private int shmemPort = -1;
-
- /** Allocate direct buffer or heap buffer. */
- private boolean directBuf = true;
-
- /** Connect timeout. */
- private long connTimeout = DFLT_CONN_TIMEOUT;
-
- /** Maximum connect timeout. */
- private long maxConnTimeout = DFLT_MAX_CONN_TIMEOUT;
-
- /** Reconnect attempts count. */
- @SuppressWarnings({"FieldAccessedSynchronizedAndUnsynchronized"})
- private int reconCnt = DFLT_RECONNECT_CNT;
-
- /** Socket send buffer. */
- private int sockSndBuf;
-
- /** Socket receive buffer. */
- private int sockRcvBuf;
-
- /** Message queue limit. */
- private int msgQueueLimit = DFLT_MSG_QUEUE_LIMIT;
-
- /** NIO server. */
- private GridNioServer<GridHadoopMessage> nioSrvr;
-
- /** Shared memory server. */
- private IpcSharedMemoryServerEndpoint shmemSrv;
-
- /** {@code TCP_NODELAY} option value for created sockets. */
- private boolean tcpNoDelay = DFLT_TCP_NODELAY;
-
- /** Shared memory accept worker. */
- private ShmemAcceptWorker shmemAcceptWorker;
-
- /** Shared memory workers. */
- private final Collection<ShmemWorker> shmemWorkers = new ConcurrentLinkedDeque8<>();
-
- /** Clients. */
- private final ConcurrentMap<UUID, GridHadoopCommunicationClient> clients = GridConcurrentFactory.newMap();
-
- /** Message listener. */
- private volatile GridHadoopMessageListener lsnr;
-
- /** Bound port. */
- private int boundTcpPort = -1;
-
- /** Bound port for shared memory server. */
- private int boundTcpShmemPort = -1;
-
- /** Count of selectors to use in TCP server. */
- private int selectorsCnt = DFLT_SELECTORS_CNT;
-
- /** Local node ID message. */
- private ProcessHandshakeMessage locIdMsg;
-
- /** Locks. */
- private final GridKeyLock locks = new GridKeyLock();
-
- /**
- * @param parentNodeId Parent node ID.
- * @param procId Process ID.
- * @param marsh Marshaller to use.
- * @param log Logger.
- * @param execSvc Executor service for message notification.
- * @param gridName Grid name.
- */
- public GridHadoopExternalCommunication(
- UUID parentNodeId,
- UUID procId,
- Marshaller marsh,
- IgniteLogger log,
- ExecutorService execSvc,
- String gridName
- ) {
- locProcDesc = new GridHadoopProcessDescriptor(parentNodeId, procId);
-
- this.marsh = marsh;
- this.log = log.getLogger(GridHadoopExternalCommunication.class);
- this.execSvc = execSvc;
- this.gridName = gridName;
- }
-
- /**
- * Sets local port for socket binding.
- * <p>
- * If not provided, default value is {@link #DFLT_PORT}.
- *
- * @param locPort Port number.
- */
- public void setLocalPort(int locPort) {
- this.locPort = locPort;
- }
-
- /**
- * Gets local port for socket binding.
- *
- * @return Local port.
- */
- public int getLocalPort() {
- return locPort;
- }
-
- /**
- * Sets local port range for local host ports (value must greater than or equal to <tt>0</tt>).
- * If provided local port (see {@link #setLocalPort(int)}} is occupied,
- * implementation will try to increment the port number for as long as it is less than
- * initial value plus this range.
- * <p>
- * If port range value is <tt>0</tt>, then implementation will try bind only to the port provided by
- * {@link #setLocalPort(int)} method and fail if binding to this port did not succeed.
- * <p>
- * Local port range is very useful during development when more than one grid nodes need to run
- * on the same physical machine.
- * <p>
- * If not provided, default value is {@link #DFLT_PORT_RANGE}.
- *
- * @param locPortRange New local port range.
- */
- public void setLocalPortRange(int locPortRange) {
- this.locPortRange = locPortRange;
- }
-
- /**
- * @return Local port range.
- */
- public int getLocalPortRange() {
- return locPortRange;
- }
-
- /**
- * Sets local port to accept shared memory connections.
- * <p>
- * If set to {@code -1} shared memory communication will be disabled.
- * <p>
- * If not provided, shared memory is disabled.
- *
- * @param shmemPort Port number.
- */
- public void setSharedMemoryPort(int shmemPort) {
- this.shmemPort = shmemPort;
- }
-
- /**
- * Gets shared memory port to accept incoming connections.
- *
- * @return Shared memory port.
- */
- public int getSharedMemoryPort() {
- return shmemPort;
- }
-
- /**
- * Sets connect timeout used when establishing connection
- * with remote nodes.
- * <p>
- * {@code 0} is interpreted as infinite timeout.
- * <p>
- * If not provided, default value is {@link #DFLT_CONN_TIMEOUT}.
- *
- * @param connTimeout Connect timeout.
- */
- public void setConnectTimeout(long connTimeout) {
- this.connTimeout = connTimeout;
- }
-
- /**
- * @return Connection timeout.
- */
- public long getConnectTimeout() {
- return connTimeout;
- }
-
- /**
- * Sets maximum connect timeout. If handshake is not established within connect timeout,
- * then SPI tries to repeat handshake procedure with increased connect timeout.
- * Connect timeout can grow till maximum timeout value,
- * if maximum timeout value is reached then the handshake is considered as failed.
- * <p>
- * {@code 0} is interpreted as infinite timeout.
- * <p>
- * If not provided, default value is {@link #DFLT_MAX_CONN_TIMEOUT}.
- *
- * @param maxConnTimeout Maximum connect timeout.
- */
- public void setMaxConnectTimeout(long maxConnTimeout) {
- this.maxConnTimeout = maxConnTimeout;
- }
-
- /**
- * Gets maximum connection timeout.
- *
- * @return Maximum connection timeout.
- */
- public long getMaxConnectTimeout() {
- return maxConnTimeout;
- }
-
- /**
- * Sets maximum number of reconnect attempts used when establishing connection
- * with remote nodes.
- * <p>
- * If not provided, default value is {@link #DFLT_RECONNECT_CNT}.
- *
- * @param reconCnt Maximum number of reconnection attempts.
- */
- public void setReconnectCount(int reconCnt) {
- this.reconCnt = reconCnt;
- }
-
- /**
- * @return Reconnect count.
- */
- public int getReconnectCount() {
- return reconCnt;
- }
-
- /**
- * Sets flag to allocate direct or heap buffer in SPI.
- * If value is {@code true}, then SPI will use {@link ByteBuffer#allocateDirect(int)} call.
- * Otherwise, SPI will use {@link ByteBuffer#allocate(int)} call.
- * <p>
- * If not provided, default value is {@code true}.
- *
- * @param directBuf Flag indicates to allocate direct or heap buffer in SPI.
- */
- public void setDirectBuffer(boolean directBuf) {
- this.directBuf = directBuf;
- }
-
- /**
- * @return Direct buffer flag.
- */
- public boolean isDirectBuffer() {
- return directBuf;
- }
-
- /**
- * Sets the count of selectors te be used in TCP server.
- * <p/>
- * If not provided, default value is {@link #DFLT_SELECTORS_CNT}.
- *
- * @param selectorsCnt Selectors count.
- */
- public void setSelectorsCount(int selectorsCnt) {
- this.selectorsCnt = selectorsCnt;
- }
-
- /**
- * @return Number of selectors to use.
- */
- public int getSelectorsCount() {
- return selectorsCnt;
- }
-
- /**
- * Sets value for {@code TCP_NODELAY} socket option. Each
- * socket will be opened using provided value.
- * <p>
- * Setting this option to {@code true} disables Nagle's algorithm
- * for socket decreasing latency and delivery time for small messages.
- * <p>
- * For systems that work under heavy network load it is advisable to
- * set this value to {@code false}.
- * <p>
- * If not provided, default value is {@link #DFLT_TCP_NODELAY}.
- *
- * @param tcpNoDelay {@code True} to disable TCP delay.
- */
- public void setTcpNoDelay(boolean tcpNoDelay) {
- this.tcpNoDelay = tcpNoDelay;
- }
-
- /**
- * @return {@code TCP_NO_DELAY} flag.
- */
- public boolean isTcpNoDelay() {
- return tcpNoDelay;
- }
-
- /**
- * Sets receive buffer size for sockets created or accepted by this SPI.
- * <p>
- * If not provided, default is {@code 0} which leaves buffer unchanged after
- * socket creation (OS defaults).
- *
- * @param sockRcvBuf Socket receive buffer size.
- */
- public void setSocketReceiveBuffer(int sockRcvBuf) {
- this.sockRcvBuf = sockRcvBuf;
- }
-
- /**
- * @return Socket receive buffer size.
- */
- public int getSocketReceiveBuffer() {
- return sockRcvBuf;
- }
-
- /**
- * Sets send buffer size for sockets created or accepted by this SPI.
- * <p>
- * If not provided, default is {@code 0} which leaves the buffer unchanged
- * after socket creation (OS defaults).
- *
- * @param sockSndBuf Socket send buffer size.
- */
- public void setSocketSendBuffer(int sockSndBuf) {
- this.sockSndBuf = sockSndBuf;
- }
-
- /**
- * @return Socket send buffer size.
- */
- public int getSocketSendBuffer() {
- return sockSndBuf;
- }
-
- /**
- * Sets message queue limit for incoming and outgoing messages.
- * <p>
- * When set to positive number send queue is limited to the configured value.
- * {@code 0} disables the size limitations.
- * <p>
- * If not provided, default is {@link #DFLT_MSG_QUEUE_LIMIT}.
- *
- * @param msgQueueLimit Send queue size limit.
- */
- public void setMessageQueueLimit(int msgQueueLimit) {
- this.msgQueueLimit = msgQueueLimit;
- }
-
- /**
- * @return Message queue size limit.
- */
- public int getMessageQueueLimit() {
- return msgQueueLimit;
- }
-
- /**
- * Sets Hadoop communication message listener.
- *
- * @param lsnr Message listener.
- */
- public void setListener(GridHadoopMessageListener lsnr) {
- this.lsnr = lsnr;
- }
-
- /**
- * @return Outbound message queue size.
- */
- public int getOutboundMessagesQueueSize() {
- return nioSrvr.outboundMessagesQueueSize();
- }
-
- /**
- * Starts communication.
- *
- * @throws IgniteCheckedException If failed.
- */
- public void start() throws IgniteCheckedException {
- try {
- locHost = U.getLocalHost();
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to initialize local address.", e);
- }
-
- try {
- shmemSrv = resetShmemServer();
- }
- catch (IgniteCheckedException e) {
- U.warn(log, "Failed to start shared memory communication server.", e);
- }
-
- try {
- // This method potentially resets local port to the value
- // local node was bound to.
- nioSrvr = resetNioServer();
- }
- catch (IgniteCheckedException e) {
- throw new IgniteCheckedException("Failed to initialize TCP server: " + locHost, e);
- }
-
- locProcDesc.address(locHost.getHostAddress());
- locProcDesc.sharedMemoryPort(boundTcpShmemPort);
- locProcDesc.tcpPort(boundTcpPort);
-
- locIdMsg = new ProcessHandshakeMessage(locProcDesc);
-
- if (shmemSrv != null) {
- shmemAcceptWorker = new ShmemAcceptWorker(shmemSrv);
-
- new IgniteThread(shmemAcceptWorker).start();
- }
-
- nioSrvr.start();
- }
-
- /**
- * Gets local process descriptor.
- *
- * @return Local process descriptor.
- */
- public GridHadoopProcessDescriptor localProcessDescriptor() {
- return locProcDesc;
- }
-
- /**
- * Gets filters used by communication.
- *
- * @return Filters array.
- */
- private GridNioFilter[] filters() {
- return new GridNioFilter[] {
- new GridNioAsyncNotifyFilter(gridName, execSvc, log),
- new HandshakeAndBackpressureFilter(),
- new GridHadoopMarshallerFilter(marsh),
- new GridNioCodecFilter(new GridBufferedParser(directBuf, ByteOrder.nativeOrder()), log, false)
- };
- }
-
- /**
- * Recreates tpcSrvr socket instance.
- *
- * @return Server instance.
- * @throws IgniteCheckedException Thrown if it's not possible to create server.
- */
- private GridNioServer<GridHadoopMessage> resetNioServer() throws IgniteCheckedException {
- if (boundTcpPort >= 0)
- throw new IgniteCheckedException("Tcp NIO server was already created on port " + boundTcpPort);
-
- IgniteCheckedException lastEx = null;
-
- // If configured TCP port is busy, find first available in range.
- for (int port = locPort; port < locPort + locPortRange; port++) {
- try {
- GridNioServer<GridHadoopMessage> srvr =
- GridNioServer.<GridHadoopMessage>builder()
- .address(locHost)
- .port(port)
- .listener(srvLsnr)
- .logger(log.getLogger(GridNioServer.class))
- .selectorCount(selectorsCnt)
- .gridName(gridName)
- .tcpNoDelay(tcpNoDelay)
- .directBuffer(directBuf)
- .byteOrder(ByteOrder.nativeOrder())
- .socketSendBufferSize(sockSndBuf)
- .socketReceiveBufferSize(sockRcvBuf)
- .sendQueueLimit(msgQueueLimit)
- .directMode(false)
- .filters(filters())
- .build();
-
- boundTcpPort = port;
-
- // Ack Port the TCP server was bound to.
- if (log.isInfoEnabled())
- log.info("Successfully bound to TCP port [port=" + boundTcpPort +
- ", locHost=" + locHost + ']');
-
- return srvr;
- }
- catch (IgniteCheckedException e) {
- lastEx = e;
-
- if (log.isDebugEnabled())
- log.debug("Failed to bind to local port (will try next port within range) [port=" + port +
- ", locHost=" + locHost + ']');
- }
- }
-
- // If free port wasn't found.
- throw new IgniteCheckedException("Failed to bind to any port within range [startPort=" + locPort +
- ", portRange=" + locPortRange + ", locHost=" + locHost + ']', lastEx);
- }
-
- /**
- * Creates new shared memory communication server.
- * @return Server.
- * @throws IgniteCheckedException If failed.
- */
- @Nullable private IpcSharedMemoryServerEndpoint resetShmemServer() throws IgniteCheckedException {
- if (boundTcpShmemPort >= 0)
- throw new IgniteCheckedException("Shared memory server was already created on port " + boundTcpShmemPort);
-
- if (shmemPort == -1 || U.isWindows())
- return null;
-
- IgniteCheckedException lastEx = null;
-
- // If configured TCP port is busy, find first available in range.
- for (int port = shmemPort; port < shmemPort + locPortRange; port++) {
- try {
- IpcSharedMemoryServerEndpoint srv = new IpcSharedMemoryServerEndpoint(
- log.getLogger(IpcSharedMemoryServerEndpoint.class),
- locProcDesc.processId(), gridName);
-
- srv.setPort(port);
-
- srv.omitOutOfResourcesWarning(true);
-
- srv.start();
-
- boundTcpShmemPort = port;
-
- // Ack Port the TCP server was bound to.
- if (log.isInfoEnabled())
- log.info("Successfully bound shared memory communication to TCP port [port=" + boundTcpShmemPort +
- ", locHost=" + locHost + ']');
-
- return srv;
- }
- catch (IgniteCheckedException e) {
- lastEx = e;
-
- if (log.isDebugEnabled())
- log.debug("Failed to bind to local port (will try next port within range) [port=" + port +
- ", locHost=" + locHost + ']');
- }
- }
-
- // If free port wasn't found.
- throw new IgniteCheckedException("Failed to bind shared memory communication to any port within range [startPort=" +
- locPort + ", portRange=" + locPortRange + ", locHost=" + locHost + ']', lastEx);
- }
-
- /**
- * Stops the server.
- *
- * @throws IgniteCheckedException
- */
- public void stop() throws IgniteCheckedException {
- // Stop TCP server.
- if (nioSrvr != null)
- nioSrvr.stop();
-
- U.cancel(shmemAcceptWorker);
- U.join(shmemAcceptWorker, log);
-
- U.cancel(shmemWorkers);
- U.join(shmemWorkers, log);
-
- shmemWorkers.clear();
-
- // Force closing on stop (safety).
- for (GridHadoopCommunicationClient client : clients.values())
- client.forceClose();
-
- // Clear resources.
- nioSrvr = null;
-
- boundTcpPort = -1;
- }
-
- /**
- * Sends message to Hadoop process.
- *
- * @param desc
- * @param msg
- * @throws IgniteCheckedException
- */
- public void sendMessage(GridHadoopProcessDescriptor desc, GridHadoopMessage msg) throws
- IgniteCheckedException {
- assert desc != null;
- assert msg != null;
-
- if (log.isTraceEnabled())
- log.trace("Sending message to Hadoop process [desc=" + desc + ", msg=" + msg + ']');
-
- GridHadoopCommunicationClient client = null;
-
- boolean closeOnRelease = true;
-
- try {
- client = reserveClient(desc);
-
- client.sendMessage(desc, msg);
-
- closeOnRelease = false;
- }
- finally {
- if (client != null) {
- if (closeOnRelease) {
- client.forceClose();
-
- clients.remove(desc.processId(), client);
- }
- else
- client.release();
- }
- }
- }
-
- /**
- * Returns existing or just created client to node.
- *
- * @param desc Node to which client should be open.
- * @return The existing or just created client.
- * @throws IgniteCheckedException Thrown if any exception occurs.
- */
- private GridHadoopCommunicationClient reserveClient(GridHadoopProcessDescriptor desc) throws IgniteCheckedException {
- assert desc != null;
-
- UUID procId = desc.processId();
-
- while (true) {
- GridHadoopCommunicationClient client = clients.get(procId);
-
- if (client == null) {
- if (log.isDebugEnabled())
- log.debug("Did not find client for remote process [locProcDesc=" + locProcDesc + ", desc=" +
- desc + ']');
-
- // Do not allow concurrent connects.
- Object sync = locks.lock(procId);
-
- try {
- client = clients.get(procId);
-
- if (client == null) {
- GridHadoopCommunicationClient old = clients.put(procId, client = createNioClient(desc));
-
- assert old == null;
- }
- }
- finally {
- locks.unlock(procId, sync);
- }
-
- assert client != null;
- }
-
- if (client.reserve())
- return client;
- else
- // Client has just been closed by idle worker. Help it and try again.
- clients.remove(procId, client);
- }
- }
-
- /**
- * @param desc Process descriptor.
- * @return Client.
- * @throws IgniteCheckedException If failed.
- */
- @Nullable protected GridHadoopCommunicationClient createNioClient(GridHadoopProcessDescriptor desc)
- throws IgniteCheckedException {
- assert desc != null;
-
- int shmemPort = desc.sharedMemoryPort();
-
- // If remote node has shared memory server enabled and has the same set of MACs
- // then we are likely to run on the same host and shared memory communication could be tried.
- if (shmemPort != -1 && locProcDesc.parentNodeId().equals(desc.parentNodeId())) {
- try {
- return createShmemClient(desc, shmemPort);
- }
- catch (IgniteCheckedException e) {
- if (e.hasCause(IpcOutOfSystemResourcesException.class))
- // Has cause or is itself the IpcOutOfSystemResourcesException.
- LT.warn(log, null, OUT_OF_RESOURCES_TCP_MSG);
- else if (log.isDebugEnabled())
- log.debug("Failed to establish shared memory connection with local hadoop process: " +
- desc);
- }
- }
-
- return createTcpClient(desc);
- }
-
- /**
- * @param desc Process descriptor.
- * @param port Port.
- * @return Client.
- * @throws IgniteCheckedException If failed.
- */
- @Nullable protected GridHadoopCommunicationClient createShmemClient(GridHadoopProcessDescriptor desc, int port)
- throws IgniteCheckedException {
- int attempt = 1;
-
- int connectAttempts = 1;
-
- long connTimeout0 = connTimeout;
-
- while (true) {
- IpcEndpoint clientEndpoint;
-
- try {
- clientEndpoint = new IpcSharedMemoryClientEndpoint(port, (int)connTimeout, log);
- }
- catch (IgniteCheckedException e) {
- // Reconnect for the second time, if connection is not established.
- if (connectAttempts < 2 && X.hasCause(e, ConnectException.class)) {
- connectAttempts++;
-
- continue;
- }
-
- throw e;
- }
-
- GridHadoopCommunicationClient client = null;
-
- try {
- ShmemWorker worker = new ShmemWorker(clientEndpoint, false);
-
- shmemWorkers.add(worker);
-
- GridNioSession ses = worker.session();
-
- HandshakeFinish fin = new HandshakeFinish();
-
- // We are in lock, it is safe to get session and attach
- ses.addMeta(HANDSHAKE_FINISH_META, fin);
-
- client = new GridHadoopTcpNioCommunicationClient(ses);
-
- new IgniteThread(worker).start();
-
- fin.await(connTimeout0);
- }
- catch (GridHadoopHandshakeTimeoutException e) {
- if (log.isDebugEnabled())
- log.debug("Handshake timed out (will retry with increased timeout) [timeout=" + connTimeout0 +
- ", err=" + e.getMessage() + ", client=" + client + ']');
-
- if (client != null)
- client.forceClose();
-
- if (attempt == reconCnt || connTimeout0 > maxConnTimeout) {
- if (log.isDebugEnabled())
- log.debug("Handshake timedout (will stop attempts to perform the handshake) " +
- "[timeout=" + connTimeout0 + ", maxConnTimeout=" + maxConnTimeout +
- ", attempt=" + attempt + ", reconCnt=" + reconCnt +
- ", err=" + e.getMessage() + ", client=" + client + ']');
-
- throw e;
- }
- else {
- attempt++;
-
- connTimeout0 *= 2;
-
- continue;
- }
- }
- catch (RuntimeException | Error e) {
- if (log.isDebugEnabled())
- log.debug(
- "Caught exception (will close client) [err=" + e.getMessage() + ", client=" + client + ']');
-
- if (client != null)
- client.forceClose();
-
- throw e;
- }
-
- return client;
- }
- }
-
- /**
- * Establish TCP connection to remote hadoop process and returns client.
- *
- * @param desc Process descriptor.
- * @return Client.
- * @throws IgniteCheckedException If failed.
- */
- protected GridHadoopCommunicationClient createTcpClient(GridHadoopProcessDescriptor desc) throws IgniteCheckedException {
- String addr = desc.address();
-
- int port = desc.tcpPort();
-
- if (log.isDebugEnabled())
- log.debug("Trying to connect to remote process [locProcDesc=" + locProcDesc + ", desc=" + desc + ']');
-
- boolean conn = false;
- GridHadoopTcpNioCommunicationClient client = null;
- IgniteCheckedException errs = null;
-
- int connectAttempts = 1;
-
- long connTimeout0 = connTimeout;
-
- int attempt = 1;
-
- while (!conn) { // Reconnection on handshake timeout.
- try {
- SocketChannel ch = SocketChannel.open();
-
- ch.configureBlocking(true);
-
- ch.socket().setTcpNoDelay(tcpNoDelay);
- ch.socket().setKeepAlive(true);
-
- if (sockRcvBuf > 0)
- ch.socket().setReceiveBufferSize(sockRcvBuf);
-
- if (sockSndBuf > 0)
- ch.socket().setSendBufferSize(sockSndBuf);
-
- ch.socket().connect(new InetSocketAddress(addr, port), (int)connTimeout);
-
- HandshakeFinish fin = new HandshakeFinish();
-
- GridNioSession ses = nioSrvr.createSession(ch, F.asMap(HANDSHAKE_FINISH_META, fin)).get();
-
- client = new GridHadoopTcpNioCommunicationClient(ses);
-
- if (log.isDebugEnabled())
- log.debug("Waiting for handshake finish for client: " + client);
-
- fin.await(connTimeout0);
-
- conn = true;
- }
- catch (GridHadoopHandshakeTimeoutException e) {
- if (client != null) {
- client.forceClose();
-
- client = null;
- }
-
- if (log.isDebugEnabled())
- log.debug(
- "Handshake timedout (will retry with increased timeout) [timeout=" + connTimeout0 +
- ", desc=" + desc + ", port=" + port + ", err=" + e + ']');
-
- if (attempt == reconCnt || connTimeout0 > maxConnTimeout) {
- if (log.isDebugEnabled())
- log.debug("Handshake timed out (will stop attempts to perform the handshake) " +
- "[timeout=" + connTimeout0 + ", maxConnTimeout=" + maxConnTimeout +
- ", attempt=" + attempt + ", reconCnt=" + reconCnt +
- ", err=" + e.getMessage() + ", addr=" + addr + ']');
-
- if (errs == null)
- errs = new IgniteCheckedException("Failed to connect to remote Hadoop process " +
- "(is process still running?) [desc=" + desc + ", addrs=" + addr + ']');
-
- errs.addSuppressed(e);
-
- break;
- }
- else {
- attempt++;
-
- connTimeout0 *= 2;
-
- // Continue loop.
- }
- }
- catch (Exception e) {
- if (client != null) {
- client.forceClose();
-
- client = null;
- }
-
- if (log.isDebugEnabled())
- log.debug("Client creation failed [addr=" + addr + ", port=" + port +
- ", err=" + e + ']');
-
- if (X.hasCause(e, SocketTimeoutException.class))
- LT.warn(log, null, "Connect timed out (consider increasing 'connTimeout' " +
- "configuration property) [addr=" + addr + ", port=" + port + ']');
-
- if (errs == null)
- errs = new IgniteCheckedException("Failed to connect to remote Hadoop process (is process still running?) " +
- "[desc=" + desc + ", addrs=" + addr + ']');
-
- errs.addSuppressed(e);
-
- // Reconnect for the second time, if connection is not established.
- if (connectAttempts < 2 &&
- (e instanceof ConnectException || X.hasCause(e, ConnectException.class))) {
- connectAttempts++;
-
- continue;
- }
-
- break;
- }
- }
-
- if (client == null) {
- assert errs != null;
-
- if (X.hasCause(errs, ConnectException.class))
- LT.warn(log, null, "Failed to connect to a remote Hadoop process (is process still running?). " +
- "Make sure operating system firewall is disabled on local and remote host) " +
- "[addrs=" + addr + ", port=" + port + ']');
-
- throw errs;
- }
-
- if (log.isDebugEnabled())
- log.debug("Created client: " + client);
-
- return client;
- }
-
- /**
- * @param desc Sender process descriptor.
- * @param msg Communication message.
- */
- protected void notifyListener(GridHadoopProcessDescriptor desc, GridHadoopMessage msg) {
- GridHadoopMessageListener lsnr = this.lsnr;
-
- if (lsnr != null)
- // Notify listener of a new message.
- lsnr.onMessageReceived(desc, msg);
- else if (log.isDebugEnabled())
- log.debug("Received communication message without any registered listeners (will ignore) " +
- "[senderProcDesc=" + desc + ", msg=" + msg + ']');
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopExternalCommunication.class, this);
- }
-
- /**
- * This worker takes responsibility to shut the server down when stopping,
- * No other thread shall stop passed server.
- */
- private class ShmemAcceptWorker extends GridWorker {
- /** */
- private final IpcSharedMemoryServerEndpoint srv;
-
- /**
- * @param srv Server.
- */
- ShmemAcceptWorker(IpcSharedMemoryServerEndpoint srv) {
- super(gridName, "shmem-communication-acceptor", log);
-
- this.srv = srv;
- }
-
- /** {@inheritDoc} */
- @Override protected void body() throws InterruptedException {
- try {
- while (!Thread.interrupted()) {
- ShmemWorker e = new ShmemWorker(srv.accept(), true);
-
- shmemWorkers.add(e);
-
- new IgniteThread(e).start();
- }
- }
- catch (IgniteCheckedException e) {
- if (!isCancelled())
- U.error(log, "Shmem server failed.", e);
- }
- finally {
- srv.close();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void cancel() {
- super.cancel();
-
- srv.close();
- }
- }
-
- /**
- *
- */
- private class ShmemWorker extends GridWorker {
- /** */
- private final IpcEndpoint endpoint;
-
- /** Adapter. */
- private GridHadoopIpcToNioAdapter<GridHadoopMessage> adapter;
-
- /**
- * @param endpoint Endpoint.
- */
- private ShmemWorker(IpcEndpoint endpoint, boolean accepted) {
- super(gridName, "shmem-worker", log);
-
- this.endpoint = endpoint;
-
- adapter = new GridHadoopIpcToNioAdapter<>(
- GridHadoopExternalCommunication.this.log,
- endpoint,
- accepted,
- srvLsnr,
- filters());
- }
-
- /** {@inheritDoc} */
- @Override protected void body() throws InterruptedException {
- try {
- adapter.serve();
- }
- finally {
- shmemWorkers.remove(this);
-
- endpoint.close();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void cancel() {
- super.cancel();
-
- endpoint.close();
- }
-
- /** @{@inheritDoc} */
- @Override protected void cleanup() {
- super.cleanup();
-
- endpoint.close();
- }
-
- /** @{@inheritDoc} */
- @Override public String toString() {
- return S.toString(ShmemWorker.class, this);
- }
-
- /**
- * @return NIO session for this worker.
- */
- public GridNioSession session() {
- return adapter.session();
- }
- }
-
- /**
- *
- */
- private static class HandshakeFinish {
- /** Await latch. */
- private CountDownLatch latch = new CountDownLatch(1);
-
- /**
- * Finishes handshake.
- */
- public void finish() {
- latch.countDown();
- }
-
- /**
- * @param time Time to wait.
- * @throws GridHadoopHandshakeTimeoutException If failed to wait.
- */
- public void await(long time) throws GridHadoopHandshakeTimeoutException {
- try {
- if (!latch.await(time, TimeUnit.MILLISECONDS))
- throw new GridHadoopHandshakeTimeoutException("Failed to wait for handshake to finish [timeout=" +
- time + ']');
- }
- catch (InterruptedException e) {
- Thread.currentThread().interrupt();
-
- throw new GridHadoopHandshakeTimeoutException("Failed to wait for handshake to finish (thread was " +
- "interrupted) [timeout=" + time + ']', e);
- }
- }
- }
-
- /**
- *
- */
- private class HandshakeAndBackpressureFilter extends GridNioFilterAdapter {
- /**
- * Assigns filter name to a filter.
- */
- protected HandshakeAndBackpressureFilter() {
- super("HadoopHandshakeFilter");
- }
-
- /** {@inheritDoc} */
- @Override public void onSessionOpened(final GridNioSession ses) throws IgniteCheckedException {
- if (ses.accepted()) {
- if (log.isDebugEnabled())
- log.debug("Accepted connection, initiating handshake: " + ses);
-
- // Server initiates handshake.
- ses.send(locIdMsg).listenAsync(new CI1<GridNioFuture<?>>() {
- @Override public void apply(GridNioFuture<?> fut) {
- try {
- // Make sure there were no errors.
- fut.get();
- }
- catch (IgniteCheckedException | IOException e) {
- log.warning("Failed to send handshake message, will close session: " + ses, e);
-
- ses.close();
- }
- }
- });
- }
- }
-
- /** {@inheritDoc} */
- @Override public void onSessionClosed(GridNioSession ses) throws IgniteCheckedException {
- proceedSessionClosed(ses);
- }
-
- /** {@inheritDoc} */
- @Override public void onExceptionCaught(GridNioSession ses, IgniteCheckedException ex) throws IgniteCheckedException {
- proceedExceptionCaught(ses, ex);
- }
-
- /** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
- if (ses.meta(PROCESS_META) == null && !(msg instanceof ProcessHandshakeMessage))
- log.warning("Writing message before handshake has finished [ses=" + ses + ", msg=" + msg + ']');
-
- return proceedSessionWrite(ses, msg);
- }
-
- /** {@inheritDoc} */
- @Override public void onMessageReceived(GridNioSession ses, Object msg) throws IgniteCheckedException {
- GridHadoopProcessDescriptor desc = ses.meta(PROCESS_META);
-
- UUID rmtProcId = desc == null ? null : desc.processId();
-
- if (rmtProcId == null) {
- if (!(msg instanceof ProcessHandshakeMessage)) {
- log.warning("Invalid handshake message received, will close connection [ses=" + ses +
- ", msg=" + msg + ']');
-
- ses.close();
-
- return;
- }
-
- ProcessHandshakeMessage nId = (ProcessHandshakeMessage)msg;
-
- if (log.isDebugEnabled())
- log.debug("Received handshake message [ses=" + ses + ", msg=" + msg + ']');
-
- ses.addMeta(PROCESS_META, nId.processDescriptor());
-
- if (!ses.accepted())
- // Send handshake reply.
- ses.send(locIdMsg);
- else {
- //
- rmtProcId = nId.processDescriptor().processId();
-
- if (log.isDebugEnabled())
- log.debug("Finished handshake with remote client: " + ses);
-
- Object sync = locks.tryLock(rmtProcId);
-
- if (sync != null) {
- try {
- if (clients.get(rmtProcId) == null) {
- if (log.isDebugEnabled())
- log.debug("Will reuse session for descriptor: " + rmtProcId);
-
- // Handshake finished flag is true.
- clients.put(rmtProcId, new GridHadoopTcpNioCommunicationClient(ses));
- }
- else {
- if (log.isDebugEnabled())
- log.debug("Will not reuse client as another already exists [locProcDesc=" +
- locProcDesc + ", desc=" + desc + ']');
- }
- }
- finally {
- locks.unlock(rmtProcId, sync);
- }
- }
- else {
- if (log.isDebugEnabled())
- log.debug("Concurrent connection is being established, will not reuse client session [" +
- "locProcDesc=" + locProcDesc + ", desc=" + desc + ']');
- }
- }
-
- if (log.isDebugEnabled())
- log.debug("Handshake is finished for session [ses=" + ses + ", locProcDesc=" + locProcDesc + ']');
-
- HandshakeFinish to = ses.meta(HANDSHAKE_FINISH_META);
-
- if (to != null)
- to.finish();
-
- // Notify session opened (both parties).
- proceedSessionOpened(ses);
- }
- else {
- if (msgQueueLimit > 0) {
- GridNioMessageTracker tracker = ses.meta(TRACKER_META);
-
- if (tracker == null) {
- GridNioMessageTracker old = ses.addMeta(TRACKER_META, tracker =
- new GridNioMessageTracker(ses, msgQueueLimit));
-
- assert old == null;
- }
-
- tracker.onMessageReceived();
- }
-
- proceedMessageReceived(ses, msg);
- }
- }
-
- /** {@inheritDoc} */
- @Override public GridNioFuture<Boolean> onSessionClose(GridNioSession ses) throws IgniteCheckedException {
- return proceedSessionClose(ses);
- }
-
- /** {@inheritDoc} */
- @Override public void onSessionIdleTimeout(GridNioSession ses) throws IgniteCheckedException {
- proceedSessionIdleTimeout(ses);
- }
-
- /** {@inheritDoc} */
- @Override public void onSessionWriteTimeout(GridNioSession ses) throws IgniteCheckedException {
- proceedSessionWriteTimeout(ses);
- }
- }
-
- /**
- * Process ID message.
- */
- @SuppressWarnings("PublicInnerClass")
- public static class ProcessHandshakeMessage implements GridHadoopMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Node ID. */
- private GridHadoopProcessDescriptor procDesc;
-
- /** */
- public ProcessHandshakeMessage() {
- // No-op.
- }
-
- /**
- * @param procDesc Process descriptor.
- */
- private ProcessHandshakeMessage(GridHadoopProcessDescriptor procDesc) {
- this.procDesc = procDesc;
- }
-
- /**
- * @return Process ID.
- */
- public GridHadoopProcessDescriptor processDescriptor() {
- return procDesc;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeObject(procDesc);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- procDesc = (GridHadoopProcessDescriptor)in.readObject();
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(ProcessHandshakeMessage.class, this);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopHandshakeTimeoutException.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopHandshakeTimeoutException.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopHandshakeTimeoutException.java
deleted file mode 100644
index e001dc9..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/GridHadoopHandshakeTimeoutException.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication;
-
-import org.apache.ignite.*;
-import org.jetbrains.annotations.*;
-
-/** Internal exception class for proper timeout handling. */
-class GridHadoopHandshakeTimeoutException extends IgniteCheckedException {
- /** */
- private static final long serialVersionUID = 0L;
-
- /**
- * @param msg Message.
- */
- GridHadoopHandshakeTimeoutException(String msg) {
- super(msg);
- }
-
- /**
- * @param msg Message.
- * @param cause Cause.
- */
- GridHadoopHandshakeTimeoutException(String msg, @Nullable Throwable cause) {
- super(msg, cause);
- }
-}
[03/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopPopularWordsTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopPopularWordsTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopPopularWordsTest.java
new file mode 100644
index 0000000..a2f2ac3
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopPopularWordsTest.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import com.google.common.collect.*;
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.*;
+import org.apache.hadoop.mapreduce.lib.output.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.Map.*;
+
+import static com.google.common.collect.Maps.*;
+import static com.google.common.collect.MinMaxPriorityQueue.*;
+import static java.util.Collections.*;
+
+/**
+ * Hadoop-based 10 popular words example: all files in a given directory are tokenized and for each word longer than
+ * 3 characters the number of occurrences ins calculated. Finally, 10 words with the highest occurrence count are
+ * output.
+ *
+ * NOTE: in order to run this example on Windows please ensure that cygwin is installed and available in the system
+ * path.
+ */
+public class HadoopPopularWordsTest {
+ /** Ignite home. */
+ private static final String IGNITE_HOME = U.getIgniteHome();
+
+ /** The path to the input directory. ALl files in that directory will be processed. */
+ private static final Path BOOKS_LOCAL_DIR =
+ new Path("file:" + IGNITE_HOME, "modules/tests/java/org/apache/ignite/grid/hadoop/books");
+
+ /** The path to the output directory. THe result file will be written to this location. */
+ private static final Path RESULT_LOCAL_DIR =
+ new Path("file:" + IGNITE_HOME, "modules/tests/java/org/apache/ignite/grid/hadoop/output");
+
+ /** Popular books source dir in DFS. */
+ private static final Path BOOKS_DFS_DIR = new Path("tmp/word-count-example/in");
+
+ /** Popular books source dir in DFS. */
+ private static final Path RESULT_DFS_DIR = new Path("tmp/word-count-example/out");
+
+ /** Path to the distributed file system configuration. */
+ private static final String DFS_CFG = "examples/config/filesystem/core-site.xml";
+
+ /** Top N words to select **/
+ private static final int POPULAR_WORDS_CNT = 10;
+
+ /**
+ * For each token in the input string the mapper emits a {word, 1} pair.
+ */
+ private static class TokenizingMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
+ /** Constant value. */
+ private static final IntWritable ONE = new IntWritable(1);
+
+ /** The word converted into the Text. */
+ private Text word = new Text();
+
+ /**
+ * Emits a entry where the key is the word and the value is always 1.
+ *
+ * @param key the current position in the input file (not used here)
+ * @param val the text string
+ * @param ctx mapper context
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Override protected void map(LongWritable key, Text val, Context ctx)
+ throws IOException, InterruptedException {
+ // Get the mapped object.
+ final String line = val.toString();
+
+ // Splits the given string to words.
+ final String[] words = line.split("[^a-zA-Z0-9]");
+
+ for (final String w : words) {
+ // Only emit counts for longer words.
+ if (w.length() <= 3)
+ continue;
+
+ word.set(w);
+
+ // Write the word into the context with the initial count equals 1.
+ ctx.write(word, ONE);
+ }
+ }
+ }
+
+ /**
+ * The reducer uses a priority queue to rank the words based on its number of occurrences.
+ */
+ private static class TopNWordsReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
+ private MinMaxPriorityQueue<Entry<Integer, String>> q;
+
+ TopNWordsReducer() {
+ q = orderedBy(reverseOrder(new Comparator<Entry<Integer, String>>() {
+ @Override public int compare(Entry<Integer, String> o1, Entry<Integer, String> o2) {
+ return o1.getKey().compareTo(o2.getKey());
+ }
+ })).expectedSize(POPULAR_WORDS_CNT).maximumSize(POPULAR_WORDS_CNT).create();
+ }
+
+ /**
+ * This method doesn't emit anything, but just keeps track of the top N words.
+ *
+ * @param key The word.
+ * @param vals The words counts.
+ * @param ctx Reducer context.
+ * @throws IOException If failed.
+ * @throws InterruptedException If failed.
+ */
+ @Override public void reduce(Text key, Iterable<IntWritable> vals, Context ctx) throws IOException,
+ InterruptedException {
+ int sum = 0;
+
+ for (IntWritable val : vals)
+ sum += val.get();
+
+ q.add(immutableEntry(sum, key.toString()));
+ }
+
+ /**
+ * This method is called after all the word entries have been processed. It writes the accumulated
+ * statistics to the job output file.
+ *
+ * @param ctx The job context.
+ * @throws IOException If failed.
+ * @throws InterruptedException If failed.
+ */
+ @Override protected void cleanup(Context ctx) throws IOException, InterruptedException {
+ IntWritable i = new IntWritable();
+
+ Text txt = new Text();
+
+ // iterate in desc order
+ while (!q.isEmpty()) {
+ Entry<Integer, String> e = q.removeFirst();
+
+ i.set(e.getKey());
+
+ txt.set(e.getValue());
+
+ ctx.write(txt, i);
+ }
+ }
+ }
+
+ /**
+ * Configures the Hadoop MapReduce job.
+ *
+ * @return Instance of the Hadoop MapRed job.
+ * @throws IOException If failed.
+ */
+ private Job createConfigBasedHadoopJob() throws IOException {
+ Job jobCfg = new Job();
+
+ Configuration cfg = jobCfg.getConfiguration();
+
+ // Use explicit configuration of distributed file system, if provided.
+ if (DFS_CFG != null)
+ cfg.addResource(U.resolveIgniteUrl(DFS_CFG));
+
+ jobCfg.setJobName("HadoopPopularWordExample");
+ jobCfg.setJarByClass(HadoopPopularWordsTest.class);
+ jobCfg.setInputFormatClass(TextInputFormat.class);
+ jobCfg.setOutputKeyClass(Text.class);
+ jobCfg.setOutputValueClass(IntWritable.class);
+ jobCfg.setMapperClass(TokenizingMapper.class);
+ jobCfg.setReducerClass(TopNWordsReducer.class);
+
+ FileInputFormat.setInputPaths(jobCfg, BOOKS_DFS_DIR);
+ FileOutputFormat.setOutputPath(jobCfg, RESULT_DFS_DIR);
+
+ // Local job tracker allows the only task per wave, but text input format
+ // replaces it with the calculated value based on input split size option.
+ if ("local".equals(cfg.get("mapred.job.tracker", "local"))) {
+ // Split job into tasks using 32MB split size.
+ FileInputFormat.setMinInputSplitSize(jobCfg, 32 * 1024 * 1024);
+ FileInputFormat.setMaxInputSplitSize(jobCfg, Long.MAX_VALUE);
+ }
+
+ return jobCfg;
+ }
+
+ /**
+ * Runs the Hadoop job.
+ *
+ * @return {@code True} if succeeded, {@code false} otherwise.
+ * @throws Exception If failed.
+ */
+ private boolean runWordCountConfigBasedHadoopJob() throws Exception {
+ Job job = createConfigBasedHadoopJob();
+
+ // Distributed file system this job will work with.
+ FileSystem fs = FileSystem.get(job.getConfiguration());
+
+ X.println(">>> Using distributed file system: " + fs.getHomeDirectory());
+
+ // Prepare input and output job directories.
+ prepareDirectories(fs);
+
+ long time = System.currentTimeMillis();
+
+ // Run job.
+ boolean res = job.waitForCompletion(true);
+
+ X.println(">>> Job execution time: " + (System.currentTimeMillis() - time) / 1000 + " sec.");
+
+ // Move job results into local file system, so you can view calculated results.
+ publishResults(fs);
+
+ return res;
+ }
+
+ /**
+ * Prepare job's data: cleanup result directories that might have left over
+ * after previous runs, copy input files from the local file system into DFS.
+ *
+ * @param fs Distributed file system to use in job.
+ * @throws IOException If failed.
+ */
+ private void prepareDirectories(FileSystem fs) throws IOException {
+ X.println(">>> Cleaning up DFS result directory: " + RESULT_DFS_DIR);
+
+ fs.delete(RESULT_DFS_DIR, true);
+
+ X.println(">>> Cleaning up DFS input directory: " + BOOKS_DFS_DIR);
+
+ fs.delete(BOOKS_DFS_DIR, true);
+
+ X.println(">>> Copy local files into DFS input directory: " + BOOKS_DFS_DIR);
+
+ fs.copyFromLocalFile(BOOKS_LOCAL_DIR, BOOKS_DFS_DIR);
+ }
+
+ /**
+ * Publish job execution results into local file system, so you can view them.
+ *
+ * @param fs Distributed file sytem used in job.
+ * @throws IOException If failed.
+ */
+ private void publishResults(FileSystem fs) throws IOException {
+ X.println(">>> Cleaning up DFS input directory: " + BOOKS_DFS_DIR);
+
+ fs.delete(BOOKS_DFS_DIR, true);
+
+ X.println(">>> Cleaning up LOCAL result directory: " + RESULT_LOCAL_DIR);
+
+ fs.delete(RESULT_LOCAL_DIR, true);
+
+ X.println(">>> Moving job results into LOCAL result directory: " + RESULT_LOCAL_DIR);
+
+ fs.copyToLocalFile(true, RESULT_DFS_DIR, RESULT_LOCAL_DIR);
+ }
+
+ /**
+ * Executes a modified version of the Hadoop word count example. Here, in addition to counting the number of
+ * occurrences of the word in the source files, the N most popular words are selected.
+ *
+ * @param args None.
+ */
+ public static void main(String[] args) {
+ try {
+ new HadoopPopularWordsTest().runWordCountConfigBasedHadoopJob();
+ }
+ catch (Exception e) {
+ X.println(">>> Failed to run word count example: " + e.getMessage());
+ }
+
+ System.exit(0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSerializationWrapperSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSerializationWrapperSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSerializationWrapperSelfTest.java
new file mode 100644
index 0000000..5d5bb94
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSerializationWrapperSelfTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.io.serializer.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+import org.apache.ignite.testframework.junits.common.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Test of wrapper of the native serialization.
+ */
+public class HadoopSerializationWrapperSelfTest extends GridCommonAbstractTest {
+ /**
+ * Tests read/write of IntWritable via native WritableSerialization.
+ * @throws Exception If fails.
+ */
+ public void testIntWritableSerialization() throws Exception {
+ HadoopSerialization ser = new HadoopSerializationWrapper(new WritableSerialization(), IntWritable.class);
+
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+
+ DataOutput out = new DataOutputStream(buf);
+
+ ser.write(out, new IntWritable(3));
+ ser.write(out, new IntWritable(-5));
+
+ assertEquals("[0, 0, 0, 3, -1, -1, -1, -5]", Arrays.toString(buf.toByteArray()));
+
+ DataInput in = new DataInputStream(new ByteArrayInputStream(buf.toByteArray()));
+
+ assertEquals(3, ((IntWritable)ser.read(in, null)).get());
+ assertEquals(-5, ((IntWritable)ser.read(in, null)).get());
+ }
+
+ /**
+ * Tests read/write of Integer via native JavaleSerialization.
+ * @throws Exception If fails.
+ */
+ public void testIntJavaSerialization() throws Exception {
+ HadoopSerialization ser = new HadoopSerializationWrapper(new JavaSerialization(), Integer.class);
+
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+
+ DataOutput out = new DataOutputStream(buf);
+
+ ser.write(out, 3);
+ ser.write(out, -5);
+ ser.close();
+
+ DataInput in = new DataInputStream(new ByteArrayInputStream(buf.toByteArray()));
+
+ assertEquals(3, ((Integer)ser.read(in, null)).intValue());
+ assertEquals(-5, ((Integer)ser.read(in, null)).intValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSharedMap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSharedMap.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSharedMap.java
new file mode 100644
index 0000000..c73ee9f
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSharedMap.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.jdk8.backport.*;
+
+import java.util.concurrent.*;
+
+/**
+ * For tests.
+ */
+public class HadoopSharedMap {
+ /** */
+ private static final ConcurrentMap<String, HadoopSharedMap> maps = new ConcurrentHashMap8<>();
+
+ /** */
+ private final ConcurrentMap<String, Object> map = new ConcurrentHashMap8<>();
+
+ /**
+ * Private.
+ */
+ private HadoopSharedMap() {
+ // No-op.
+ }
+
+ /**
+ * Puts object by key.
+ *
+ * @param key Key.
+ * @param val Value.
+ */
+ public <T> T put(String key, T val) {
+ Object old = map.putIfAbsent(key, val);
+
+ return old == null ? val : (T)old;
+ }
+
+ /**
+ * @param cls Class.
+ * @return Map of static fields.
+ */
+ public static HadoopSharedMap map(Class<?> cls) {
+ HadoopSharedMap m = maps.get(cls.getName());
+
+ if (m != null)
+ return m;
+
+ HadoopSharedMap old = maps.putIfAbsent(cls.getName(), m = new HadoopSharedMap());
+
+ return old == null ? m : old;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSortingExternalTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSortingExternalTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSortingExternalTest.java
new file mode 100644
index 0000000..772e77d
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSortingExternalTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.configuration.*;
+
+/**
+ * External test for sorting.
+ */
+public class HadoopSortingExternalTest extends HadoopSortingTest {
+ /** {@inheritDoc} */
+ @Override public HadoopConfiguration hadoopConfiguration(String gridName) {
+ HadoopConfiguration cfg = super.hadoopConfiguration(gridName);
+
+ cfg.setExternalExecution(true);
+
+ return cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSortingTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSortingTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSortingTest.java
new file mode 100644
index 0000000..3f6594a
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSortingTest.java
@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.io.serializer.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.*;
+import org.apache.hadoop.mapreduce.lib.output.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.internal.util.typedef.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Tests correct sorting.
+ */
+public class HadoopSortingTest extends HadoopAbstractSelfTest {
+ /** */
+ private static final String PATH_INPUT = "/test-in";
+
+ /** */
+ private static final String PATH_OUTPUT = "/test-out";
+
+ /** {@inheritDoc} */
+ @Override protected int gridCount() {
+ return 3;
+ }
+
+ /**
+ * @return {@code True} if IGFS is enabled on Hadoop nodes.
+ */
+ @Override protected boolean igfsEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ startGrids(gridCount());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopConfiguration hadoopConfiguration(String gridName) {
+ HadoopConfiguration cfg = super.hadoopConfiguration(gridName);
+
+ cfg.setExternalExecution(false);
+
+ return cfg;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSortSimple() throws Exception {
+ // Generate test data.
+ Job job = Job.getInstance();
+
+ job.setInputFormatClass(InFormat.class);
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(NullWritable.class);
+
+ job.setMapperClass(Mapper.class);
+ job.setNumReduceTasks(0);
+
+ setupFileSystems(job.getConfiguration());
+
+ FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_INPUT));
+
+ X.printerrln("Data generation started.");
+
+ grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1),
+ createJobInfo(job.getConfiguration())).get(180000);
+
+ X.printerrln("Data generation complete.");
+
+ // Run main map-reduce job.
+ job = Job.getInstance();
+
+ setupFileSystems(job.getConfiguration());
+
+ job.getConfiguration().set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, JavaSerialization.class.getName() +
+ "," + WritableSerialization.class.getName());
+
+ FileInputFormat.setInputPaths(job, new Path(igfsScheme() + PATH_INPUT));
+ FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT));
+
+ job.setSortComparatorClass(JavaSerializationComparator.class);
+
+ job.setMapperClass(MyMapper.class);
+ job.setReducerClass(MyReducer.class);
+
+ job.setNumReduceTasks(2);
+
+ job.setMapOutputKeyClass(UUID.class);
+ job.setMapOutputValueClass(NullWritable.class);
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(NullWritable.class);
+
+ X.printerrln("Job started.");
+
+ grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 2),
+ createJobInfo(job.getConfiguration())).get(180000);
+
+ X.printerrln("Job complete.");
+
+ // Check result.
+ Path outDir = new Path(igfsScheme() + PATH_OUTPUT);
+
+ AbstractFileSystem fs = AbstractFileSystem.get(new URI(igfsScheme()), job.getConfiguration());
+
+ for (FileStatus file : fs.listStatus(outDir)) {
+ X.printerrln("__ file: " + file);
+
+ if (file.getLen() == 0)
+ continue;
+
+ FSDataInputStream in = fs.open(file.getPath());
+
+ Scanner sc = new Scanner(in);
+
+ UUID prev = null;
+
+ while(sc.hasNextLine()) {
+ UUID next = UUID.fromString(sc.nextLine());
+
+// X.printerrln("___ check: " + next);
+
+ if (prev != null)
+ assertTrue(prev.compareTo(next) < 0);
+
+ prev = next;
+ }
+ }
+ }
+
+ public static class InFormat extends InputFormat<Text, NullWritable> {
+ /** {@inheritDoc} */
+ @Override public List<InputSplit> getSplits(JobContext ctx) throws IOException, InterruptedException {
+ List<InputSplit> res = new ArrayList<>();
+
+ FakeSplit split = new FakeSplit(20);
+
+ for (int i = 0; i < 10; i++)
+ res.add(split);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public RecordReader<Text, NullWritable> createRecordReader(final InputSplit split,
+ TaskAttemptContext ctx) throws IOException, InterruptedException {
+ return new RecordReader<Text, NullWritable>() {
+ /** */
+ int cnt;
+
+ /** */
+ Text txt = new Text();
+
+ @Override public void initialize(InputSplit split, TaskAttemptContext ctx) {
+ // No-op.
+ }
+
+ @Override public boolean nextKeyValue() throws IOException, InterruptedException {
+ return ++cnt <= split.getLength();
+ }
+
+ @Override public Text getCurrentKey() {
+ txt.set(UUID.randomUUID().toString());
+
+// X.printerrln("___ read: " + txt);
+
+ return txt;
+ }
+
+ @Override public NullWritable getCurrentValue() {
+ return NullWritable.get();
+ }
+
+ @Override public float getProgress() throws IOException, InterruptedException {
+ return (float)cnt / split.getLength();
+ }
+
+ @Override public void close() {
+ // No-op.
+ }
+ };
+ }
+ }
+
+ public static class MyMapper extends Mapper<LongWritable, Text, UUID, NullWritable> {
+ /** {@inheritDoc} */
+ @Override protected void map(LongWritable key, Text val, Context ctx) throws IOException, InterruptedException {
+// X.printerrln("___ map: " + val);
+
+ ctx.write(UUID.fromString(val.toString()), NullWritable.get());
+ }
+ }
+
+ public static class MyReducer extends Reducer<UUID, NullWritable, Text, NullWritable> {
+ /** */
+ private Text text = new Text();
+
+ /** {@inheritDoc} */
+ @Override protected void reduce(UUID key, Iterable<NullWritable> vals, Context ctx)
+ throws IOException, InterruptedException {
+// X.printerrln("___ rdc: " + key);
+
+ text.set(key.toString());
+
+ ctx.write(text, NullWritable.get());
+ }
+ }
+
+ public static class FakeSplit extends InputSplit implements Writable {
+ /** */
+ private static final String[] HOSTS = {"127.0.0.1"};
+
+ /** */
+ private int len;
+
+ /**
+ * @param len Length.
+ */
+ public FakeSplit(int len) {
+ this.len = len;
+ }
+
+ /**
+ *
+ */
+ public FakeSplit() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLength() throws IOException, InterruptedException {
+ return len;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String[] getLocations() throws IOException, InterruptedException {
+ return HOSTS;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(DataOutput out) throws IOException {
+ out.writeInt(len);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFields(DataInput in) throws IOException {
+ len = in.readInt();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSplitWrapperSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSplitWrapperSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSplitWrapperSelfTest.java
new file mode 100644
index 0000000..ee490be
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSplitWrapperSelfTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.mapreduce.lib.input.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+import org.apache.ignite.testframework.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+/**
+ * Self test of {@link org.apache.ignite.internal.processors.hadoop.v2.HadoopSplitWrapper}.
+ */
+public class HadoopSplitWrapperSelfTest extends HadoopAbstractSelfTest {
+ /**
+ * Tests serialization of wrapper and the wrapped native split.
+ * @throws Exception If fails.
+ */
+ public void testSerialization() throws Exception {
+ FileSplit nativeSplit = new FileSplit(new Path("/path/to/file"), 100, 500, new String[]{"host1", "host2"});
+
+ assertEquals("/path/to/file:100+500", nativeSplit.toString());
+
+ HadoopSplitWrapper split = HadoopUtils.wrapSplit(10, nativeSplit, nativeSplit.getLocations());
+
+ assertEquals("[host1, host2]", Arrays.toString(split.hosts()));
+
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+
+ ObjectOutput out = new ObjectOutputStream(buf);
+
+ out.writeObject(split);
+
+ ObjectInput in = new ObjectInputStream(new ByteArrayInputStream(buf.toByteArray()));
+
+ final HadoopSplitWrapper res = (HadoopSplitWrapper)in.readObject();
+
+ assertEquals("/path/to/file:100+500", HadoopUtils.unwrapSplit(res).toString());
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ res.hosts();
+
+ return null;
+ }
+ }, AssertionError.class, null);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopStartup.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopStartup.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopStartup.java
new file mode 100644
index 0000000..1a93223
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopStartup.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.conf.*;
+import org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem;
+import org.apache.ignite.internal.util.typedef.*;
+
+/**
+ * Hadoop node startup.
+ */
+public class HadoopStartup {
+ /**
+ * @param args Arguments.
+ */
+ public static void main(String[] args) {
+ G.start("config/hadoop/default-config.xml");
+ }
+
+ /**
+ * @return Configuration for job run.
+ */
+ @SuppressWarnings("UnnecessaryFullyQualifiedName")
+ public static Configuration configuration() {
+ Configuration cfg = new Configuration();
+
+ cfg.set("fs.defaultFS", "igfs://igfs@localhost");
+
+ cfg.set("fs.igfs.impl", org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.class.getName());
+ cfg.set("fs.AbstractFileSystem.igfs.impl", IgniteHadoopFileSystem.class.getName());
+
+ cfg.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
+
+ cfg.set("mapreduce.framework.name", "ignite");
+ cfg.set("mapreduce.jobtracker.address", "localhost:11211");
+
+ return cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskExecutionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskExecutionSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskExecutionSelfTest.java
new file mode 100644
index 0000000..20c5db2
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskExecutionSelfTest.java
@@ -0,0 +1,551 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.*;
+import org.apache.hadoop.mapreduce.lib.output.*;
+import org.apache.ignite.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.hadoop.fs.v1.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.testframework.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Tests map-reduce task execution basics.
+ */
+public class HadoopTaskExecutionSelfTest extends HadoopAbstractSelfTest {
+ /** */
+ private static HadoopSharedMap m = HadoopSharedMap.map(HadoopTaskExecutionSelfTest.class);
+
+ /** Line count. */
+ private static final AtomicInteger totalLineCnt = m.put("totalLineCnt", new AtomicInteger());
+
+ /** Executed tasks. */
+ private static final AtomicInteger executedTasks = m.put("executedTasks", new AtomicInteger());
+
+ /** Cancelled tasks. */
+ private static final AtomicInteger cancelledTasks = m.put("cancelledTasks", new AtomicInteger());
+
+ /** Working directory of each task. */
+ private static final Map<String, String> taskWorkDirs = m.put("taskWorkDirs",
+ new ConcurrentHashMap<String, String>());
+
+ /** Mapper id to fail. */
+ private static final AtomicInteger failMapperId = m.put("failMapperId", new AtomicInteger());
+
+ /** Number of splits of the current input. */
+ private static final AtomicInteger splitsCount = m.put("splitsCount", new AtomicInteger());
+
+ /** Test param. */
+ private static final String MAP_WRITE = "test.map.write";
+
+
+ /** {@inheritDoc} */
+ @Override public FileSystemConfiguration igfsConfiguration() {
+ FileSystemConfiguration cfg = super.igfsConfiguration();
+
+ cfg.setFragmentizerEnabled(false);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean igfsEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ startGrids(gridCount());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ super.afterTestsStopped();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ grid(0).fileSystem(igfsName).format();
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopConfiguration hadoopConfiguration(String gridName) {
+ HadoopConfiguration cfg = super.hadoopConfiguration(gridName);
+
+ cfg.setMaxParallelTasks(5);
+ cfg.setExternalExecution(false);
+
+ return cfg;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMapRun() throws Exception {
+ int lineCnt = 10000;
+ String fileName = "/testFile";
+
+ prepareFile(fileName, lineCnt);
+
+ totalLineCnt.set(0);
+ taskWorkDirs.clear();
+
+ Configuration cfg = new Configuration();
+
+ cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
+
+ Job job = Job.getInstance(cfg);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ job.setMapperClass(TestMapper.class);
+
+ job.setNumReduceTasks(0);
+
+ job.setInputFormatClass(TextInputFormat.class);
+
+ FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
+ FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output/"));
+
+ job.setJarByClass(getClass());
+
+ IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1),
+ createJobInfo(job.getConfiguration()));
+
+ fut.get();
+
+ assertEquals(lineCnt, totalLineCnt.get());
+
+ assertEquals(32, taskWorkDirs.size());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMapCombineRun() throws Exception {
+ int lineCnt = 10001;
+ String fileName = "/testFile";
+
+ prepareFile(fileName, lineCnt);
+
+ totalLineCnt.set(0);
+ taskWorkDirs.clear();
+
+ Configuration cfg = new Configuration();
+
+ cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
+ cfg.setBoolean(MAP_WRITE, true);
+
+ Job job = Job.getInstance(cfg);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ job.setMapperClass(TestMapper.class);
+ job.setCombinerClass(TestCombiner.class);
+ job.setReducerClass(TestReducer.class);
+
+ job.setNumReduceTasks(2);
+
+ job.setInputFormatClass(TextInputFormat.class);
+
+ FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
+ FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output"));
+
+ job.setJarByClass(getClass());
+
+ HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 2);
+
+ IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));
+
+ fut.get();
+
+ assertEquals(lineCnt, totalLineCnt.get());
+
+ assertEquals(34, taskWorkDirs.size());
+
+ for (int g = 0; g < gridCount(); g++)
+ grid(g).hadoop().finishFuture(jobId).get();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMapperException() throws Exception {
+ prepareFile("/testFile", 1000);
+
+ Configuration cfg = new Configuration();
+
+ cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
+
+ Job job = Job.getInstance(cfg);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ job.setMapperClass(FailMapper.class);
+
+ job.setNumReduceTasks(0);
+
+ job.setInputFormatClass(TextInputFormat.class);
+
+ FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
+ FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output/"));
+
+ job.setJarByClass(getClass());
+
+ final IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 3),
+ createJobInfo(job.getConfiguration()));
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fut.get();
+
+ return null;
+ }
+ }, IgniteCheckedException.class, null);
+ }
+
+ /**
+ * @param fileName File name.
+ * @param lineCnt Line count.
+ * @throws Exception If failed.
+ */
+ private void prepareFile(String fileName, int lineCnt) throws Exception {
+ IgniteFileSystem igfs = grid(0).fileSystem(igfsName);
+
+ try (OutputStream os = igfs.create(new IgfsPath(fileName), true)) {
+ PrintWriter w = new PrintWriter(new OutputStreamWriter(os));
+
+ for (int i = 0; i < lineCnt; i++)
+ w.print("Hello, Hadoop map-reduce!\n");
+
+ w.flush();
+ }
+ }
+
+ /**
+ * Prepare job with mappers to cancel.
+ * @return Fully configured job.
+ * @throws Exception If fails.
+ */
+ private Configuration prepareJobForCancelling() throws Exception {
+ prepareFile("/testFile", 1500);
+
+ executedTasks.set(0);
+ cancelledTasks.set(0);
+ failMapperId.set(0);
+ splitsCount.set(0);
+
+ Configuration cfg = new Configuration();
+
+ setupFileSystems(cfg);
+
+ Job job = Job.getInstance(cfg);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ job.setMapperClass(CancellingTestMapper.class);
+
+ job.setNumReduceTasks(0);
+
+ job.setInputFormatClass(InFormat.class);
+
+ FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
+ FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output/"));
+
+ job.setJarByClass(getClass());
+
+ return job.getConfiguration();
+ }
+
+ /**
+ * Test input format.
+ */
+ private static class InFormat extends TextInputFormat {
+ @Override public List<InputSplit> getSplits(JobContext ctx) throws IOException {
+ List<InputSplit> res = super.getSplits(ctx);
+
+ splitsCount.set(res.size());
+
+ X.println("___ split of input: " + splitsCount.get());
+
+ return res;
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testTaskCancelling() throws Exception {
+ Configuration cfg = prepareJobForCancelling();
+
+ HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 1);
+
+ final IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(cfg));
+
+ if (!GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return splitsCount.get() > 0;
+ }
+ }, 20000)) {
+ U.dumpThreads(log);
+
+ assertTrue(false);
+ }
+
+ if (!GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return executedTasks.get() == splitsCount.get();
+ }
+ }, 20000)) {
+ U.dumpThreads(log);
+
+ assertTrue(false);
+ }
+
+ // Fail mapper with id "1", cancels others
+ failMapperId.set(1);
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fut.get();
+
+ return null;
+ }
+ }, IgniteCheckedException.class, null);
+
+ assertEquals(executedTasks.get(), cancelledTasks.get() + 1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testJobKill() throws Exception {
+ Configuration cfg = prepareJobForCancelling();
+
+ Hadoop hadoop = grid(0).hadoop();
+
+ HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 1);
+
+ //Kill unknown job.
+ boolean killRes = hadoop.kill(jobId);
+
+ assertFalse(killRes);
+
+ final IgniteInternalFuture<?> fut = hadoop.submit(jobId, createJobInfo(cfg));
+
+ if (!GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return splitsCount.get() > 0;
+ }
+ }, 20000)) {
+ U.dumpThreads(log);
+
+ assertTrue(false);
+ }
+
+ if (!GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ X.println("___ executed tasks: " + executedTasks.get());
+
+ return executedTasks.get() == splitsCount.get();
+ }
+ }, 20000)) {
+ U.dumpThreads(log);
+
+ fail();
+ }
+
+ //Kill really ran job.
+ killRes = hadoop.kill(jobId);
+
+ assertTrue(killRes);
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fut.get();
+
+ return null;
+ }
+ }, IgniteCheckedException.class, null);
+
+ assertEquals(executedTasks.get(), cancelledTasks.get());
+
+ //Kill the same job again.
+ killRes = hadoop.kill(jobId);
+
+ assertTrue(killRes);
+ }
+
+ private static class CancellingTestMapper extends Mapper<Object, Text, Text, IntWritable> {
+ private int mapperId;
+
+ /** {@inheritDoc} */
+ @Override protected void setup(Context ctx) throws IOException, InterruptedException {
+ mapperId = executedTasks.incrementAndGet();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run(Context ctx) throws IOException, InterruptedException {
+ try {
+ super.run(ctx);
+ }
+ catch (HadoopTaskCancelledException e) {
+ cancelledTasks.incrementAndGet();
+
+ throw e;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
+ if (mapperId == failMapperId.get())
+ throw new IOException();
+
+ Thread.sleep(1000);
+ }
+ }
+
+ /**
+ * Test failing mapper.
+ */
+ private static class FailMapper extends Mapper<Object, Text, Text, IntWritable> {
+ /** {@inheritDoc} */
+ @Override protected void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
+ throw new IOException("Expected");
+ }
+ }
+
+ /**
+ * Mapper calculates number of lines.
+ */
+ private static class TestMapper extends Mapper<Object, Text, Text, IntWritable> {
+ /** Writable integer constant of '1'. */
+ private static final IntWritable ONE = new IntWritable(1);
+
+ /** Line count constant. */
+ public static final Text LINE_COUNT = new Text("lineCount");
+
+ /** {@inheritDoc} */
+ @Override protected void setup(Context ctx) throws IOException, InterruptedException {
+ X.println("___ Mapper: " + ctx.getTaskAttemptID());
+
+ String taskId = ctx.getTaskAttemptID().toString();
+
+ LocalFileSystem locFs = FileSystem.getLocal(ctx.getConfiguration());
+
+ String workDir = locFs.getWorkingDirectory().toString();
+
+ assertNull(taskWorkDirs.put(workDir, taskId));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
+ if (ctx.getConfiguration().getBoolean(MAP_WRITE, false))
+ ctx.write(LINE_COUNT, ONE);
+ else
+ totalLineCnt.incrementAndGet();
+ }
+ }
+
+ /**
+ * Combiner calculates number of lines.
+ */
+ private static class TestCombiner extends Reducer<Text, IntWritable, Text, IntWritable> {
+ /** */
+ IntWritable sum = new IntWritable();
+
+ /** {@inheritDoc} */
+ @Override protected void setup(Context ctx) throws IOException, InterruptedException {
+ X.println("___ Combiner: ");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void reduce(Text key, Iterable<IntWritable> values, Context ctx) throws IOException,
+ InterruptedException {
+ int lineCnt = 0;
+
+ for (IntWritable value : values)
+ lineCnt += value.get();
+
+ sum.set(lineCnt);
+
+ X.println("___ combo: " + lineCnt);
+
+ ctx.write(key, sum);
+ }
+ }
+
+ /**
+ * Combiner calculates number of lines.
+ */
+ private static class TestReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
+ /** */
+ IntWritable sum = new IntWritable();
+
+ /** {@inheritDoc} */
+ @Override protected void setup(Context ctx) throws IOException, InterruptedException {
+ X.println("___ Reducer: " + ctx.getTaskAttemptID());
+
+ String taskId = ctx.getTaskAttemptID().toString();
+ String workDir = FileSystem.getLocal(ctx.getConfiguration()).getWorkingDirectory().toString();
+
+ assertNull(taskWorkDirs.put(workDir, taskId));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void reduce(Text key, Iterable<IntWritable> values, Context ctx) throws IOException,
+ InterruptedException {
+ int lineCnt = 0;
+
+ for (IntWritable value : values) {
+ lineCnt += value.get();
+
+ X.println("___ rdcr: " + value.get());
+ }
+
+ sum.set(lineCnt);
+
+ ctx.write(key, sum);
+
+ X.println("___ RDCR SUM: " + lineCnt);
+
+ totalLineCnt.addAndGet(lineCnt);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTasksAllVersionsTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTasksAllVersionsTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTasksAllVersionsTest.java
new file mode 100644
index 0000000..aaf0f92
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTasksAllVersionsTest.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import com.google.common.base.*;
+import org.apache.hadoop.io.*;
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.processors.hadoop.examples.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+/**
+ * Tests of Map, Combine and Reduce task executions of any version of hadoop API.
+ */
+abstract class HadoopTasksAllVersionsTest extends HadoopAbstractWordCountTest {
+ /** Empty hosts array. */
+ private static final String[] HOSTS = new String[0];
+
+ /**
+ * Creates some grid hadoop job. Override this method to create tests for any job implementation.
+ *
+ * @param inFile Input file name for the job.
+ * @param outFile Output file name for the job.
+ * @return Hadoop job.
+ * @throws IOException If fails.
+ */
+ public abstract HadoopV2Job getHadoopJob(String inFile, String outFile) throws Exception;
+
+ /**
+ * @return prefix of reducer output file name. It's "part-" for v1 and "part-r-" for v2 API
+ */
+ public abstract String getOutputFileNamePrefix();
+
+ /**
+ * Tests map task execution.
+ *
+ * @throws Exception If fails.
+ */
+ @SuppressWarnings("ConstantConditions")
+ public void testMapTask() throws Exception {
+ IgfsPath inDir = new IgfsPath(PATH_INPUT);
+
+ igfs.mkdirs(inDir);
+
+ IgfsPath inFile = new IgfsPath(inDir, HadoopWordCount2.class.getSimpleName() + "-input");
+
+ URI inFileUri = URI.create(igfsScheme() + inFile.toString());
+
+ try (PrintWriter pw = new PrintWriter(igfs.create(inFile, true))) {
+ pw.println("hello0 world0");
+ pw.println("world1 hello1");
+ }
+
+ HadoopFileBlock fileBlock1 = new HadoopFileBlock(HOSTS, inFileUri, 0, igfs.info(inFile).length() - 1);
+
+ try (PrintWriter pw = new PrintWriter(igfs.append(inFile, false))) {
+ pw.println("hello2 world2");
+ pw.println("world3 hello3");
+ }
+ HadoopFileBlock fileBlock2 = new HadoopFileBlock(HOSTS, inFileUri, fileBlock1.length(),
+ igfs.info(inFile).length() - fileBlock1.length());
+
+ HadoopV2Job gridJob = getHadoopJob(igfsScheme() + inFile.toString(), igfsScheme() + PATH_OUTPUT);
+
+ HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.MAP, gridJob.id(), 0, 0, fileBlock1);
+
+ HadoopTestTaskContext ctx = new HadoopTestTaskContext(taskInfo, gridJob);
+
+ ctx.mockOutput().clear();
+
+ ctx.run();
+
+ assertEquals("hello0,1; world0,1; world1,1; hello1,1", Joiner.on("; ").join(ctx.mockOutput()));
+
+ ctx.mockOutput().clear();
+
+ ctx.taskInfo(new HadoopTaskInfo(HadoopTaskType.MAP, gridJob.id(), 0, 0, fileBlock2));
+
+ ctx.run();
+
+ assertEquals("hello2,1; world2,1; world3,1; hello3,1", Joiner.on("; ").join(ctx.mockOutput()));
+ }
+
+ /**
+ * Generates input data for reduce-like operation into mock context input and runs the operation.
+ *
+ * @param gridJob Job is to create reduce task from.
+ * @param taskType Type of task - combine or reduce.
+ * @param taskNum Number of task in job.
+ * @param words Pairs of words and its counts.
+ * @return Context with mock output.
+ * @throws IgniteCheckedException If fails.
+ */
+ private HadoopTestTaskContext runTaskWithInput(HadoopV2Job gridJob, HadoopTaskType taskType,
+ int taskNum, String... words) throws IgniteCheckedException {
+ HadoopTaskInfo taskInfo = new HadoopTaskInfo(taskType, gridJob.id(), taskNum, 0, null);
+
+ HadoopTestTaskContext ctx = new HadoopTestTaskContext(taskInfo, gridJob);
+
+ for (int i = 0; i < words.length; i+=2) {
+ List<IntWritable> valList = new ArrayList<>();
+
+ for (int j = 0; j < Integer.parseInt(words[i + 1]); j++)
+ valList.add(new IntWritable(1));
+
+ ctx.mockInput().put(new Text(words[i]), valList);
+ }
+
+ ctx.run();
+
+ return ctx;
+ }
+
+ /**
+ * Tests reduce task execution.
+ *
+ * @throws Exception If fails.
+ */
+ public void testReduceTask() throws Exception {
+ HadoopV2Job gridJob = getHadoopJob(igfsScheme() + PATH_INPUT, igfsScheme() + PATH_OUTPUT);
+
+ runTaskWithInput(gridJob, HadoopTaskType.REDUCE, 0, "word1", "5", "word2", "10");
+ runTaskWithInput(gridJob, HadoopTaskType.REDUCE, 1, "word3", "7", "word4", "15");
+
+ assertEquals(
+ "word1\t5\n" +
+ "word2\t10\n",
+ readAndSortFile(PATH_OUTPUT + "/_temporary/0/task_00000000-0000-0000-0000-000000000000_0000_r_000000/" +
+ getOutputFileNamePrefix() + "00000")
+ );
+
+ assertEquals(
+ "word3\t7\n" +
+ "word4\t15\n",
+ readAndSortFile(PATH_OUTPUT + "/_temporary/0/task_00000000-0000-0000-0000-000000000000_0000_r_000001/" +
+ getOutputFileNamePrefix() + "00001")
+ );
+ }
+
+ /**
+ * Tests combine task execution.
+ *
+ * @throws Exception If fails.
+ */
+ public void testCombinerTask() throws Exception {
+ HadoopV2Job gridJob = getHadoopJob("/", "/");
+
+ HadoopTestTaskContext ctx =
+ runTaskWithInput(gridJob, HadoopTaskType.COMBINE, 0, "word1", "5", "word2", "10");
+
+ assertEquals("word1,5; word2,10", Joiner.on("; ").join(ctx.mockOutput()));
+
+ ctx = runTaskWithInput(gridJob, HadoopTaskType.COMBINE, 1, "word3", "7", "word4", "15");
+
+ assertEquals("word3,7; word4,15", Joiner.on("; ").join(ctx.mockOutput()));
+ }
+
+ /**
+ * Runs chain of map-combine task on file block.
+ *
+ * @param fileBlock block of input file to be processed.
+ * @param gridJob Hadoop job implementation.
+ * @return Context of combine task with mock output.
+ * @throws IgniteCheckedException If fails.
+ */
+ private HadoopTestTaskContext runMapCombineTask(HadoopFileBlock fileBlock, HadoopV2Job gridJob)
+ throws IgniteCheckedException {
+ HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.MAP, gridJob.id(), 0, 0, fileBlock);
+
+ HadoopTestTaskContext mapCtx = new HadoopTestTaskContext(taskInfo, gridJob);
+
+ mapCtx.run();
+
+ //Prepare input for combine
+ taskInfo = new HadoopTaskInfo(HadoopTaskType.COMBINE, gridJob.id(), 0, 0, null);
+
+ HadoopTestTaskContext combineCtx = new HadoopTestTaskContext(taskInfo, gridJob);
+
+ combineCtx.makeTreeOfWritables(mapCtx.mockOutput());
+
+ combineCtx.run();
+
+ return combineCtx;
+ }
+
+ /**
+ * Tests all job in complex.
+ * Runs 2 chains of map-combine tasks and sends result into one reduce task.
+ *
+ * @throws Exception If fails.
+ */
+ @SuppressWarnings("ConstantConditions")
+ public void testAllTasks() throws Exception {
+ IgfsPath inDir = new IgfsPath(PATH_INPUT);
+
+ igfs.mkdirs(inDir);
+
+ IgfsPath inFile = new IgfsPath(inDir, HadoopWordCount2.class.getSimpleName() + "-input");
+
+ URI inFileUri = URI.create(igfsScheme() + inFile.toString());
+
+ generateTestFile(inFile.toString(), "red", 100, "blue", 200, "green", 150, "yellow", 70);
+
+ //Split file into two blocks
+ long fileLen = igfs.info(inFile).length();
+
+ Long l = fileLen / 2;
+
+ HadoopFileBlock fileBlock1 = new HadoopFileBlock(HOSTS, inFileUri, 0, l);
+ HadoopFileBlock fileBlock2 = new HadoopFileBlock(HOSTS, inFileUri, l, fileLen - l);
+
+ HadoopV2Job gridJob = getHadoopJob(inFileUri.toString(), igfsScheme() + PATH_OUTPUT);
+
+ HadoopTestTaskContext combine1Ctx = runMapCombineTask(fileBlock1, gridJob);
+
+ HadoopTestTaskContext combine2Ctx = runMapCombineTask(fileBlock2, gridJob);
+
+ //Prepare input for combine
+ HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.REDUCE, gridJob.id(), 0, 0, null);
+
+ HadoopTestTaskContext reduceCtx = new HadoopTestTaskContext(taskInfo, gridJob);
+
+ reduceCtx.makeTreeOfWritables(combine1Ctx.mockOutput());
+ reduceCtx.makeTreeOfWritables(combine2Ctx.mockOutput());
+
+ reduceCtx.run();
+
+ reduceCtx.taskInfo(new HadoopTaskInfo(HadoopTaskType.COMMIT, gridJob.id(), 0, 0, null));
+
+ reduceCtx.run();
+
+ assertEquals(
+ "blue\t200\n" +
+ "green\t150\n" +
+ "red\t100\n" +
+ "yellow\t70\n",
+ readAndSortFile(PATH_OUTPUT + "/" + getOutputFileNamePrefix() + "00000")
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTasksV1Test.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTasksV1Test.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTasksV1Test.java
new file mode 100644
index 0000000..b41a260
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTasksV1Test.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.mapred.*;
+import org.apache.ignite.internal.processors.hadoop.examples.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Tests of Map, Combine and Reduce task executions via running of job of hadoop API v1.
+ */
+public class HadoopTasksV1Test extends HadoopTasksAllVersionsTest {
+ /**
+ * Creates WordCount hadoop job for API v1.
+ *
+ * @param inFile Input file name for the job.
+ * @param outFile Output file name for the job.
+ * @return Hadoop job.
+ * @throws IOException If fails.
+ */
+ @Override public HadoopV2Job getHadoopJob(String inFile, String outFile) throws Exception {
+ JobConf jobConf = HadoopWordCount1.getJob(inFile, outFile);
+
+ setupFileSystems(jobConf);
+
+ HadoopDefaultJobInfo jobInfo = createJobInfo(jobConf);
+
+ HadoopJobId jobId = new HadoopJobId(new UUID(0, 0), 0);
+
+ return new HadoopV2Job(jobId, jobInfo, log);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getOutputFileNamePrefix() {
+ return "part-";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTasksV2Test.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTasksV2Test.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTasksV2Test.java
new file mode 100644
index 0000000..b677c63
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTasksV2Test.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.*;
+import org.apache.hadoop.mapreduce.lib.output.*;
+import org.apache.ignite.internal.processors.hadoop.examples.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Tests of Map, Combine and Reduce task executions via running of job of hadoop API v2.
+ */
+public class HadoopTasksV2Test extends HadoopTasksAllVersionsTest {
+ /**
+ * Creates WordCount hadoop job for API v2.
+ *
+ * @param inFile Input file name for the job.
+ * @param outFile Output file name for the job.
+ * @return Hadoop job.
+ * @throws Exception if fails.
+ */
+ @Override public HadoopV2Job getHadoopJob(String inFile, String outFile) throws Exception {
+ Job job = Job.getInstance();
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ HadoopWordCount2.setTasksClasses(job, true, true, true);
+
+ Configuration conf = job.getConfiguration();
+
+ setupFileSystems(conf);
+
+ FileInputFormat.setInputPaths(job, new Path(inFile));
+ FileOutputFormat.setOutputPath(job, new Path(outFile));
+
+ job.setJarByClass(HadoopWordCount2.class);
+
+ Job hadoopJob = HadoopWordCount2.getJob(inFile, outFile);
+
+ HadoopDefaultJobInfo jobInfo = createJobInfo(hadoopJob.getConfiguration());
+
+ HadoopJobId jobId = new HadoopJobId(new UUID(0, 0), 0);
+
+ return new HadoopV2Job(jobId, jobInfo, log);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getOutputFileNamePrefix() {
+ return "part-r-";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestRoundRobinMrPlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestRoundRobinMrPlanner.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestRoundRobinMrPlanner.java
new file mode 100644
index 0000000..a56c7c7
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestRoundRobinMrPlanner.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.apache.ignite.cluster.*;
+import org.apache.ignite.internal.processors.hadoop.planner.*;
+import org.jetbrains.annotations.*;
+
+import java.util.*;
+
+/**
+ * Round-robin mr planner.
+ */
+public class HadoopTestRoundRobinMrPlanner implements HadoopMapReducePlanner {
+ /** {@inheritDoc} */
+ @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> top,
+ @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
+ if (top.isEmpty())
+ throw new IllegalArgumentException("Topology is empty");
+
+ // Has at least one element.
+ Iterator<ClusterNode> it = top.iterator();
+
+ Map<UUID, Collection<HadoopInputSplit>> mappers = new HashMap<>();
+
+ for (HadoopInputSplit block : job.input()) {
+ ClusterNode node = it.next();
+
+ Collection<HadoopInputSplit> nodeBlocks = mappers.get(node.id());
+
+ if (nodeBlocks == null) {
+ nodeBlocks = new ArrayList<>();
+
+ mappers.put(node.id(), nodeBlocks);
+ }
+
+ nodeBlocks.add(block);
+
+ if (!it.hasNext())
+ it = top.iterator();
+ }
+
+ int[] rdc = new int[job.info().reducers()];
+
+ for (int i = 0; i < rdc.length; i++)
+ rdc[i] = i;
+
+ return new HadoopDefaultMapReducePlan(mappers, Collections.singletonMap(it.next().id(), rdc));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestTaskContext.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestTaskContext.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestTaskContext.java
new file mode 100644
index 0000000..e444270
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestTaskContext.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Context for test purpose.
+ */
+class HadoopTestTaskContext extends HadoopV2TaskContext {
+ /**
+ * Simple key-vale pair.
+ * @param <K> Key class.
+ * @param <V> Value class.
+ */
+ public static class Pair<K,V> {
+ /** Key */
+ private K key;
+
+ /** Value */
+ private V val;
+
+ /**
+ * @param key key.
+ * @param val value.
+ */
+ Pair(K key, V val) {
+ this.key = key;
+ this.val = val;
+ }
+
+ /**
+ * Getter of key.
+ * @return key.
+ */
+ K key() {
+ return key;
+ }
+
+ /**
+ * Getter of value.
+ * @return value.
+ */
+ V value() {
+ return val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return key + "," + val;
+ }
+ }
+
+ /** Mock output container- result data of task execution if it is not overridden. */
+ private List<Pair<String, Integer>> mockOutput = new ArrayList<>();
+
+ /** Mock input container- input data if it is not overridden. */
+ private Map<Object,List> mockInput = new TreeMap<>();
+
+ /** Context output implementation to write data into mockOutput. */
+ private HadoopTaskOutput output = new HadoopTaskOutput() {
+ /** {@inheritDoc} */
+ @Override public void write(Object key, Object val) {
+ //Check of casting and extract/copy values
+ String strKey = new String(((Text)key).getBytes());
+ int intVal = ((IntWritable)val).get();
+
+ mockOutput().add(new Pair<>(strKey, intVal));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ /** Context input implementation to read data from mockInput. */
+ private HadoopTaskInput input = new HadoopTaskInput() {
+ /** Iterator of keys and associated lists of values. */
+ Iterator<Map.Entry<Object, List>> iter;
+
+ /** Current key and associated value list. */
+ Map.Entry<Object, List> currEntry;
+
+ /** {@inheritDoc} */
+ @Override public boolean next() {
+ if (iter == null)
+ iter = mockInput().entrySet().iterator();
+
+ if (iter.hasNext())
+ currEntry = iter.next();
+ else
+ currEntry = null;
+
+ return currEntry != null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object key() {
+ return currEntry.getKey();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Iterator<?> values() {
+ return currEntry.getValue().iterator() ;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ /**
+ * Getter of mock output container - result of task if it is not overridden.
+ *
+ * @return mock output.
+ */
+ public List<Pair<String, Integer>> mockOutput() {
+ return mockOutput;
+ }
+
+ /**
+ * Getter of mock input container- input data if it is not overridden.
+ *
+ * @return mock output.
+ */
+ public Map<Object, List> mockInput() {
+ return mockInput;
+ }
+
+ /**
+ * Generate one-key-multiple-values tree from array of key-value pairs, and wrap its into Writable objects.
+ * The result is placed into mock input.
+ *
+ * @param flatData list of key-value pair.
+ */
+ public void makeTreeOfWritables(Iterable<Pair<String, Integer>> flatData) {
+ Text key = new Text();
+
+ for (HadoopTestTaskContext.Pair<String, Integer> pair : flatData) {
+ key.set(pair.key);
+ ArrayList<IntWritable> valList;
+
+ if (!mockInput.containsKey(key)) {
+ valList = new ArrayList<>();
+ mockInput.put(key, valList);
+ key = new Text();
+ }
+ else
+ valList = (ArrayList<IntWritable>) mockInput.get(key);
+ valList.add(new IntWritable(pair.value()));
+ }
+ }
+
+ /**
+ * @param taskInfo Task info.
+ * @param gridJob Grid Hadoop job.
+ */
+ public HadoopTestTaskContext(HadoopTaskInfo taskInfo, HadoopJob gridJob) throws IgniteCheckedException {
+ super(taskInfo, gridJob, gridJob.id(), null, jobConfDataInput(gridJob));
+ }
+
+ /**
+ * Creates DataInput to read JobConf.
+ *
+ * @param job Job.
+ * @return DataInput with JobConf.
+ * @throws IgniteCheckedException If failed.
+ */
+ private static DataInput jobConfDataInput(HadoopJob job) throws IgniteCheckedException {
+ JobConf jobConf = new JobConf();
+
+ for (Map.Entry<String, String> e : ((HadoopDefaultJobInfo)job.info()).properties().entrySet())
+ jobConf.set(e.getKey(), e.getValue());
+
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+
+ try {
+ jobConf.write(new DataOutputStream(buf));
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+
+ return new DataInputStream(new ByteArrayInputStream(buf.toByteArray()));
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopTaskOutput output() {
+ return output;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopTaskInput input() {
+ return input;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java
new file mode 100644
index 0000000..ef60762
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.internal.util.typedef.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.junit.Assert.*;
+
+/**
+ * Utility class for tests.
+ */
+public class HadoopTestUtils {
+ /**
+ * Checks that job statistics file contains valid strings only.
+ *
+ * @param reader Buffered reader to get lines of job statistics.
+ * @return Amount of events.
+ * @throws IOException If failed.
+ */
+ public static long simpleCheckJobStatFile(BufferedReader reader) throws IOException {
+ Collection<String> phases = new HashSet<>();
+
+ phases.add("submit");
+ phases.add("prepare");
+ phases.add("start");
+ phases.add("finish");
+ phases.add("requestId");
+ phases.add("responseId");
+
+ Collection<String> evtTypes = new HashSet<>();
+
+ evtTypes.add("JOB");
+ evtTypes.add("SETUP");
+ evtTypes.add("MAP");
+ evtTypes.add("SHUFFLE");
+ evtTypes.add("REDUCE");
+ evtTypes.add("COMBINE");
+ evtTypes.add("COMMIT");
+
+ long evtCnt = 0;
+ String line;
+
+ Map<Long, String> reduceNodes = new HashMap<>();
+
+ while((line = reader.readLine()) != null) {
+ String[] splitLine = line.split(":");
+
+ //Try parse timestamp
+ Long.parseLong(splitLine[1]);
+
+ String[] evt = splitLine[0].split(" ");
+
+ assertTrue("Unknown event '" + evt[0] + "'", evtTypes.contains(evt[0]));
+
+ String phase;
+
+ if ("JOB".equals(evt[0]))
+ phase = evt[1];
+ else {
+ assertEquals(4, evt.length);
+ assertTrue("The node id is not defined", !F.isEmpty(evt[3]));
+
+ long taskNum = Long.parseLong(evt[1]);
+
+ if (("REDUCE".equals(evt[0]) || "SHUFFLE".equals(evt[0]))) {
+ String nodeId = reduceNodes.get(taskNum);
+
+ if (nodeId == null)
+ reduceNodes.put(taskNum, evt[3]);
+ else
+ assertEquals("Different nodes for SHUFFLE and REDUCE tasks", nodeId, evt[3]);
+ }
+
+ phase = evt[2];
+ }
+
+ assertTrue("Unknown phase '" + phase + "' in " + Arrays.toString(evt), phases.contains(phase));
+
+ evtCnt++;
+ }
+
+ return evtCnt;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopV2JobSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopV2JobSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopV2JobSelfTest.java
new file mode 100644
index 0000000..ebc89f4
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopV2JobSelfTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.io.serializer.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Self test of {@link org.apache.ignite.internal.processors.hadoop.v2.HadoopV2Job}.
+ */
+public class HadoopV2JobSelfTest extends HadoopAbstractSelfTest {
+ /** */
+ private static final String TEST_SERIALIZED_VALUE = "Test serialized value";
+
+ /**
+ * Custom serialization class that accepts {@link Writable}.
+ */
+ private static class CustomSerialization extends WritableSerialization {
+ /** {@inheritDoc} */
+ @Override public Deserializer<Writable> getDeserializer(Class<Writable> c) {
+ return new Deserializer<Writable>() {
+ @Override public void open(InputStream in) { }
+
+ @Override public Writable deserialize(Writable writable) {
+ return new Text(TEST_SERIALIZED_VALUE);
+ }
+
+ @Override public void close() { }
+ };
+ }
+ }
+
+ /**
+ * Tests that {@link HadoopJob} provides wrapped serializer if it's set in configuration.
+ *
+ * @throws IgniteCheckedException If fails.
+ */
+ public void testCustomSerializationApplying() throws IgniteCheckedException {
+ JobConf cfg = new JobConf();
+
+ cfg.setMapOutputKeyClass(IntWritable.class);
+ cfg.setMapOutputValueClass(Text.class);
+ cfg.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, CustomSerialization.class.getName());
+
+ HadoopJob job = new HadoopV2Job(new HadoopJobId(UUID.randomUUID(), 1), createJobInfo(cfg), log);
+
+ HadoopTaskContext taskCtx = job.getTaskContext(new HadoopTaskInfo(HadoopTaskType.MAP, null, 0, 0,
+ null));
+
+ HadoopSerialization ser = taskCtx.keySerialization();
+
+ assertEquals(HadoopSerializationWrapper.class.getName(), ser.getClass().getName());
+
+ DataInput in = new DataInputStream(new ByteArrayInputStream(new byte[0]));
+
+ assertEquals(TEST_SERIALIZED_VALUE, ser.read(in, null).toString());
+
+ ser = taskCtx.valueSerialization();
+
+ assertEquals(HadoopSerializationWrapper.class.getName(), ser.getClass().getName());
+
+ assertEquals(TEST_SERIALIZED_VALUE, ser.read(in, null).toString());
+ }
+}
[21/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutor.java
new file mode 100644
index 0000000..04a96de
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutor.java
@@ -0,0 +1,960 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.child.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.future.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
+import org.apache.ignite.spi.*;
+import org.jdk8.backport.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.locks.*;
+
+import static org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopTaskState.*;
+
+/**
+ * External process registry. Handles external process lifecycle.
+ */
+public class HadoopExternalTaskExecutor extends HadoopTaskExecutorAdapter {
+ /** Hadoop context. */
+ private HadoopContext ctx;
+
+ /** */
+ private String javaCmd;
+
+ /** Logger. */
+ private IgniteLogger log;
+
+ /** Node process descriptor. */
+ private HadoopProcessDescriptor nodeDesc;
+
+ /** Output base. */
+ private File outputBase;
+
+ /** Path separator. */
+ private String pathSep;
+
+ /** Hadoop external communication. */
+ private HadoopExternalCommunication comm;
+
+ /** Starting processes. */
+ private final ConcurrentMap<UUID, HadoopProcess> runningProcsByProcId = new ConcurrentHashMap8<>();
+
+ /** Starting processes. */
+ private final ConcurrentMap<HadoopJobId, HadoopProcess> runningProcsByJobId = new ConcurrentHashMap8<>();
+
+ /** Busy lock. */
+ private final GridSpinReadWriteLock busyLock = new GridSpinReadWriteLock();
+
+ /** Job tracker. */
+ private HadoopJobTracker jobTracker;
+
+ /** {@inheritDoc} */
+ @Override public void start(HadoopContext ctx) throws IgniteCheckedException {
+ this.ctx = ctx;
+
+ log = ctx.kernalContext().log(HadoopExternalTaskExecutor.class);
+
+ outputBase = U.resolveWorkDirectory("hadoop", false);
+
+ pathSep = System.getProperty("path.separator", U.isWindows() ? ";" : ":");
+
+ initJavaCommand();
+
+ comm = new HadoopExternalCommunication(
+ ctx.localNodeId(),
+ UUID.randomUUID(),
+ ctx.kernalContext().config().getMarshaller(),
+ log,
+ ctx.kernalContext().getSystemExecutorService(),
+ ctx.kernalContext().gridName());
+
+ comm.setListener(new MessageListener());
+
+ comm.start();
+
+ nodeDesc = comm.localProcessDescriptor();
+
+ ctx.kernalContext().ports().registerPort(nodeDesc.tcpPort(), IgnitePortProtocol.TCP,
+ HadoopExternalTaskExecutor.class);
+
+ if (nodeDesc.sharedMemoryPort() != -1)
+ ctx.kernalContext().ports().registerPort(nodeDesc.sharedMemoryPort(), IgnitePortProtocol.TCP,
+ HadoopExternalTaskExecutor.class);
+
+ jobTracker = ctx.jobTracker();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop(boolean cancel) {
+ busyLock.writeLock();
+
+ try {
+ comm.stop();
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to gracefully stop external hadoop communication server (will shutdown anyway)", e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onJobStateChanged(final HadoopJobMetadata meta) {
+ final HadoopProcess proc = runningProcsByJobId.get(meta.jobId());
+
+ // If we have a local process for this job.
+ if (proc != null) {
+ if (log.isDebugEnabled())
+ log.debug("Updating job information for remote task process [proc=" + proc + ", meta=" + meta + ']');
+
+ if (meta.phase() == HadoopJobPhase.PHASE_COMPLETE) {
+ if (log.isDebugEnabled())
+ log.debug("Completed job execution, will terminate child process [jobId=" + meta.jobId() +
+ ", proc=" + proc + ']');
+
+ runningProcsByJobId.remove(meta.jobId());
+ runningProcsByProcId.remove(proc.descriptor().processId());
+
+ proc.terminate();
+
+ return;
+ }
+
+ if (proc.initFut.isDone()) {
+ if (!proc.initFut.isFailed())
+ sendJobInfoUpdate(proc, meta);
+ else if (log.isDebugEnabled())
+ log.debug("Failed to initialize child process (will skip job state notification) " +
+ "[jobId=" + meta.jobId() + ", meta=" + meta + ']');
+ }
+ else {
+ proc.initFut.listenAsync(new CI1<IgniteInternalFuture<IgniteBiTuple<Process, HadoopProcessDescriptor>>>() {
+ @Override public void apply(IgniteInternalFuture<IgniteBiTuple<Process, HadoopProcessDescriptor>> f) {
+ try {
+ f.get();
+
+ sendJobInfoUpdate(proc, meta);
+ }
+ catch (IgniteCheckedException e) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to initialize child process (will skip job state notification) " +
+ "[jobId=" + meta.jobId() + ", meta=" + meta + ", err=" + e + ']');
+ }
+
+ }
+ });
+ }
+ }
+ else if (ctx.isParticipating(meta)) {
+ HadoopJob job;
+
+ try {
+ job = jobTracker.job(meta.jobId(), meta.jobInfo());
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to get job: " + meta.jobId(), e);
+
+ return;
+ }
+
+ startProcess(job, meta.mapReducePlan());
+ }
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("ConstantConditions")
+ @Override public void run(final HadoopJob job, final Collection<HadoopTaskInfo> tasks) throws IgniteCheckedException {
+ if (!busyLock.tryReadLock()) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to start hadoop tasks (grid is stopping, will ignore).");
+
+ return;
+ }
+
+ try {
+ HadoopProcess proc = runningProcsByJobId.get(job.id());
+
+ HadoopTaskType taskType = F.first(tasks).type();
+
+ if (taskType == HadoopTaskType.SETUP || taskType == HadoopTaskType.ABORT ||
+ taskType == HadoopTaskType.COMMIT) {
+ if (proc == null || proc.terminated()) {
+ runningProcsByJobId.remove(job.id(), proc);
+
+ // Start new process for ABORT task since previous processes were killed.
+ proc = startProcess(job, jobTracker.plan(job.id()));
+
+ if (log.isDebugEnabled())
+ log.debug("Starting new process for maintenance task [jobId=" + job.id() +
+ ", proc=" + proc + ", taskType=" + taskType + ']');
+ }
+ }
+ else
+ assert proc != null : "Missing started process for task execution request: " + job.id() +
+ ", tasks=" + tasks;
+
+ final HadoopProcess proc0 = proc;
+
+ proc.initFut.listenAsync(new CI1<IgniteInternalFuture<IgniteBiTuple<Process, HadoopProcessDescriptor>>>() {
+ @Override public void apply(
+ IgniteInternalFuture<IgniteBiTuple<Process, HadoopProcessDescriptor>> f) {
+ if (!busyLock.tryReadLock())
+ return;
+
+ try {
+ f.get();
+
+ proc0.addTasks(tasks);
+
+ if (log.isDebugEnabled())
+ log.debug("Sending task execution request to child process [jobId=" + job.id() +
+ ", proc=" + proc0 + ", tasks=" + tasks + ']');
+
+ sendExecutionRequest(proc0, job, tasks);
+ }
+ catch (IgniteCheckedException e) {
+ notifyTasksFailed(tasks, FAILED, e);
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+ });
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancelTasks(HadoopJobId jobId) {
+ HadoopProcess proc = runningProcsByJobId.get(jobId);
+
+ if (proc != null)
+ proc.terminate();
+ }
+
+ /**
+ * Sends execution request to remote node.
+ *
+ * @param proc Process to send request to.
+ * @param job Job instance.
+ * @param tasks Collection of tasks to execute in started process.
+ */
+ private void sendExecutionRequest(HadoopProcess proc, HadoopJob job, Collection<HadoopTaskInfo> tasks)
+ throws IgniteCheckedException {
+ // Must synchronize since concurrent process crash may happen and will receive onConnectionLost().
+ proc.lock();
+
+ try {
+ if (proc.terminated()) {
+ notifyTasksFailed(tasks, CRASHED, null);
+
+ return;
+ }
+
+ HadoopTaskExecutionRequest req = new HadoopTaskExecutionRequest();
+
+ req.jobId(job.id());
+ req.jobInfo(job.info());
+ req.tasks(tasks);
+
+ comm.sendMessage(proc.descriptor(), req);
+ }
+ finally {
+ proc.unlock();
+ }
+ }
+
+ /**
+ * @return External task metadata.
+ */
+ private HadoopExternalTaskMetadata buildTaskMeta() {
+ HadoopExternalTaskMetadata meta = new HadoopExternalTaskMetadata();
+
+ meta.classpath(Arrays.asList(System.getProperty("java.class.path").split(File.pathSeparator)));
+ meta.jvmOptions(Arrays.asList("-Xmx1g", "-ea", "-XX:+UseConcMarkSweepGC", "-XX:+CMSClassUnloadingEnabled",
+ "-DIGNITE_HOME=" + U.getIgniteHome()));
+
+ return meta;
+ }
+
+ /**
+ * @param tasks Tasks to notify about.
+ * @param state Fail state.
+ * @param e Optional error.
+ */
+ private void notifyTasksFailed(Iterable<HadoopTaskInfo> tasks, HadoopTaskState state, Throwable e) {
+ HadoopTaskStatus fail = new HadoopTaskStatus(state, e);
+
+ for (HadoopTaskInfo task : tasks)
+ jobTracker.onTaskFinished(task, fail);
+ }
+
+ /**
+ * Starts process template that will be ready to execute Hadoop tasks.
+ *
+ * @param job Job instance.
+ * @param plan Map reduce plan.
+ */
+ private HadoopProcess startProcess(final HadoopJob job, final HadoopMapReducePlan plan) {
+ final UUID childProcId = UUID.randomUUID();
+
+ HadoopJobId jobId = job.id();
+
+ final HadoopProcessFuture fut = new HadoopProcessFuture(childProcId, jobId, ctx.kernalContext());
+
+ final HadoopProcess proc = new HadoopProcess(jobId, fut, plan.reducers(ctx.localNodeId()));
+
+ HadoopProcess old = runningProcsByJobId.put(jobId, proc);
+
+ assert old == null;
+
+ old = runningProcsByProcId.put(childProcId, proc);
+
+ assert old == null;
+
+ ctx.kernalContext().closure().runLocalSafe(new Runnable() {
+ @Override public void run() {
+ if (!busyLock.tryReadLock()) {
+ fut.onDone(new IgniteCheckedException("Failed to start external process (grid is stopping)."));
+
+ return;
+ }
+
+ try {
+ HadoopExternalTaskMetadata startMeta = buildTaskMeta();
+
+ if (log.isDebugEnabled())
+ log.debug("Created hadoop child process metadata for job [job=" + job +
+ ", childProcId=" + childProcId + ", taskMeta=" + startMeta + ']');
+
+ Process proc = startJavaProcess(childProcId, startMeta, job);
+
+ BufferedReader rdr = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+
+ String line;
+
+ // Read up all the process output.
+ while ((line = rdr.readLine()) != null) {
+ if (log.isDebugEnabled())
+ log.debug("Tracing process output: " + line);
+
+ if ("Started".equals(line)) {
+ // Process started successfully, it should not write anything more to the output stream.
+ if (log.isDebugEnabled())
+ log.debug("Successfully started child process [childProcId=" + childProcId +
+ ", meta=" + job + ']');
+
+ fut.onProcessStarted(proc);
+
+ break;
+ }
+ else if ("Failed".equals(line)) {
+ StringBuilder sb = new StringBuilder("Failed to start child process: " + job + "\n");
+
+ while ((line = rdr.readLine()) != null)
+ sb.append(" ").append(line).append("\n");
+
+ // Cut last character.
+ sb.setLength(sb.length() - 1);
+
+ log.warning(sb.toString());
+
+ fut.onDone(new IgniteCheckedException(sb.toString()));
+
+ break;
+ }
+ }
+ }
+ catch (Throwable e) {
+ fut.onDone(new IgniteCheckedException("Failed to initialize child process: " + job, e));
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+ }, true);
+
+ fut.listenAsync(new CI1<IgniteInternalFuture<IgniteBiTuple<Process, HadoopProcessDescriptor>>>() {
+ @Override public void apply(IgniteInternalFuture<IgniteBiTuple<Process, HadoopProcessDescriptor>> f) {
+ try {
+ // Make sure there were no exceptions.
+ f.get();
+
+ prepareForJob(proc, job, plan);
+ }
+ catch (IgniteCheckedException ignore) {
+ // Exception is printed in future's onDone() method.
+ }
+ }
+ });
+
+ return proc;
+ }
+
+ /**
+ * Checks that java local command is available.
+ *
+ * @throws IgniteCheckedException If initialization failed.
+ */
+ private void initJavaCommand() throws IgniteCheckedException {
+ String javaHome = System.getProperty("java.home");
+
+ if (javaHome == null)
+ javaHome = System.getenv("JAVA_HOME");
+
+ if (javaHome == null)
+ throw new IgniteCheckedException("Failed to locate JAVA_HOME.");
+
+ javaCmd = javaHome + File.separator + "bin" + File.separator + (U.isWindows() ? "java.exe" : "java");
+
+ try {
+ Process proc = new ProcessBuilder(javaCmd, "-version").redirectErrorStream(true).start();
+
+ Collection<String> out = readProcessOutput(proc);
+
+ int res = proc.waitFor();
+
+ if (res != 0)
+ throw new IgniteCheckedException("Failed to execute 'java -version' command (process finished with nonzero " +
+ "code) [exitCode=" + res + ", javaCmd='" + javaCmd + "', msg=" + F.first(out) + ']');
+
+ if (log.isInfoEnabled()) {
+ log.info("Will use java for external task execution: ");
+
+ for (String s : out)
+ log.info(" " + s);
+ }
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to check java for external task execution.", e);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteCheckedException("Failed to wait for process completion (thread got interrupted).", e);
+ }
+ }
+
+ /**
+ * Reads process output line-by-line.
+ *
+ * @param proc Process to read output.
+ * @return Read lines.
+ * @throws IOException If read failed.
+ */
+ private Collection<String> readProcessOutput(Process proc) throws IOException {
+ BufferedReader rdr = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+
+ Collection<String> res = new ArrayList<>();
+
+ String s;
+
+ while ((s = rdr.readLine()) != null)
+ res.add(s);
+
+ return res;
+ }
+
+ /**
+ * Builds process from metadata.
+ *
+ * @param childProcId Child process ID.
+ * @param startMeta Metadata.
+ * @param job Job.
+ * @return Started process.
+ */
+ private Process startJavaProcess(UUID childProcId, HadoopExternalTaskMetadata startMeta,
+ HadoopJob job) throws Exception {
+ String outFldr = jobWorkFolder(job.id()) + File.separator + childProcId;
+
+ if (log.isDebugEnabled())
+ log.debug("Will write process log output to: " + outFldr);
+
+ List<String> cmd = new ArrayList<>();
+
+ File workDir = U.resolveWorkDirectory("", false);
+
+ cmd.add(javaCmd);
+ cmd.addAll(startMeta.jvmOptions());
+ cmd.add("-cp");
+ cmd.add(buildClasspath(startMeta.classpath()));
+ cmd.add(HadoopExternalProcessStarter.class.getName());
+ cmd.add("-cpid");
+ cmd.add(String.valueOf(childProcId));
+ cmd.add("-ppid");
+ cmd.add(String.valueOf(nodeDesc.processId()));
+ cmd.add("-nid");
+ cmd.add(String.valueOf(nodeDesc.parentNodeId()));
+ cmd.add("-addr");
+ cmd.add(nodeDesc.address());
+ cmd.add("-tport");
+ cmd.add(String.valueOf(nodeDesc.tcpPort()));
+ cmd.add("-sport");
+ cmd.add(String.valueOf(nodeDesc.sharedMemoryPort()));
+ cmd.add("-out");
+ cmd.add(outFldr);
+ cmd.add("-wd");
+ cmd.add(workDir.getAbsolutePath());
+
+ return new ProcessBuilder(cmd)
+ .redirectErrorStream(true)
+ .directory(workDir)
+ .start();
+ }
+
+ /**
+ * Gets job work folder.
+ *
+ * @param jobId Job ID.
+ * @return Job work folder.
+ */
+ private String jobWorkFolder(HadoopJobId jobId) {
+ return outputBase + File.separator + "Job_" + jobId;
+ }
+
+ /**
+ * @param cp Classpath collection.
+ * @return Classpath string.
+ */
+ private String buildClasspath(Collection<String> cp) {
+ assert !cp.isEmpty();
+
+ StringBuilder sb = new StringBuilder();
+
+ for (String s : cp)
+ sb.append(s).append(pathSep);
+
+ sb.setLength(sb.length() - 1);
+
+ return sb.toString();
+ }
+
+ /**
+ * Sends job info update request to remote process.
+ *
+ * @param proc Process to send request to.
+ * @param meta Job metadata.
+ */
+ private void sendJobInfoUpdate(HadoopProcess proc, HadoopJobMetadata meta) {
+ Map<Integer, HadoopProcessDescriptor> rdcAddrs = meta.reducersAddresses();
+
+ int rdcNum = meta.mapReducePlan().reducers();
+
+ HadoopProcessDescriptor[] addrs = null;
+
+ if (rdcAddrs != null && rdcAddrs.size() == rdcNum) {
+ addrs = new HadoopProcessDescriptor[rdcNum];
+
+ for (int i = 0; i < rdcNum; i++) {
+ HadoopProcessDescriptor desc = rdcAddrs.get(i);
+
+ assert desc != null : "Missing reducing address [meta=" + meta + ", rdc=" + i + ']';
+
+ addrs[i] = desc;
+ }
+ }
+
+ try {
+ comm.sendMessage(proc.descriptor(), new HadoopJobInfoUpdateRequest(proc.jobId, meta.phase(), addrs));
+ }
+ catch (IgniteCheckedException e) {
+ if (!proc.terminated()) {
+ log.error("Failed to send job state update message to remote child process (will kill the process) " +
+ "[jobId=" + proc.jobId + ", meta=" + meta + ']', e);
+
+ proc.terminate();
+ }
+ }
+ }
+
+ /**
+ * Sends prepare request to remote process.
+ *
+ * @param proc Process to send request to.
+ * @param job Job.
+ * @param plan Map reduce plan.
+ */
+ private void prepareForJob(HadoopProcess proc, HadoopJob job, HadoopMapReducePlan plan) {
+ try {
+ comm.sendMessage(proc.descriptor(), new HadoopPrepareForJobRequest(job.id(), job.info(),
+ plan.reducers(), plan.reducers(ctx.localNodeId())));
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to send job prepare request to remote process [proc=" + proc + ", job=" + job +
+ ", plan=" + plan + ']', e);
+
+ proc.terminate();
+ }
+ }
+
+ /**
+ * Processes task finished message.
+ *
+ * @param desc Remote process descriptor.
+ * @param taskMsg Task finished message.
+ */
+ private void processTaskFinishedMessage(HadoopProcessDescriptor desc, HadoopTaskFinishedMessage taskMsg) {
+ HadoopProcess proc = runningProcsByProcId.get(desc.processId());
+
+ if (proc != null)
+ proc.removeTask(taskMsg.taskInfo());
+
+ jobTracker.onTaskFinished(taskMsg.taskInfo(), taskMsg.status());
+ }
+
+ /**
+ *
+ */
+ private class MessageListener implements HadoopMessageListener {
+ /** {@inheritDoc} */
+ @Override public void onMessageReceived(HadoopProcessDescriptor desc, HadoopMessage msg) {
+ if (!busyLock.tryReadLock())
+ return;
+
+ try {
+ if (msg instanceof HadoopProcessStartedAck) {
+ HadoopProcess proc = runningProcsByProcId.get(desc.processId());
+
+ assert proc != null : "Missing child process for processId: " + desc;
+
+ HadoopProcessFuture fut = proc.initFut;
+
+ if (fut != null)
+ fut.onReplyReceived(desc);
+ // Safety.
+ else
+ log.warning("Failed to find process start future (will ignore): " + desc);
+ }
+ else if (msg instanceof HadoopTaskFinishedMessage) {
+ HadoopTaskFinishedMessage taskMsg = (HadoopTaskFinishedMessage)msg;
+
+ processTaskFinishedMessage(desc, taskMsg);
+ }
+ else
+ log.warning("Unexpected message received by node [desc=" + desc + ", msg=" + msg + ']');
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onConnectionLost(HadoopProcessDescriptor desc) {
+ if (!busyLock.tryReadLock())
+ return;
+
+ try {
+ if (desc == null) {
+ U.warn(log, "Handshake failed.");
+
+ return;
+ }
+
+ // Notify job tracker about failed tasks.
+ HadoopProcess proc = runningProcsByProcId.get(desc.processId());
+
+ if (proc != null) {
+ Collection<HadoopTaskInfo> tasks = proc.tasks();
+
+ if (!F.isEmpty(tasks)) {
+ log.warning("Lost connection with alive process (will terminate): " + desc);
+
+ HadoopTaskStatus status = new HadoopTaskStatus(CRASHED,
+ new IgniteCheckedException("Failed to run tasks (external process finished unexpectedly): " + desc));
+
+ for (HadoopTaskInfo info : tasks)
+ jobTracker.onTaskFinished(info, status);
+
+ runningProcsByJobId.remove(proc.jobId(), proc);
+ }
+
+ // Safety.
+ proc.terminate();
+ }
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+ }
+
+ /**
+ * Hadoop process.
+ */
+ private static class HadoopProcess extends ReentrantLock {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Job ID. */
+ private final HadoopJobId jobId;
+
+ /** Process. */
+ private Process proc;
+
+ /** Init future. Completes when process is ready to receive messages. */
+ private final HadoopProcessFuture initFut;
+
+ /** Process descriptor. */
+ private HadoopProcessDescriptor procDesc;
+
+ /** Reducers planned for this process. */
+ private Collection<Integer> reducers;
+
+ /** Tasks. */
+ private final Collection<HadoopTaskInfo> tasks = new ConcurrentLinkedDeque8<>();
+
+ /** Terminated flag. */
+ private volatile boolean terminated;
+
+ /**
+ * @param jobId Job ID.
+ * @param initFut Init future.
+ */
+ private HadoopProcess(HadoopJobId jobId, HadoopProcessFuture initFut,
+ int[] reducers) {
+ this.jobId = jobId;
+ this.initFut = initFut;
+
+ if (!F.isEmpty(reducers)) {
+ this.reducers = new ArrayList<>(reducers.length);
+
+ for (int r : reducers)
+ this.reducers.add(r);
+ }
+ }
+
+ /**
+ * @return Communication process descriptor.
+ */
+ private HadoopProcessDescriptor descriptor() {
+ return procDesc;
+ }
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /**
+ * Initialized callback.
+ *
+ * @param proc Java process representation.
+ * @param procDesc Process descriptor.
+ */
+ private void onInitialized(Process proc, HadoopProcessDescriptor procDesc) {
+ this.proc = proc;
+ this.procDesc = procDesc;
+ }
+
+ /**
+ * Terminates process (kills it).
+ */
+ private void terminate() {
+ // Guard against concurrent message sending.
+ lock();
+
+ try {
+ terminated = true;
+
+ if (!initFut.isDone())
+ initFut.listenAsync(new CI1<IgniteInternalFuture<IgniteBiTuple<Process, HadoopProcessDescriptor>>>() {
+ @Override public void apply(
+ IgniteInternalFuture<IgniteBiTuple<Process, HadoopProcessDescriptor>> f) {
+ proc.destroy();
+ }
+ });
+ else
+ proc.destroy();
+ }
+ finally {
+ unlock();
+ }
+ }
+
+ /**
+ * @return Terminated flag.
+ */
+ private boolean terminated() {
+ return terminated;
+ }
+
+ /**
+ * Sets process tasks.
+ *
+ * @param tasks Tasks to set.
+ */
+ private void addTasks(Collection<HadoopTaskInfo> tasks) {
+ this.tasks.addAll(tasks);
+ }
+
+ /**
+ * Removes task when it was completed.
+ *
+ * @param task Task to remove.
+ */
+ private void removeTask(HadoopTaskInfo task) {
+ if (tasks != null)
+ tasks.remove(task);
+ }
+
+ /**
+ * @return Collection of tasks.
+ */
+ private Collection<HadoopTaskInfo> tasks() {
+ return tasks;
+ }
+
+ /**
+ * @return Planned reducers.
+ */
+ private Collection<Integer> reducers() {
+ return reducers;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopProcess.class, this);
+ }
+ }
+
+ /**
+ *
+ */
+ private class HadoopProcessFuture extends GridFutureAdapter<IgniteBiTuple<Process, HadoopProcessDescriptor>> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Child process ID. */
+ private UUID childProcId;
+
+ /** Job ID. */
+ private HadoopJobId jobId;
+
+ /** Process descriptor. */
+ private HadoopProcessDescriptor desc;
+
+ /** Running process. */
+ private Process proc;
+
+ /** Process started flag. */
+ private volatile boolean procStarted;
+
+ /** Reply received flag. */
+ private volatile boolean replyReceived;
+
+ /** Logger. */
+ private final IgniteLogger log = HadoopExternalTaskExecutor.this.log;
+
+ /**
+ * Empty constructor.
+ */
+ public HadoopProcessFuture() {
+ // No-op.
+ }
+
+ /**
+ * @param ctx Kernal context.
+ */
+ private HadoopProcessFuture(UUID childProcId, HadoopJobId jobId, GridKernalContext ctx) {
+ super(ctx);
+
+ this.childProcId = childProcId;
+ this.jobId = jobId;
+ }
+
+ /**
+ * Process started callback.
+ */
+ public void onProcessStarted(Process proc) {
+ this.proc = proc;
+
+ procStarted = true;
+
+ if (procStarted && replyReceived)
+ onDone(F.t(proc, desc));
+ }
+
+ /**
+ * Reply received callback.
+ */
+ public void onReplyReceived(HadoopProcessDescriptor desc) {
+ assert childProcId.equals(desc.processId());
+
+ this.desc = desc;
+
+ replyReceived = true;
+
+ if (procStarted && replyReceived)
+ onDone(F.t(proc, desc));
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean onDone(@Nullable IgniteBiTuple<Process, HadoopProcessDescriptor> res,
+ @Nullable Throwable err) {
+ if (err == null) {
+ HadoopProcess proc = runningProcsByProcId.get(childProcId);
+
+ assert proc != null;
+
+ assert proc.initFut == this;
+
+ proc.onInitialized(res.get1(), res.get2());
+
+ if (!F.isEmpty(proc.reducers()))
+ jobTracker.onExternalMappersInitialized(jobId, proc.reducers(), desc);
+ }
+ else {
+ // Clean up since init failed.
+ runningProcsByJobId.remove(jobId);
+ runningProcsByProcId.remove(childProcId);
+ }
+
+ if (super.onDone(res, err)) {
+ if (err == null) {
+ if (log.isDebugEnabled())
+ log.debug("Initialized child process for external task execution [jobId=" + jobId +
+ ", desc=" + desc + ", initTime=" + duration() + ']');
+ }
+ else
+ U.error(log, "Failed to initialize child process for external task execution [jobId=" + jobId +
+ ", desc=" + desc + ']', err);
+
+ return true;
+ }
+
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskMetadata.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskMetadata.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskMetadata.java
new file mode 100644
index 0000000..f0acc9f
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskMetadata.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
+
+import org.apache.ignite.internal.util.tostring.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.util.*;
+
+/**
+ * External task metadata (classpath, JVM options) needed to start external process execution.
+ */
+public class HadoopExternalTaskMetadata {
+ /** Process classpath. */
+ private Collection<String> classpath;
+
+ /** JVM options. */
+ @GridToStringInclude
+ private Collection<String> jvmOpts;
+
+ /**
+ * @return JVM Options.
+ */
+ public Collection<String> jvmOptions() {
+ return jvmOpts;
+ }
+
+ /**
+ * @param jvmOpts JVM options.
+ */
+ public void jvmOptions(Collection<String> jvmOpts) {
+ this.jvmOpts = jvmOpts;
+ }
+
+ /**
+ * @return Classpath.
+ */
+ public Collection<String> classpath() {
+ return classpath;
+ }
+
+ /**
+ * @param classpath Classpath.
+ */
+ public void classpath(Collection<String> classpath) {
+ this.classpath = classpath;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopExternalTaskMetadata.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopJobInfoUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopJobInfoUpdateRequest.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopJobInfoUpdateRequest.java
new file mode 100644
index 0000000..25c9408
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopJobInfoUpdateRequest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
+
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.util.tostring.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+
+/**
+ * Job info update request.
+ */
+public class HadoopJobInfoUpdateRequest implements HadoopMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Job ID. */
+ @GridToStringInclude
+ private HadoopJobId jobId;
+
+ /** Job phase. */
+ @GridToStringInclude
+ private HadoopJobPhase jobPhase;
+
+ /** Reducers addresses. */
+ @GridToStringInclude
+ private HadoopProcessDescriptor[] reducersAddrs;
+
+ /**
+ * Constructor required by {@link Externalizable}.
+ */
+ public HadoopJobInfoUpdateRequest() {
+ // No-op.
+ }
+
+ /**
+ * @param jobId Job ID.
+ * @param jobPhase Job phase.
+ * @param reducersAddrs Reducers addresses.
+ */
+ public HadoopJobInfoUpdateRequest(HadoopJobId jobId, HadoopJobPhase jobPhase,
+ HadoopProcessDescriptor[] reducersAddrs) {
+ assert jobId != null;
+
+ this.jobId = jobId;
+ this.jobPhase = jobPhase;
+ this.reducersAddrs = reducersAddrs;
+ }
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /**
+ * @return Job phase.
+ */
+ public HadoopJobPhase jobPhase() {
+ return jobPhase;
+ }
+
+ /**
+ * @return Reducers addresses.
+ */
+ public HadoopProcessDescriptor[] reducersAddresses() {
+ return reducersAddrs;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ jobId.writeExternal(out);
+
+ out.writeObject(jobPhase);
+ U.writeArray(out, reducersAddrs);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobId = new HadoopJobId();
+ jobId.readExternal(in);
+
+ jobPhase = (HadoopJobPhase)in.readObject();
+ reducersAddrs = (HadoopProcessDescriptor[])U.readArray(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopJobInfoUpdateRequest.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopPrepareForJobRequest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopPrepareForJobRequest.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopPrepareForJobRequest.java
new file mode 100644
index 0000000..df44dd7
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopPrepareForJobRequest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
+
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.util.tostring.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+
+/**
+ * Child process initialization request.
+ */
+public class HadoopPrepareForJobRequest implements HadoopMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Job ID. */
+ @GridToStringInclude
+ private HadoopJobId jobId;
+
+ /** Job info. */
+ @GridToStringInclude
+ private HadoopJobInfo jobInfo;
+
+ /** Total amount of reducers in the job. */
+ @GridToStringInclude
+ private int totalReducersCnt;
+
+ /** Reducers to be executed on current node. */
+ @GridToStringInclude
+ private int[] locReducers;
+
+ /**
+ * Constructor required by {@link Externalizable}.
+ */
+ public HadoopPrepareForJobRequest() {
+ // No-op.
+ }
+
+ /**
+ * @param jobId Job ID.
+ * @param jobInfo Job info.
+ * @param totalReducersCnt Number of reducers in the job.
+ * @param locReducers Reducers to be executed on current node.
+ */
+ public HadoopPrepareForJobRequest(HadoopJobId jobId, HadoopJobInfo jobInfo, int totalReducersCnt,
+ int[] locReducers) {
+ assert jobId != null;
+
+ this.jobId = jobId;
+ this.jobInfo = jobInfo;
+ this.totalReducersCnt = totalReducersCnt;
+ this.locReducers = locReducers;
+ }
+
+ /**
+ * @return Job info.
+ */
+ public HadoopJobInfo jobInfo() {
+ return jobInfo;
+ }
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /**
+ * @return Reducers to be executed on current node.
+ */
+ public int[] localReducers() {
+ return locReducers;
+ }
+
+ /**
+ * @return Number of reducers in job.
+ */
+ public int totalReducerCount() {
+ return totalReducersCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ jobId.writeExternal(out);
+
+ out.writeObject(jobInfo);
+ out.writeInt(totalReducersCnt);
+
+ U.writeIntArray(out, locReducers);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobId = new HadoopJobId();
+ jobId.readExternal(in);
+
+ jobInfo = (HadoopJobInfo)in.readObject();
+ totalReducersCnt = in.readInt();
+
+ locReducers = U.readIntArray(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopPrepareForJobRequest.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopProcessDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopProcessDescriptor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopProcessDescriptor.java
new file mode 100644
index 0000000..dea73c3
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopProcessDescriptor.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
+
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Process descriptor used to identify process for which task is running.
+ */
+public class HadoopProcessDescriptor implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Parent node ID. */
+ private UUID parentNodeId;
+
+ /** Process ID. */
+ private UUID procId;
+
+ /** Address. */
+ private String addr;
+
+ /** TCP port. */
+ private int tcpPort;
+
+ /** Shared memory port. */
+ private int shmemPort;
+
+ /**
+ * @param parentNodeId Parent node ID.
+ * @param procId Process ID.
+ */
+ public HadoopProcessDescriptor(UUID parentNodeId, UUID procId) {
+ this.parentNodeId = parentNodeId;
+ this.procId = procId;
+ }
+
+ /**
+ * Gets process ID.
+ *
+ * @return Process ID.
+ */
+ public UUID processId() {
+ return procId;
+ }
+
+ /**
+ * Gets parent node ID.
+ *
+ * @return Parent node ID.
+ */
+ public UUID parentNodeId() {
+ return parentNodeId;
+ }
+
+ /**
+ * Gets host address.
+ *
+ * @return Host address.
+ */
+ public String address() {
+ return addr;
+ }
+
+ /**
+ * Sets host address.
+ *
+ * @param addr Host address.
+ */
+ public void address(String addr) {
+ this.addr = addr;
+ }
+
+ /**
+ * @return Shared memory port.
+ */
+ public int sharedMemoryPort() {
+ return shmemPort;
+ }
+
+ /**
+ * Sets shared memory port.
+ *
+ * @param shmemPort Shared memory port.
+ */
+ public void sharedMemoryPort(int shmemPort) {
+ this.shmemPort = shmemPort;
+ }
+
+ /**
+ * @return TCP port.
+ */
+ public int tcpPort() {
+ return tcpPort;
+ }
+
+ /**
+ * Sets TCP port.
+ *
+ * @param tcpPort TCP port.
+ */
+ public void tcpPort(int tcpPort) {
+ this.tcpPort = tcpPort;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof HadoopProcessDescriptor))
+ return false;
+
+ HadoopProcessDescriptor that = (HadoopProcessDescriptor)o;
+
+ return parentNodeId.equals(that.parentNodeId) && procId.equals(that.procId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int result = parentNodeId.hashCode();
+
+ result = 31 * result + procId.hashCode();
+
+ return result;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopProcessDescriptor.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopProcessStartedAck.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopProcessStartedAck.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopProcessStartedAck.java
new file mode 100644
index 0000000..49ff4bf
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopProcessStartedAck.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
+
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+
+/**
+ * Process started message.
+ */
+public class HadoopProcessStartedAck implements HadoopMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopProcessStartedAck.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopTaskExecutionRequest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopTaskExecutionRequest.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopTaskExecutionRequest.java
new file mode 100644
index 0000000..05e12ef
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopTaskExecutionRequest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
+
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.util.tostring.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Message sent from node to child process to start task(s) execution.
+ */
+public class HadoopTaskExecutionRequest implements HadoopMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Job ID. */
+ @GridToStringInclude
+ private HadoopJobId jobId;
+
+ /** Job info. */
+ @GridToStringInclude
+ private HadoopJobInfo jobInfo;
+
+ /** Mappers. */
+ @GridToStringInclude
+ private Collection<HadoopTaskInfo> tasks;
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /**
+ * @param jobId Job ID.
+ */
+ public void jobId(HadoopJobId jobId) {
+ this.jobId = jobId;
+ }
+
+ /**
+ * @return Jon info.
+ */
+ public HadoopJobInfo jobInfo() {
+ return jobInfo;
+ }
+
+ /**
+ * @param jobInfo Job info.
+ */
+ public void jobInfo(HadoopJobInfo jobInfo) {
+ this.jobInfo = jobInfo;
+ }
+
+ /**
+ * @return Tasks.
+ */
+ public Collection<HadoopTaskInfo> tasks() {
+ return tasks;
+ }
+
+ /**
+ * @param tasks Tasks.
+ */
+ public void tasks(Collection<HadoopTaskInfo> tasks) {
+ this.tasks = tasks;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopTaskExecutionRequest.class, this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ jobId.writeExternal(out);
+
+ out.writeObject(jobInfo);
+ U.writeCollection(out, tasks);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobId = new HadoopJobId();
+ jobId.readExternal(in);
+
+ jobInfo = (HadoopJobInfo)in.readObject();
+ tasks = U.readCollection(in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopTaskFinishedMessage.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopTaskFinishedMessage.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopTaskFinishedMessage.java
new file mode 100644
index 0000000..d3639c7
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopTaskFinishedMessage.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
+
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+
+/**
+ * Task finished message. Sent when local task finishes execution.
+ */
+public class HadoopTaskFinishedMessage implements HadoopMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Finished task info. */
+ private HadoopTaskInfo taskInfo;
+
+ /** Task finish status. */
+ private HadoopTaskStatus status;
+
+ /**
+ * Constructor required by {@link Externalizable}.
+ */
+ public HadoopTaskFinishedMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param taskInfo Finished task info.
+ * @param status Task finish status.
+ */
+ public HadoopTaskFinishedMessage(HadoopTaskInfo taskInfo, HadoopTaskStatus status) {
+ assert taskInfo != null;
+ assert status != null;
+
+ this.taskInfo = taskInfo;
+ this.status = status;
+ }
+
+ /**
+ * @return Finished task info.
+ */
+ public HadoopTaskInfo taskInfo() {
+ return taskInfo;
+ }
+
+ /**
+ * @return Task finish status.
+ */
+ public HadoopTaskStatus status() {
+ return status;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopTaskFinishedMessage.class, this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ taskInfo.writeExternal(out);
+ status.writeExternal(out);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ taskInfo = new HadoopTaskInfo();
+ taskInfo.readExternal(in);
+
+ status = new HadoopTaskStatus();
+ status.readExternal(in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/GridHadoopChildProcessRunner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/GridHadoopChildProcessRunner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/GridHadoopChildProcessRunner.java
deleted file mode 100644
index 2d00222..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/GridHadoopChildProcessRunner.java
+++ /dev/null
@@ -1,440 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.child;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.processors.hadoop.shuffle.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.*;
-import org.apache.ignite.internal.util.future.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopTaskType.*;
-
-/**
- * Hadoop process base.
- */
-@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
-public class GridHadoopChildProcessRunner {
- /** Node process descriptor. */
- private GridHadoopProcessDescriptor nodeDesc;
-
- /** Message processing executor service. */
- private ExecutorService msgExecSvc;
-
- /** Task executor service. */
- private GridHadoopExecutorService execSvc;
-
- /** */
- protected GridUnsafeMemory mem = new GridUnsafeMemory(0);
-
- /** External communication. */
- private GridHadoopExternalCommunication comm;
-
- /** Logger. */
- private IgniteLogger log;
-
- /** Init guard. */
- private final AtomicBoolean initGuard = new AtomicBoolean();
-
- /** Start time. */
- private long startTime;
-
- /** Init future. */
- private final GridFutureAdapterEx<?> initFut = new GridFutureAdapterEx<>();
-
- /** Job instance. */
- private GridHadoopJob job;
-
- /** Number of uncompleted tasks. */
- private final AtomicInteger pendingTasks = new AtomicInteger();
-
- /** Shuffle job. */
- private GridHadoopShuffleJob<GridHadoopProcessDescriptor> shuffleJob;
-
- /** Concurrent mappers. */
- private int concMappers;
-
- /** Concurrent reducers. */
- private int concReducers;
-
- /**
- * Starts child process runner.
- */
- public void start(GridHadoopExternalCommunication comm, GridHadoopProcessDescriptor nodeDesc,
- ExecutorService msgExecSvc, IgniteLogger parentLog)
- throws IgniteCheckedException {
- this.comm = comm;
- this.nodeDesc = nodeDesc;
- this.msgExecSvc = msgExecSvc;
-
- comm.setListener(new MessageListener());
- log = parentLog.getLogger(GridHadoopChildProcessRunner.class);
-
- startTime = U.currentTimeMillis();
-
- // At this point node knows that this process has started.
- comm.sendMessage(this.nodeDesc, new GridHadoopProcessStartedAck());
- }
-
- /**
- * Initializes process for task execution.
- *
- * @param req Initialization request.
- */
- private void prepareProcess(GridHadoopPrepareForJobRequest req) {
- if (initGuard.compareAndSet(false, true)) {
- try {
- if (log.isDebugEnabled())
- log.debug("Initializing external hadoop task: " + req);
-
- assert job == null;
-
- job = req.jobInfo().createJob(req.jobId(), log);
-
- job.initialize(true, nodeDesc.processId());
-
- shuffleJob = new GridHadoopShuffleJob<>(comm.localProcessDescriptor(), log, job, mem,
- req.totalReducerCount(), req.localReducers());
-
- initializeExecutors(req);
-
- if (log.isDebugEnabled())
- log.debug("External process initialized [initWaitTime=" +
- (U.currentTimeMillis() - startTime) + ']');
-
- initFut.onDone(null, null);
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to initialize process: " + req, e);
-
- initFut.onDone(e);
- }
- }
- else
- log.warning("Duplicate initialize process request received (will ignore): " + req);
- }
-
- /**
- * @param req Task execution request.
- */
- private void runTasks(final GridHadoopTaskExecutionRequest req) {
- if (!initFut.isDone() && log.isDebugEnabled())
- log.debug("Will wait for process initialization future completion: " + req);
-
- initFut.listenAsync(new CI1<IgniteInternalFuture<?>>() {
- @Override public void apply(IgniteInternalFuture<?> f) {
- try {
- // Make sure init was successful.
- f.get();
-
- boolean set = pendingTasks.compareAndSet(0, req.tasks().size());
-
- assert set;
-
- GridHadoopTaskInfo info = F.first(req.tasks());
-
- assert info != null;
-
- int size = info.type() == MAP ? concMappers : concReducers;
-
-// execSvc.setCorePoolSize(size);
-// execSvc.setMaximumPoolSize(size);
-
- if (log.isDebugEnabled())
- log.debug("Set executor service size for task type [type=" + info.type() +
- ", size=" + size + ']');
-
- for (GridHadoopTaskInfo taskInfo : req.tasks()) {
- if (log.isDebugEnabled())
- log.debug("Submitted task for external execution: " + taskInfo);
-
- execSvc.submit(new GridHadoopRunnableTask(log, job, mem, taskInfo, nodeDesc.parentNodeId()) {
- @Override protected void onTaskFinished(GridHadoopTaskStatus status) {
- onTaskFinished0(this, status);
- }
-
- @Override protected GridHadoopTaskInput createInput(GridHadoopTaskContext ctx)
- throws IgniteCheckedException {
- return shuffleJob.input(ctx);
- }
-
- @Override protected GridHadoopTaskOutput createOutput(GridHadoopTaskContext ctx)
- throws IgniteCheckedException {
- return shuffleJob.output(ctx);
- }
- });
- }
- }
- catch (IgniteCheckedException e) {
- for (GridHadoopTaskInfo info : req.tasks())
- notifyTaskFinished(info, new GridHadoopTaskStatus(GridHadoopTaskState.FAILED, e), false);
- }
- }
- });
- }
-
- /**
- * Creates executor services.
- *
- * @param req Init child process request.
- */
- private void initializeExecutors(GridHadoopPrepareForJobRequest req) {
- int cpus = Runtime.getRuntime().availableProcessors();
-//
-// concMappers = get(req.jobInfo(), EXTERNAL_CONCURRENT_MAPPERS, cpus);
-// concReducers = get(req.jobInfo(), EXTERNAL_CONCURRENT_REDUCERS, cpus);
-
- execSvc = new GridHadoopExecutorService(log, "", cpus * 2, 1024);
- }
-
- /**
- * Updates external process map so that shuffle can proceed with sending messages to reducers.
- *
- * @param req Update request.
- */
- private void updateTasks(final GridHadoopJobInfoUpdateRequest req) {
- initFut.listenAsync(new CI1<IgniteInternalFuture<?>>() {
- @Override public void apply(IgniteInternalFuture<?> gridFut) {
- assert initGuard.get();
-
- assert req.jobId().equals(job.id());
-
- if (req.reducersAddresses() != null) {
- if (shuffleJob.initializeReduceAddresses(req.reducersAddresses())) {
- shuffleJob.startSending("external",
- new IgniteInClosure2X<GridHadoopProcessDescriptor, GridHadoopShuffleMessage>() {
- @Override public void applyx(GridHadoopProcessDescriptor dest,
- GridHadoopShuffleMessage msg) throws IgniteCheckedException {
- comm.sendMessage(dest, msg);
- }
- });
- }
- }
- }
- });
- }
-
- /**
- * Stops all executors and running tasks.
- */
- private void shutdown() {
- if (execSvc != null)
- execSvc.shutdown(5000);
-
- if (msgExecSvc != null)
- msgExecSvc.shutdownNow();
-
- try {
- job.dispose(true);
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to dispose job.", e);
- }
- }
-
- /**
- * Notifies node about task finish.
- *
- * @param run Finished task runnable.
- * @param status Task status.
- */
- private void onTaskFinished0(GridHadoopRunnableTask run, GridHadoopTaskStatus status) {
- GridHadoopTaskInfo info = run.taskInfo();
-
- int pendingTasks0 = pendingTasks.decrementAndGet();
-
- if (log.isDebugEnabled())
- log.debug("Hadoop task execution finished [info=" + info
- + ", state=" + status.state() + ", waitTime=" + run.waitTime() + ", execTime=" + run.executionTime() +
- ", pendingTasks=" + pendingTasks0 +
- ", err=" + status.failCause() + ']');
-
- assert info.type() == MAP || info.type() == REDUCE : "Only MAP or REDUCE tasks are supported.";
-
- boolean flush = pendingTasks0 == 0 && info.type() == MAP;
-
- notifyTaskFinished(info, status, flush);
- }
-
- /**
- * @param taskInfo Finished task info.
- * @param status Task status.
- */
- private void notifyTaskFinished(final GridHadoopTaskInfo taskInfo, final GridHadoopTaskStatus status,
- boolean flush) {
-
- final GridHadoopTaskState state = status.state();
- final Throwable err = status.failCause();
-
- if (!flush) {
- try {
- if (log.isDebugEnabled())
- log.debug("Sending notification to parent node [taskInfo=" + taskInfo + ", state=" + state +
- ", err=" + err + ']');
-
- comm.sendMessage(nodeDesc, new GridHadoopTaskFinishedMessage(taskInfo, status));
- }
- catch (IgniteCheckedException e) {
- log.error("Failed to send message to parent node (will terminate child process).", e);
-
- shutdown();
-
- terminate();
- }
- }
- else {
- if (log.isDebugEnabled())
- log.debug("Flushing shuffle messages before sending last task completion notification [taskInfo=" +
- taskInfo + ", state=" + state + ", err=" + err + ']');
-
- final long start = U.currentTimeMillis();
-
- try {
- shuffleJob.flush().listenAsync(new CI1<IgniteInternalFuture<?>>() {
- @Override public void apply(IgniteInternalFuture<?> f) {
- long end = U.currentTimeMillis();
-
- if (log.isDebugEnabled())
- log.debug("Finished flushing shuffle messages [taskInfo=" + taskInfo +
- ", flushTime=" + (end - start) + ']');
-
- try {
- // Check for errors on shuffle.
- f.get();
-
- notifyTaskFinished(taskInfo, status, false);
- }
- catch (IgniteCheckedException e) {
- log.error("Failed to flush shuffle messages (will fail the task) [taskInfo=" + taskInfo +
- ", state=" + state + ", err=" + err + ']', e);
-
- notifyTaskFinished(taskInfo,
- new GridHadoopTaskStatus(GridHadoopTaskState.FAILED, e), false);
- }
- }
- });
- }
- catch (IgniteCheckedException e) {
- log.error("Failed to flush shuffle messages (will fail the task) [taskInfo=" + taskInfo +
- ", state=" + state + ", err=" + err + ']', e);
-
- notifyTaskFinished(taskInfo, new GridHadoopTaskStatus(GridHadoopTaskState.FAILED, e), false);
- }
- }
- }
-
- /**
- * Checks if message was received from parent node and prints warning if not.
- *
- * @param desc Sender process ID.
- * @param msg Received message.
- * @return {@code True} if received from parent node.
- */
- private boolean validateNodeMessage(GridHadoopProcessDescriptor desc, GridHadoopMessage msg) {
- if (!nodeDesc.processId().equals(desc.processId())) {
- log.warning("Received process control request from unknown process (will ignore) [desc=" + desc +
- ", msg=" + msg + ']');
-
- return false;
- }
-
- return true;
- }
-
- /**
- * Stops execution of this process.
- */
- private void terminate() {
- System.exit(1);
- }
-
- /**
- * Message listener.
- */
- private class MessageListener implements GridHadoopMessageListener {
- /** {@inheritDoc} */
- @Override public void onMessageReceived(final GridHadoopProcessDescriptor desc, final GridHadoopMessage msg) {
- if (msg instanceof GridHadoopTaskExecutionRequest) {
- if (validateNodeMessage(desc, msg))
- runTasks((GridHadoopTaskExecutionRequest)msg);
- }
- else if (msg instanceof GridHadoopJobInfoUpdateRequest) {
- if (validateNodeMessage(desc, msg))
- updateTasks((GridHadoopJobInfoUpdateRequest)msg);
- }
- else if (msg instanceof GridHadoopPrepareForJobRequest) {
- if (validateNodeMessage(desc, msg))
- prepareProcess((GridHadoopPrepareForJobRequest)msg);
- }
- else if (msg instanceof GridHadoopShuffleMessage) {
- if (log.isTraceEnabled())
- log.trace("Received shuffle message [desc=" + desc + ", msg=" + msg + ']');
-
- initFut.listenAsync(new CI1<IgniteInternalFuture<?>>() {
- @Override public void apply(IgniteInternalFuture<?> f) {
- try {
- GridHadoopShuffleMessage m = (GridHadoopShuffleMessage)msg;
-
- shuffleJob.onShuffleMessage(m);
-
- comm.sendMessage(desc, new GridHadoopShuffleAck(m.id(), m.jobId()));
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to process hadoop shuffle message [desc=" + desc + ", msg=" + msg + ']', e);
- }
- }
- });
- }
- else if (msg instanceof GridHadoopShuffleAck) {
- if (log.isTraceEnabled())
- log.trace("Received shuffle ack [desc=" + desc + ", msg=" + msg + ']');
-
- shuffleJob.onShuffleAck((GridHadoopShuffleAck)msg);
- }
- else
- log.warning("Unknown message received (will ignore) [desc=" + desc + ", msg=" + msg + ']');
- }
-
- /** {@inheritDoc} */
- @Override public void onConnectionLost(GridHadoopProcessDescriptor desc) {
- if (log.isDebugEnabled())
- log.debug("Lost connection with remote process: " + desc);
-
- if (desc == null)
- U.warn(log, "Handshake failed.");
- else if (desc.processId().equals(nodeDesc.processId())) {
- log.warning("Child process lost connection with parent node (will terminate child process).");
-
- shutdown();
-
- terminate();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/GridHadoopExternalProcessStarter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/GridHadoopExternalProcessStarter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/GridHadoopExternalProcessStarter.java
deleted file mode 100644
index 5aeeeee..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/GridHadoopExternalProcessStarter.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external.child;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.lang.*;
-import org.apache.ignite.logger.log4j.*;
-import org.apache.ignite.marshaller.optimized.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-/**
- * Hadoop external process base class.
- */
-public class GridHadoopExternalProcessStarter {
- /** Path to Log4j configuration file. */
- public static final String DFLT_LOG4J_CONFIG = "config/ignite-log4j.xml";
-
- /** Arguments. */
- private Args args;
-
- /** System out. */
- private OutputStream out;
-
- /** System err. */
- private OutputStream err;
-
- /**
- * @param args Parsed arguments.
- */
- public GridHadoopExternalProcessStarter(Args args) {
- this.args = args;
- }
-
- /**
- * @param cmdArgs Process arguments.
- */
- public static void main(String[] cmdArgs) {
- try {
- Args args = arguments(cmdArgs);
-
- new GridHadoopExternalProcessStarter(args).run();
- }
- catch (Exception e) {
- System.err.println("Failed");
-
- System.err.println(e.getMessage());
-
- e.printStackTrace(System.err);
- }
- }
-
- /**
- *
- * @throws Exception
- */
- public void run() throws Exception {
- U.setWorkDirectory(args.workDir, U.getIgniteHome());
-
- File outputDir = outputDirectory();
-
- initializeStreams(outputDir);
-
- ExecutorService msgExecSvc = Executors.newFixedThreadPool(
- Integer.getInteger("MSG_THREAD_POOL_SIZE", Runtime.getRuntime().availableProcessors() * 2));
-
- IgniteLogger log = logger(outputDir);
-
- GridHadoopExternalCommunication comm = new GridHadoopExternalCommunication(
- args.nodeId,
- args.childProcId,
- new OptimizedMarshaller(),
- log,
- msgExecSvc,
- "external"
- );
-
- comm.start();
-
- GridHadoopProcessDescriptor nodeDesc = new GridHadoopProcessDescriptor(args.nodeId, args.parentProcId);
- nodeDesc.address(args.addr);
- nodeDesc.tcpPort(args.tcpPort);
- nodeDesc.sharedMemoryPort(args.shmemPort);
-
- GridHadoopChildProcessRunner runner = new GridHadoopChildProcessRunner();
-
- runner.start(comm, nodeDesc, msgExecSvc, log);
-
- System.err.println("Started");
- System.err.flush();
-
- System.setOut(new PrintStream(out));
- System.setErr(new PrintStream(err));
- }
-
- /**
- * @param outputDir Directory for process output.
- * @throws Exception
- */
- private void initializeStreams(File outputDir) throws Exception {
- out = new FileOutputStream(new File(outputDir, args.childProcId + ".out"));
- err = new FileOutputStream(new File(outputDir, args.childProcId + ".err"));
- }
-
- /**
- * @return Path to output directory.
- * @throws IOException If failed.
- */
- private File outputDirectory() throws IOException {
- File f = new File(args.out);
-
- if (!f.exists()) {
- if (!f.mkdirs())
- throw new IOException("Failed to create output directory: " + args.out);
- }
- else {
- if (f.isFile())
- throw new IOException("Output directory is a file: " + args.out);
- }
-
- return f;
- }
-
- /**
- * @param outputDir Directory for process output.
- * @return Logger.
- */
- private IgniteLogger logger(final File outputDir) {
- final URL url = U.resolveIgniteUrl(DFLT_LOG4J_CONFIG);
-
- Log4JLogger logger;
-
- try {
- logger = url != null ? new Log4JLogger(url) : new Log4JLogger(true);
- }
- catch (IgniteCheckedException e) {
- System.err.println("Failed to create URL-based logger. Will use default one.");
-
- e.printStackTrace();
-
- logger = new Log4JLogger(true);
- }
-
- logger.updateFilePath(new IgniteClosure<String, String>() {
- @Override public String apply(String s) {
- return new File(outputDir, args.childProcId + ".log").getAbsolutePath();
- }
- });
-
- return logger;
- }
-
- /**
- * @param processArgs Process arguments.
- * @return Child process instance.
- */
- private static Args arguments(String[] processArgs) throws Exception {
- Args args = new Args();
-
- for (int i = 0; i < processArgs.length; i++) {
- String arg = processArgs[i];
-
- switch (arg) {
- case "-cpid": {
- if (i == processArgs.length - 1)
- throw new Exception("Missing process ID for '-cpid' parameter");
-
- String procIdStr = processArgs[++i];
-
- args.childProcId = UUID.fromString(procIdStr);
-
- break;
- }
-
- case "-ppid": {
- if (i == processArgs.length - 1)
- throw new Exception("Missing process ID for '-ppid' parameter");
-
- String procIdStr = processArgs[++i];
-
- args.parentProcId = UUID.fromString(procIdStr);
-
- break;
- }
-
- case "-nid": {
- if (i == processArgs.length - 1)
- throw new Exception("Missing node ID for '-nid' parameter");
-
- String nodeIdStr = processArgs[++i];
-
- args.nodeId = UUID.fromString(nodeIdStr);
-
- break;
- }
-
- case "-addr": {
- if (i == processArgs.length - 1)
- throw new Exception("Missing node address for '-addr' parameter");
-
- args.addr = processArgs[++i];
-
- break;
- }
-
- case "-tport": {
- if (i == processArgs.length - 1)
- throw new Exception("Missing tcp port for '-tport' parameter");
-
- args.tcpPort = Integer.parseInt(processArgs[++i]);
-
- break;
- }
-
- case "-sport": {
- if (i == processArgs.length - 1)
- throw new Exception("Missing shared memory port for '-sport' parameter");
-
- args.shmemPort = Integer.parseInt(processArgs[++i]);
-
- break;
- }
-
- case "-out": {
- if (i == processArgs.length - 1)
- throw new Exception("Missing output folder name for '-out' parameter");
-
- args.out = processArgs[++i];
-
- break;
- }
-
- case "-wd": {
- if (i == processArgs.length - 1)
- throw new Exception("Missing work folder name for '-wd' parameter");
-
- args.workDir = processArgs[++i];
-
- break;
- }
- }
- }
-
- return args;
- }
-
- /**
- * Execution arguments.
- */
- private static class Args {
- /** Process ID. */
- private UUID childProcId;
-
- /** Process ID. */
- private UUID parentProcId;
-
- /** Process ID. */
- private UUID nodeId;
-
- /** Node address. */
- private String addr;
-
- /** TCP port */
- private int tcpPort;
-
- /** Shmem port. */
- private int shmemPort = -1;
-
- /** Output folder. */
- private String out;
-
- /** Work directory. */
- private String workDir;
- }
-}
[15/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2TaskContext.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2TaskContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2TaskContext.java
new file mode 100644
index 0000000..24f10a6
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2TaskContext.java
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.io.serializer.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapred.TaskAttemptID;
+import org.apache.hadoop.mapred.TaskID;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
+import org.apache.ignite.internal.processors.hadoop.fs.*;
+import org.apache.ignite.internal.processors.hadoop.v1.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*;
+import static org.apache.ignite.internal.processors.hadoop.HadoopUtils.*;
+
+/**
+ * Context for task execution.
+ */
+public class HadoopV2TaskContext extends HadoopTaskContext {
+ /** */
+ private static final boolean COMBINE_KEY_GROUPING_SUPPORTED;
+
+ /**
+ * Check for combiner grouping support (available since Hadoop 2.3).
+ */
+ static {
+ boolean ok;
+
+ try {
+ JobContext.class.getDeclaredMethod("getCombinerKeyGroupingComparator");
+
+ ok = true;
+ }
+ catch (NoSuchMethodException ignore) {
+ ok = false;
+ }
+
+ COMBINE_KEY_GROUPING_SUPPORTED = ok;
+ }
+
+ /** Flag is set if new context-object code is used for running the mapper. */
+ private final boolean useNewMapper;
+
+ /** Flag is set if new context-object code is used for running the reducer. */
+ private final boolean useNewReducer;
+
+ /** Flag is set if new context-object code is used for running the combiner. */
+ private final boolean useNewCombiner;
+
+ /** */
+ private final JobContextImpl jobCtx;
+
+ /** Set if task is to cancelling. */
+ private volatile boolean cancelled;
+
+ /** Current task. */
+ private volatile HadoopTask task;
+
+ /** Local node ID */
+ private UUID locNodeId;
+
+ /** Counters for task. */
+ private final HadoopCounters cntrs = new HadoopCountersImpl();
+
+ /**
+ * @param taskInfo Task info.
+ * @param job Job.
+ * @param jobId Job ID.
+ * @param locNodeId Local node ID.
+ * @param jobConfDataInput DataInput for read JobConf.
+ */
+ public HadoopV2TaskContext(HadoopTaskInfo taskInfo, HadoopJob job, HadoopJobId jobId,
+ @Nullable UUID locNodeId, DataInput jobConfDataInput) throws IgniteCheckedException {
+ super(taskInfo, job);
+ this.locNodeId = locNodeId;
+
+ // Before create JobConf instance we should set new context class loader.
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+
+ try {
+ JobConf jobConf = new JobConf();
+
+ try {
+ jobConf.readFields(jobConfDataInput);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+
+ // For map-reduce jobs prefer local writes.
+ jobConf.setBooleanIfUnset(PARAM_IGFS_PREFER_LOCAL_WRITES, true);
+
+ jobCtx = new JobContextImpl(jobConf, new JobID(jobId.globalId().toString(), jobId.localId()));
+
+ useNewMapper = jobConf.getUseNewMapper();
+ useNewReducer = jobConf.getUseNewReducer();
+ useNewCombiner = jobConf.getCombinerClass() == null;
+ }
+ finally {
+ Thread.currentThread().setContextClassLoader(null);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T extends HadoopCounter> T counter(String grp, String name, Class<T> cls) {
+ return cntrs.counter(grp, name, cls);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopCounters counters() {
+ return cntrs;
+ }
+
+ /**
+ * Creates appropriate task from current task info.
+ *
+ * @return Task.
+ */
+ private HadoopTask createTask() {
+ boolean isAbort = taskInfo().type() == HadoopTaskType.ABORT;
+
+ switch (taskInfo().type()) {
+ case SETUP:
+ return useNewMapper ? new HadoopV2SetupTask(taskInfo()) : new HadoopV1SetupTask(taskInfo());
+
+ case MAP:
+ return useNewMapper ? new HadoopV2MapTask(taskInfo()) : new HadoopV1MapTask(taskInfo());
+
+ case REDUCE:
+ return useNewReducer ? new HadoopV2ReduceTask(taskInfo(), true) :
+ new HadoopV1ReduceTask(taskInfo(), true);
+
+ case COMBINE:
+ return useNewCombiner ? new HadoopV2ReduceTask(taskInfo(), false) :
+ new HadoopV1ReduceTask(taskInfo(), false);
+
+ case COMMIT:
+ case ABORT:
+ return useNewReducer ? new HadoopV2CleanupTask(taskInfo(), isAbort) :
+ new HadoopV1CleanupTask(taskInfo(), isAbort);
+
+ default:
+ return null;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() throws IgniteCheckedException {
+ try {
+ Thread.currentThread().setContextClassLoader(jobConf().getClassLoader());
+
+ try {
+ task = createTask();
+ }
+ catch (Throwable e) {
+ throw transformException(e);
+ }
+
+ if (cancelled)
+ throw new HadoopTaskCancelledException("Task cancelled.");
+
+ try {
+ task.run(this);
+ }
+ catch (Throwable e) {
+ throw transformException(e);
+ }
+ }
+ finally {
+ task = null;
+
+ Thread.currentThread().setContextClassLoader(null);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancel() {
+ cancelled = true;
+
+ HadoopTask t = task;
+
+ if (t != null)
+ t.cancel();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void prepareTaskEnvironment() throws IgniteCheckedException {
+ File locDir;
+
+ switch(taskInfo().type()) {
+ case MAP:
+ case REDUCE:
+ job().prepareTaskEnvironment(taskInfo());
+
+ locDir = taskLocalDir(locNodeId, taskInfo());
+
+ break;
+
+ default:
+ locDir = jobLocalDir(locNodeId, taskInfo().jobId());
+ }
+
+ Thread.currentThread().setContextClassLoader(jobConf().getClassLoader());
+
+ try {
+ FileSystem fs = FileSystem.get(jobConf());
+
+ HadoopFileSystemsUtils.setUser(fs, jobConf().getUser());
+
+ LocalFileSystem locFs = FileSystem.getLocal(jobConf());
+
+ locFs.setWorkingDirectory(new Path(locDir.getAbsolutePath()));
+ }
+ catch (Throwable e) {
+ throw transformException(e);
+ }
+ finally {
+ Thread.currentThread().setContextClassLoader(null);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cleanupTaskEnvironment() throws IgniteCheckedException {
+ job().cleanupTaskEnvironment(taskInfo());
+ }
+
+ /**
+ * Creates Hadoop attempt ID.
+ *
+ * @return Attempt ID.
+ */
+ public TaskAttemptID attemptId() {
+ TaskID tid = new TaskID(jobCtx.getJobID(), taskType(taskInfo().type()), taskInfo().taskNumber());
+
+ return new TaskAttemptID(tid, taskInfo().attempt());
+ }
+
+ /**
+ * @param type Task type.
+ * @return Hadoop task type.
+ */
+ private TaskType taskType(HadoopTaskType type) {
+ switch (type) {
+ case SETUP:
+ return TaskType.JOB_SETUP;
+ case MAP:
+ case COMBINE:
+ return TaskType.MAP;
+
+ case REDUCE:
+ return TaskType.REDUCE;
+
+ case COMMIT:
+ case ABORT:
+ return TaskType.JOB_CLEANUP;
+
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Gets job configuration of the task.
+ *
+ * @return Job configuration.
+ */
+ public JobConf jobConf() {
+ return jobCtx.getJobConf();
+ }
+
+ /**
+ * Gets job context of the task.
+ *
+ * @return Job context.
+ */
+ public JobContextImpl jobContext() {
+ return jobCtx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopPartitioner partitioner() throws IgniteCheckedException {
+ Class<?> partClsOld = jobConf().getClass("mapred.partitioner.class", null);
+
+ if (partClsOld != null)
+ return new HadoopV1Partitioner(jobConf().getPartitionerClass(), jobConf());
+
+ try {
+ return new HadoopV2Partitioner(jobCtx.getPartitionerClass(), jobConf());
+ }
+ catch (ClassNotFoundException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /**
+ * Gets serializer for specified class.
+ *
+ * @param cls Class.
+ * @param jobConf Job configuration.
+ * @return Appropriate serializer.
+ */
+ @SuppressWarnings("unchecked")
+ private HadoopSerialization getSerialization(Class<?> cls, Configuration jobConf) throws IgniteCheckedException {
+ A.notNull(cls, "cls");
+
+ SerializationFactory factory = new SerializationFactory(jobConf);
+
+ Serialization<?> serialization = factory.getSerialization(cls);
+
+ if (serialization == null)
+ throw new IgniteCheckedException("Failed to find serialization for: " + cls.getName());
+
+ if (serialization.getClass() == WritableSerialization.class)
+ return new HadoopWritableSerialization((Class<? extends Writable>)cls);
+
+ return new HadoopSerializationWrapper(serialization, cls);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopSerialization keySerialization() throws IgniteCheckedException {
+ return getSerialization(jobCtx.getMapOutputKeyClass(), jobConf());
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopSerialization valueSerialization() throws IgniteCheckedException {
+ return getSerialization(jobCtx.getMapOutputValueClass(), jobConf());
+ }
+
+ /** {@inheritDoc} */
+ @Override public Comparator<Object> sortComparator() {
+ return (Comparator<Object>)jobCtx.getSortComparator();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Comparator<Object> groupComparator() {
+ Comparator<?> res;
+
+ switch (taskInfo().type()) {
+ case COMBINE:
+ res = COMBINE_KEY_GROUPING_SUPPORTED ?
+ jobContext().getCombinerKeyGroupingComparator() : jobContext().getGroupingComparator();
+
+ break;
+
+ case REDUCE:
+ res = jobContext().getGroupingComparator();
+
+ break;
+
+ default:
+ return null;
+ }
+
+ if (res != null && res.getClass() != sortComparator().getClass())
+ return (Comparator<Object>)res;
+
+ return null;
+ }
+
+ /**
+ * @param split Split.
+ * @return Native Hadoop split.
+ * @throws IgniteCheckedException if failed.
+ */
+ @SuppressWarnings("unchecked")
+ public Object getNativeSplit(HadoopInputSplit split) throws IgniteCheckedException {
+ if (split instanceof HadoopExternalSplit)
+ return readExternalSplit((HadoopExternalSplit)split);
+
+ if (split instanceof HadoopSplitWrapper)
+ return unwrapSplit((HadoopSplitWrapper)split);
+
+ throw new IllegalStateException("Unknown split: " + split);
+ }
+
+ /**
+ * @param split External split.
+ * @return Native input split.
+ * @throws IgniteCheckedException If failed.
+ */
+ @SuppressWarnings("unchecked")
+ private Object readExternalSplit(HadoopExternalSplit split) throws IgniteCheckedException {
+ Path jobDir = new Path(jobConf().get(MRJobConfig.MAPREDUCE_JOB_DIR));
+
+ try (FileSystem fs = FileSystem.get(jobDir.toUri(), jobConf());
+ FSDataInputStream in = fs.open(JobSubmissionFiles.getJobSplitFile(jobDir))) {
+
+ in.seek(split.offset());
+
+ String clsName = Text.readString(in);
+
+ Class<?> cls = jobConf().getClassByName(clsName);
+
+ assert cls != null;
+
+ Serialization serialization = new SerializationFactory(jobConf()).getSerialization(cls);
+
+ Deserializer deserializer = serialization.getDeserializer(cls);
+
+ deserializer.open(in);
+
+ Object res = deserializer.deserialize(null);
+
+ deserializer.close();
+
+ assert res != null;
+
+ return res;
+ }
+ catch (IOException | ClassNotFoundException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopWritableSerialization.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopWritableSerialization.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopWritableSerialization.java
new file mode 100644
index 0000000..3920dd5
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopWritableSerialization.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import org.apache.hadoop.io.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * Optimized serialization for Hadoop {@link Writable} types.
+ */
+public class HadoopWritableSerialization implements HadoopSerialization {
+ /** */
+ private final Class<? extends Writable> cls;
+
+ /**
+ * @param cls Class.
+ */
+ public HadoopWritableSerialization(Class<? extends Writable> cls) {
+ assert cls != null;
+
+ this.cls = cls;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(DataOutput out, Object obj) throws IgniteCheckedException {
+ assert cls.isAssignableFrom(obj.getClass()) : cls + " " + obj.getClass();
+
+ try {
+ ((Writable)obj).write(out);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object read(DataInput in, @Nullable Object obj) throws IgniteCheckedException {
+ Writable w = obj == null ? U.newInstance(cls) : cls.cast(obj);
+
+ try {
+ w.readFields(in);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+
+ return w;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/resources/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/resources/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider b/modules/hadoop/src/main/resources/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider
index fe35d5e..8d5957b 100644
--- a/modules/hadoop/src/main/resources/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider
+++ b/modules/hadoop/src/main/resources/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider
@@ -1 +1 @@
-org.apache.ignite.client.hadoop.GridHadoopClientProtocolProvider
+org.apache.ignite.hadoop.mapreduce.IgniteHadoopClientProtocolProvider
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocolEmbeddedSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocolEmbeddedSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocolEmbeddedSelfTest.java
deleted file mode 100644
index 780ce67..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocolEmbeddedSelfTest.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.client.hadoop;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-
-/**
- * Hadoop client protocol tests in embedded process mode.
- */
-public class GridHadoopClientProtocolEmbeddedSelfTest extends GridHadoopClientProtocolSelfTest {
- /** {@inheritDoc} */
- @Override public GridHadoopConfiguration hadoopConfiguration(String gridName) {
- GridHadoopConfiguration cfg = super.hadoopConfiguration(gridName);
-
- cfg.setExternalExecution(false);
-
- return cfg;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocolSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocolSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocolSelfTest.java
deleted file mode 100644
index ff8798b..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocolSelfTest.java
+++ /dev/null
@@ -1,633 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.client.hadoop;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.hadoop.mapreduce.lib.output.*;
-import org.apache.hadoop.mapreduce.protocol.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.testframework.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Hadoop client protocol tests in external process mode.
- */
-@SuppressWarnings("ResultOfMethodCallIgnored")
-public class GridHadoopClientProtocolSelfTest extends GridHadoopAbstractSelfTest {
- /** Input path. */
- private static final String PATH_INPUT = "/input";
-
- /** Output path. */
- private static final String PATH_OUTPUT = "/output";
-
- /** Job name. */
- private static final String JOB_NAME = "myJob";
-
- /** Setup lock file. */
- private static File setupLockFile = new File(U.isWindows() ? System.getProperty("java.io.tmpdir") : "/tmp",
- "ignite-lock-setup.file");
-
- /** Map lock file. */
- private static File mapLockFile = new File(U.isWindows() ? System.getProperty("java.io.tmpdir") : "/tmp",
- "ignite-lock-map.file");
-
- /** Reduce lock file. */
- private static File reduceLockFile = new File(U.isWindows() ? System.getProperty("java.io.tmpdir") : "/tmp",
- "ignite-lock-reduce.file");
-
- /** {@inheritDoc} */
- @Override protected int gridCount() {
- return 2;
- }
-
- /** {@inheritDoc} */
- @Override protected boolean igfsEnabled() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override protected boolean restEnabled() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
- startGrids(gridCount());
-
- setupLockFile.delete();
- mapLockFile.delete();
- reduceLockFile.delete();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- stopAllGrids();
-
- super.afterTestsStopped();
-
-// GridHadoopClientProtocolProvider.cliMap.clear();
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- setupLockFile.createNewFile();
- mapLockFile.createNewFile();
- reduceLockFile.createNewFile();
-
- setupLockFile.deleteOnExit();
- mapLockFile.deleteOnExit();
- reduceLockFile.deleteOnExit();
-
- super.beforeTest();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- grid(0).fileSystem(GridHadoopAbstractSelfTest.igfsName).format();
-
- setupLockFile.delete();
- mapLockFile.delete();
- reduceLockFile.delete();
-
- super.afterTest();
- }
-
- /**
- * Test next job ID generation.
- *
- * @throws Exception If failed.
- */
- @SuppressWarnings("ConstantConditions")
- private void tstNextJobId() throws Exception {
- GridHadoopClientProtocolProvider provider = provider();
-
- ClientProtocol proto = provider.create(config(GridHadoopAbstractSelfTest.REST_PORT));
-
- JobID jobId = proto.getNewJobID();
-
- assert jobId != null;
- assert jobId.getJtIdentifier() != null;
-
- JobID nextJobId = proto.getNewJobID();
-
- assert nextJobId != null;
- assert nextJobId.getJtIdentifier() != null;
-
- assert !F.eq(jobId, nextJobId);
- }
-
- /**
- * Tests job counters retrieval.
- *
- * @throws Exception If failed.
- */
- public void testJobCounters() throws Exception {
- IgniteFs igfs = grid(0).fileSystem(GridHadoopAbstractSelfTest.igfsName);
-
- igfs.mkdirs(new IgfsPath(PATH_INPUT));
-
- try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(igfs.create(
- new IgfsPath(PATH_INPUT + "/test.file"), true)))) {
-
- bw.write(
- "alpha\n" +
- "beta\n" +
- "gamma\n" +
- "alpha\n" +
- "beta\n" +
- "gamma\n" +
- "alpha\n" +
- "beta\n" +
- "gamma\n"
- );
- }
-
- Configuration conf = config(GridHadoopAbstractSelfTest.REST_PORT);
-
- final Job job = Job.getInstance(conf);
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- job.setMapperClass(TestCountingMapper.class);
- job.setReducerClass(TestCountingReducer.class);
- job.setCombinerClass(TestCountingCombiner.class);
-
- FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
- FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
-
- job.submit();
-
- final Counter cntr = job.getCounters().findCounter(TestCounter.COUNTER1);
-
- assertEquals(0, cntr.getValue());
-
- cntr.increment(10);
-
- assertEquals(10, cntr.getValue());
-
- // Transferring to map phase.
- setupLockFile.delete();
-
- // Transferring to reduce phase.
- mapLockFile.delete();
-
- job.waitForCompletion(false);
-
- assertEquals("job must end successfully", JobStatus.State.SUCCEEDED, job.getStatus().getState());
-
- final Counters counters = job.getCounters();
-
- assertNotNull("counters cannot be null", counters);
- assertEquals("wrong counters count", 3, counters.countCounters());
- assertEquals("wrong counter value", 15, counters.findCounter(TestCounter.COUNTER1).getValue());
- assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER2).getValue());
- assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER3).getValue());
- }
-
- /**
- * Tests job counters retrieval for unknown job id.
- *
- * @throws Exception If failed.
- */
- private void tstUnknownJobCounters() throws Exception {
- GridHadoopClientProtocolProvider provider = provider();
-
- ClientProtocol proto = provider.create(config(GridHadoopAbstractSelfTest.REST_PORT));
-
- try {
- proto.getJobCounters(new JobID(UUID.randomUUID().toString(), -1));
- fail("exception must be thrown");
- }
- catch (Exception e) {
- assert e instanceof IOException : "wrong error has been thrown";
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- private void tstJobSubmitMap() throws Exception {
- checkJobSubmit(true, true);
- }
-
- /**
- * @throws Exception If failed.
- */
- private void tstJobSubmitMapCombine() throws Exception {
- checkJobSubmit(false, true);
- }
-
- /**
- * @throws Exception If failed.
- */
- private void tstJobSubmitMapReduce() throws Exception {
- checkJobSubmit(true, false);
- }
-
- /**
- * @throws Exception If failed.
- */
- private void tstJobSubmitMapCombineReduce() throws Exception {
- checkJobSubmit(false, false);
- }
-
- /**
- * Test job submission.
- *
- * @param noCombiners Whether there are no combiners.
- * @param noReducers Whether there are no reducers.
- * @throws Exception If failed.
- */
- public void checkJobSubmit(boolean noCombiners, boolean noReducers) throws Exception {
- IgniteFs igfs = grid(0).fileSystem(GridHadoopAbstractSelfTest.igfsName);
-
- igfs.mkdirs(new IgfsPath(PATH_INPUT));
-
- try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(igfs.create(
- new IgfsPath(PATH_INPUT + "/test.file"), true)))) {
-
- bw.write("word");
- }
-
- Configuration conf = config(GridHadoopAbstractSelfTest.REST_PORT);
-
- final Job job = Job.getInstance(conf);
-
- job.setJobName(JOB_NAME);
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- job.setMapperClass(TestMapper.class);
- job.setReducerClass(TestReducer.class);
-
- if (!noCombiners)
- job.setCombinerClass(TestCombiner.class);
-
- if (noReducers)
- job.setNumReduceTasks(0);
-
- job.setInputFormatClass(TextInputFormat.class);
- job.setOutputFormatClass(TestOutputFormat.class);
-
- FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
- FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
-
- job.submit();
-
- JobID jobId = job.getJobID();
-
- // Setup phase.
- JobStatus jobStatus = job.getStatus();
- checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
- assert jobStatus.getSetupProgress() >= 0.0f && jobStatus.getSetupProgress() < 1.0f;
- assert jobStatus.getMapProgress() == 0.0f;
- assert jobStatus.getReduceProgress() == 0.0f;
-
- U.sleep(2100);
-
- JobStatus recentJobStatus = job.getStatus();
-
- assert recentJobStatus.getSetupProgress() > jobStatus.getSetupProgress() :
- "Old=" + jobStatus.getSetupProgress() + ", new=" + recentJobStatus.getSetupProgress();
-
- // Transferring to map phase.
- setupLockFile.delete();
-
- assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- try {
- return F.eq(1.0f, job.getStatus().getSetupProgress());
- }
- catch (Exception e) {
- throw new RuntimeException("Unexpected exception.", e);
- }
- }
- }, 5000L);
-
- // Map phase.
- jobStatus = job.getStatus();
- checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
- assert jobStatus.getSetupProgress() == 1.0f;
- assert jobStatus.getMapProgress() >= 0.0f && jobStatus.getMapProgress() < 1.0f;
- assert jobStatus.getReduceProgress() == 0.0f;
-
- U.sleep(2100);
-
- recentJobStatus = job.getStatus();
-
- assert recentJobStatus.getMapProgress() > jobStatus.getMapProgress() :
- "Old=" + jobStatus.getMapProgress() + ", new=" + recentJobStatus.getMapProgress();
-
- // Transferring to reduce phase.
- mapLockFile.delete();
-
- assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- try {
- return F.eq(1.0f, job.getStatus().getMapProgress());
- }
- catch (Exception e) {
- throw new RuntimeException("Unexpected exception.", e);
- }
- }
- }, 5000L);
-
- if (!noReducers) {
- // Reduce phase.
- jobStatus = job.getStatus();
- checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
- assert jobStatus.getSetupProgress() == 1.0f;
- assert jobStatus.getMapProgress() == 1.0f;
- assert jobStatus.getReduceProgress() >= 0.0f && jobStatus.getReduceProgress() < 1.0f;
-
- // Ensure that reduces progress increases.
- U.sleep(2100);
-
- recentJobStatus = job.getStatus();
-
- assert recentJobStatus.getReduceProgress() > jobStatus.getReduceProgress() :
- "Old=" + jobStatus.getReduceProgress() + ", new=" + recentJobStatus.getReduceProgress();
-
- reduceLockFile.delete();
- }
-
- job.waitForCompletion(false);
-
- jobStatus = job.getStatus();
- checkJobStatus(job.getStatus(), jobId, JOB_NAME, JobStatus.State.SUCCEEDED, 1.0f);
- assert jobStatus.getSetupProgress() == 1.0f;
- assert jobStatus.getMapProgress() == 1.0f;
- assert jobStatus.getReduceProgress() == 1.0f;
-
- dumpIgfs(igfs, new IgfsPath(PATH_OUTPUT));
- }
-
- /**
- * Dump IGFS content.
- *
- * @param igfs IGFS.
- * @param path Path.
- * @throws Exception If failed.
- */
- @SuppressWarnings("ConstantConditions")
- private static void dumpIgfs(IgniteFs igfs, IgfsPath path) throws Exception {
- IgfsFile file = igfs.info(path);
-
- assert file != null;
-
- System.out.println(file.path());
-
- if (file.isDirectory()) {
- for (IgfsPath child : igfs.listPaths(path))
- dumpIgfs(igfs, child);
- }
- else {
- try (BufferedReader br = new BufferedReader(new InputStreamReader(igfs.open(path)))) {
- String line = br.readLine();
-
- while (line != null) {
- System.out.println(line);
-
- line = br.readLine();
- }
- }
- }
- }
-
- /**
- * Check job status.
- *
- * @param status Job status.
- * @param expJobId Expected job ID.
- * @param expJobName Expected job name.
- * @param expState Expected state.
- * @param expCleanupProgress Expected cleanup progress.
- * @throws Exception If failed.
- */
- private static void checkJobStatus(JobStatus status, JobID expJobId, String expJobName,
- JobStatus.State expState, float expCleanupProgress) throws Exception {
- assert F.eq(status.getJobID(), expJobId) : "Expected=" + expJobId + ", actual=" + status.getJobID();
- assert F.eq(status.getJobName(), expJobName) : "Expected=" + expJobName + ", actual=" + status.getJobName();
- assert F.eq(status.getState(), expState) : "Expected=" + expState + ", actual=" + status.getState();
- assert F.eq(status.getCleanupProgress(), expCleanupProgress) :
- "Expected=" + expCleanupProgress + ", actual=" + status.getCleanupProgress();
- }
-
- /**
- * @return Configuration.
- */
- private Configuration config(int port) {
- Configuration conf = new Configuration();
-
- setupFileSystems(conf);
-
- conf.set(MRConfig.FRAMEWORK_NAME, GridHadoopClientProtocol.FRAMEWORK_NAME);
- conf.set(MRConfig.MASTER_ADDRESS, "127.0.0.1:" + port);
-
- conf.set("fs.defaultFS", "igfs://:" + getTestGridName(0) + "@/");
-
- return conf;
- }
-
- /**
- * @return Protocol provider.
- */
- private GridHadoopClientProtocolProvider provider() {
- return new GridHadoopClientProtocolProvider();
- }
-
- /**
- * Test mapper.
- */
- public static class TestMapper extends Mapper<Object, Text, Text, IntWritable> {
- /** Writable container for writing word. */
- private Text word = new Text();
-
- /** Writable integer constant of '1' is writing as count of found words. */
- private static final IntWritable one = new IntWritable(1);
-
- /** {@inheritDoc} */
- @Override public void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
- while (mapLockFile.exists())
- Thread.sleep(50);
-
- StringTokenizer wordList = new StringTokenizer(val.toString());
-
- while (wordList.hasMoreTokens()) {
- word.set(wordList.nextToken());
-
- ctx.write(word, one);
- }
- }
- }
-
- /**
- * Test Hadoop counters.
- */
- public enum TestCounter {
- COUNTER1, COUNTER2, COUNTER3
- }
-
- /**
- * Test mapper that uses counters.
- */
- public static class TestCountingMapper extends TestMapper {
- /** {@inheritDoc} */
- @Override public void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
- super.map(key, val, ctx);
- ctx.getCounter(TestCounter.COUNTER1).increment(1);
- }
- }
-
- /**
- * Test combiner that counts invocations.
- */
- public static class TestCountingCombiner extends TestReducer {
- @Override public void reduce(Text key, Iterable<IntWritable> values,
- Context ctx) throws IOException, InterruptedException {
- ctx.getCounter(TestCounter.COUNTER1).increment(1);
- ctx.getCounter(TestCounter.COUNTER2).increment(1);
-
- int sum = 0;
- for (IntWritable value : values) {
- sum += value.get();
- }
-
- ctx.write(key, new IntWritable(sum));
- }
- }
-
- /**
- * Test reducer that counts invocations.
- */
- public static class TestCountingReducer extends TestReducer {
- @Override public void reduce(Text key, Iterable<IntWritable> values,
- Context ctx) throws IOException, InterruptedException {
- ctx.getCounter(TestCounter.COUNTER1).increment(1);
- ctx.getCounter(TestCounter.COUNTER3).increment(1);
- }
- }
-
- /**
- * Test combiner.
- */
- public static class TestCombiner extends Reducer<Text, IntWritable, Text, IntWritable> {
- // No-op.
- }
-
- public static class TestOutputFormat<K, V> extends TextOutputFormat<K, V> {
- /** {@inheritDoc} */
- @Override public synchronized OutputCommitter getOutputCommitter(TaskAttemptContext ctx)
- throws IOException {
- return new TestOutputCommitter(ctx, (FileOutputCommitter)super.getOutputCommitter(ctx));
- }
- }
-
- /**
- * Test output committer.
- */
- private static class TestOutputCommitter extends FileOutputCommitter {
- /** Delegate. */
- private final FileOutputCommitter delegate;
-
- /**
- * Constructor.
- *
- * @param ctx Task attempt context.
- * @param delegate Delegate.
- * @throws IOException If failed.
- */
- private TestOutputCommitter(TaskAttemptContext ctx, FileOutputCommitter delegate) throws IOException {
- super(FileOutputFormat.getOutputPath(ctx), ctx);
-
- this.delegate = delegate;
- }
-
- /** {@inheritDoc} */
- @Override public void setupJob(JobContext jobCtx) throws IOException {
- try {
- while (setupLockFile.exists())
- Thread.sleep(50);
- }
- catch (InterruptedException ignored) {
- throw new IOException("Interrupted.");
- }
-
- delegate.setupJob(jobCtx);
- }
-
- /** {@inheritDoc} */
- @Override public void setupTask(TaskAttemptContext taskCtx) throws IOException {
- delegate.setupTask(taskCtx);
- }
-
- /** {@inheritDoc} */
- @Override public boolean needsTaskCommit(TaskAttemptContext taskCtx) throws IOException {
- return delegate.needsTaskCommit(taskCtx);
- }
-
- /** {@inheritDoc} */
- @Override public void commitTask(TaskAttemptContext taskCtx) throws IOException {
- delegate.commitTask(taskCtx);
- }
-
- /** {@inheritDoc} */
- @Override public void abortTask(TaskAttemptContext taskCtx) throws IOException {
- delegate.abortTask(taskCtx);
- }
- }
-
- /**
- * Test reducer.
- */
- public static class TestReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
- /** Writable container for writing sum of word counts. */
- private IntWritable totalWordCnt = new IntWritable();
-
- /** {@inheritDoc} */
- @Override public void reduce(Text key, Iterable<IntWritable> values, Context ctx) throws IOException,
- InterruptedException {
- while (reduceLockFile.exists())
- Thread.sleep(50);
-
- int wordCnt = 0;
-
- for (IntWritable value : values)
- wordCnt += value.get();
-
- totalWordCnt.set(wordCnt);
-
- ctx.write(key, totalWordCnt);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolEmbeddedSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolEmbeddedSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolEmbeddedSelfTest.java
new file mode 100644
index 0000000..ffa20d1
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolEmbeddedSelfTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.client.hadoop;
+
+import org.apache.ignite.configuration.*;
+
+/**
+ * Hadoop client protocol tests in embedded process mode.
+ */
+public class HadoopClientProtocolEmbeddedSelfTest extends HadoopClientProtocolSelfTest {
+ /** {@inheritDoc} */
+ @Override public HadoopConfiguration hadoopConfiguration(String gridName) {
+ HadoopConfiguration cfg = super.hadoopConfiguration(gridName);
+
+ cfg.setExternalExecution(false);
+
+ return cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolSelfTest.java
new file mode 100644
index 0000000..d19a8ea
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolSelfTest.java
@@ -0,0 +1,635 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.client.hadoop;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.*;
+import org.apache.hadoop.mapreduce.lib.output.*;
+import org.apache.hadoop.mapreduce.protocol.*;
+import org.apache.ignite.*;
+import org.apache.ignite.hadoop.mapreduce.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.proto.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.testframework.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Hadoop client protocol tests in external process mode.
+ */
+@SuppressWarnings("ResultOfMethodCallIgnored")
+public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
+ /** Input path. */
+ private static final String PATH_INPUT = "/input";
+
+ /** Output path. */
+ private static final String PATH_OUTPUT = "/output";
+
+ /** Job name. */
+ private static final String JOB_NAME = "myJob";
+
+ /** Setup lock file. */
+ private static File setupLockFile = new File(U.isWindows() ? System.getProperty("java.io.tmpdir") : "/tmp",
+ "ignite-lock-setup.file");
+
+ /** Map lock file. */
+ private static File mapLockFile = new File(U.isWindows() ? System.getProperty("java.io.tmpdir") : "/tmp",
+ "ignite-lock-map.file");
+
+ /** Reduce lock file. */
+ private static File reduceLockFile = new File(U.isWindows() ? System.getProperty("java.io.tmpdir") : "/tmp",
+ "ignite-lock-reduce.file");
+
+ /** {@inheritDoc} */
+ @Override protected int gridCount() {
+ return 2;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean igfsEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean restEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ startGrids(gridCount());
+
+ setupLockFile.delete();
+ mapLockFile.delete();
+ reduceLockFile.delete();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ super.afterTestsStopped();
+
+// IgniteHadoopClientProtocolProvider.cliMap.clear();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ setupLockFile.createNewFile();
+ mapLockFile.createNewFile();
+ reduceLockFile.createNewFile();
+
+ setupLockFile.deleteOnExit();
+ mapLockFile.deleteOnExit();
+ reduceLockFile.deleteOnExit();
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ grid(0).fileSystem(HadoopAbstractSelfTest.igfsName).format();
+
+ setupLockFile.delete();
+ mapLockFile.delete();
+ reduceLockFile.delete();
+
+ super.afterTest();
+ }
+
+ /**
+ * Test next job ID generation.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ConstantConditions")
+ private void tstNextJobId() throws Exception {
+ IgniteHadoopClientProtocolProvider provider = provider();
+
+ ClientProtocol proto = provider.create(config(HadoopAbstractSelfTest.REST_PORT));
+
+ JobID jobId = proto.getNewJobID();
+
+ assert jobId != null;
+ assert jobId.getJtIdentifier() != null;
+
+ JobID nextJobId = proto.getNewJobID();
+
+ assert nextJobId != null;
+ assert nextJobId.getJtIdentifier() != null;
+
+ assert !F.eq(jobId, nextJobId);
+ }
+
+ /**
+ * Tests job counters retrieval.
+ *
+ * @throws Exception If failed.
+ */
+ public void testJobCounters() throws Exception {
+ IgniteFileSystem igfs = grid(0).fileSystem(HadoopAbstractSelfTest.igfsName);
+
+ igfs.mkdirs(new IgfsPath(PATH_INPUT));
+
+ try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(igfs.create(
+ new IgfsPath(PATH_INPUT + "/test.file"), true)))) {
+
+ bw.write(
+ "alpha\n" +
+ "beta\n" +
+ "gamma\n" +
+ "alpha\n" +
+ "beta\n" +
+ "gamma\n" +
+ "alpha\n" +
+ "beta\n" +
+ "gamma\n"
+ );
+ }
+
+ Configuration conf = config(HadoopAbstractSelfTest.REST_PORT);
+
+ final Job job = Job.getInstance(conf);
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ job.setMapperClass(TestCountingMapper.class);
+ job.setReducerClass(TestCountingReducer.class);
+ job.setCombinerClass(TestCountingCombiner.class);
+
+ FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
+ FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
+
+ job.submit();
+
+ final Counter cntr = job.getCounters().findCounter(TestCounter.COUNTER1);
+
+ assertEquals(0, cntr.getValue());
+
+ cntr.increment(10);
+
+ assertEquals(10, cntr.getValue());
+
+ // Transferring to map phase.
+ setupLockFile.delete();
+
+ // Transferring to reduce phase.
+ mapLockFile.delete();
+
+ job.waitForCompletion(false);
+
+ assertEquals("job must end successfully", JobStatus.State.SUCCEEDED, job.getStatus().getState());
+
+ final Counters counters = job.getCounters();
+
+ assertNotNull("counters cannot be null", counters);
+ assertEquals("wrong counters count", 3, counters.countCounters());
+ assertEquals("wrong counter value", 15, counters.findCounter(TestCounter.COUNTER1).getValue());
+ assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER2).getValue());
+ assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER3).getValue());
+ }
+
+ /**
+ * Tests job counters retrieval for unknown job id.
+ *
+ * @throws Exception If failed.
+ */
+ private void tstUnknownJobCounters() throws Exception {
+ IgniteHadoopClientProtocolProvider provider = provider();
+
+ ClientProtocol proto = provider.create(config(HadoopAbstractSelfTest.REST_PORT));
+
+ try {
+ proto.getJobCounters(new JobID(UUID.randomUUID().toString(), -1));
+ fail("exception must be thrown");
+ }
+ catch (Exception e) {
+ assert e instanceof IOException : "wrong error has been thrown";
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void tstJobSubmitMap() throws Exception {
+ checkJobSubmit(true, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void tstJobSubmitMapCombine() throws Exception {
+ checkJobSubmit(false, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void tstJobSubmitMapReduce() throws Exception {
+ checkJobSubmit(true, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void tstJobSubmitMapCombineReduce() throws Exception {
+ checkJobSubmit(false, false);
+ }
+
+ /**
+ * Test job submission.
+ *
+ * @param noCombiners Whether there are no combiners.
+ * @param noReducers Whether there are no reducers.
+ * @throws Exception If failed.
+ */
+ public void checkJobSubmit(boolean noCombiners, boolean noReducers) throws Exception {
+ IgniteFileSystem igfs = grid(0).fileSystem(HadoopAbstractSelfTest.igfsName);
+
+ igfs.mkdirs(new IgfsPath(PATH_INPUT));
+
+ try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(igfs.create(
+ new IgfsPath(PATH_INPUT + "/test.file"), true)))) {
+
+ bw.write("word");
+ }
+
+ Configuration conf = config(HadoopAbstractSelfTest.REST_PORT);
+
+ final Job job = Job.getInstance(conf);
+
+ job.setJobName(JOB_NAME);
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ job.setMapperClass(TestMapper.class);
+ job.setReducerClass(TestReducer.class);
+
+ if (!noCombiners)
+ job.setCombinerClass(TestCombiner.class);
+
+ if (noReducers)
+ job.setNumReduceTasks(0);
+
+ job.setInputFormatClass(TextInputFormat.class);
+ job.setOutputFormatClass(TestOutputFormat.class);
+
+ FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
+ FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
+
+ job.submit();
+
+ JobID jobId = job.getJobID();
+
+ // Setup phase.
+ JobStatus jobStatus = job.getStatus();
+ checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
+ assert jobStatus.getSetupProgress() >= 0.0f && jobStatus.getSetupProgress() < 1.0f;
+ assert jobStatus.getMapProgress() == 0.0f;
+ assert jobStatus.getReduceProgress() == 0.0f;
+
+ U.sleep(2100);
+
+ JobStatus recentJobStatus = job.getStatus();
+
+ assert recentJobStatus.getSetupProgress() > jobStatus.getSetupProgress() :
+ "Old=" + jobStatus.getSetupProgress() + ", new=" + recentJobStatus.getSetupProgress();
+
+ // Transferring to map phase.
+ setupLockFile.delete();
+
+ assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ try {
+ return F.eq(1.0f, job.getStatus().getSetupProgress());
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Unexpected exception.", e);
+ }
+ }
+ }, 5000L);
+
+ // Map phase.
+ jobStatus = job.getStatus();
+ checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
+ assert jobStatus.getSetupProgress() == 1.0f;
+ assert jobStatus.getMapProgress() >= 0.0f && jobStatus.getMapProgress() < 1.0f;
+ assert jobStatus.getReduceProgress() == 0.0f;
+
+ U.sleep(2100);
+
+ recentJobStatus = job.getStatus();
+
+ assert recentJobStatus.getMapProgress() > jobStatus.getMapProgress() :
+ "Old=" + jobStatus.getMapProgress() + ", new=" + recentJobStatus.getMapProgress();
+
+ // Transferring to reduce phase.
+ mapLockFile.delete();
+
+ assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ try {
+ return F.eq(1.0f, job.getStatus().getMapProgress());
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Unexpected exception.", e);
+ }
+ }
+ }, 5000L);
+
+ if (!noReducers) {
+ // Reduce phase.
+ jobStatus = job.getStatus();
+ checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
+ assert jobStatus.getSetupProgress() == 1.0f;
+ assert jobStatus.getMapProgress() == 1.0f;
+ assert jobStatus.getReduceProgress() >= 0.0f && jobStatus.getReduceProgress() < 1.0f;
+
+ // Ensure that reduces progress increases.
+ U.sleep(2100);
+
+ recentJobStatus = job.getStatus();
+
+ assert recentJobStatus.getReduceProgress() > jobStatus.getReduceProgress() :
+ "Old=" + jobStatus.getReduceProgress() + ", new=" + recentJobStatus.getReduceProgress();
+
+ reduceLockFile.delete();
+ }
+
+ job.waitForCompletion(false);
+
+ jobStatus = job.getStatus();
+ checkJobStatus(job.getStatus(), jobId, JOB_NAME, JobStatus.State.SUCCEEDED, 1.0f);
+ assert jobStatus.getSetupProgress() == 1.0f;
+ assert jobStatus.getMapProgress() == 1.0f;
+ assert jobStatus.getReduceProgress() == 1.0f;
+
+ dumpIgfs(igfs, new IgfsPath(PATH_OUTPUT));
+ }
+
+ /**
+ * Dump IGFS content.
+ *
+ * @param igfs IGFS.
+ * @param path Path.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ConstantConditions")
+ private static void dumpIgfs(IgniteFileSystem igfs, IgfsPath path) throws Exception {
+ IgfsFile file = igfs.info(path);
+
+ assert file != null;
+
+ System.out.println(file.path());
+
+ if (file.isDirectory()) {
+ for (IgfsPath child : igfs.listPaths(path))
+ dumpIgfs(igfs, child);
+ }
+ else {
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(igfs.open(path)))) {
+ String line = br.readLine();
+
+ while (line != null) {
+ System.out.println(line);
+
+ line = br.readLine();
+ }
+ }
+ }
+ }
+
+ /**
+ * Check job status.
+ *
+ * @param status Job status.
+ * @param expJobId Expected job ID.
+ * @param expJobName Expected job name.
+ * @param expState Expected state.
+ * @param expCleanupProgress Expected cleanup progress.
+ * @throws Exception If failed.
+ */
+ private static void checkJobStatus(JobStatus status, JobID expJobId, String expJobName,
+ JobStatus.State expState, float expCleanupProgress) throws Exception {
+ assert F.eq(status.getJobID(), expJobId) : "Expected=" + expJobId + ", actual=" + status.getJobID();
+ assert F.eq(status.getJobName(), expJobName) : "Expected=" + expJobName + ", actual=" + status.getJobName();
+ assert F.eq(status.getState(), expState) : "Expected=" + expState + ", actual=" + status.getState();
+ assert F.eq(status.getCleanupProgress(), expCleanupProgress) :
+ "Expected=" + expCleanupProgress + ", actual=" + status.getCleanupProgress();
+ }
+
+ /**
+ * @return Configuration.
+ */
+ private Configuration config(int port) {
+ Configuration conf = new Configuration();
+
+ setupFileSystems(conf);
+
+ conf.set(MRConfig.FRAMEWORK_NAME, HadoopClientProtocol.FRAMEWORK_NAME);
+ conf.set(MRConfig.MASTER_ADDRESS, "127.0.0.1:" + port);
+
+ conf.set("fs.defaultFS", "igfs://:" + getTestGridName(0) + "@/");
+
+ return conf;
+ }
+
+ /**
+ * @return Protocol provider.
+ */
+ private IgniteHadoopClientProtocolProvider provider() {
+ return new IgniteHadoopClientProtocolProvider();
+ }
+
+ /**
+ * Test mapper.
+ */
+ public static class TestMapper extends Mapper<Object, Text, Text, IntWritable> {
+ /** Writable container for writing word. */
+ private Text word = new Text();
+
+ /** Writable integer constant of '1' is writing as count of found words. */
+ private static final IntWritable one = new IntWritable(1);
+
+ /** {@inheritDoc} */
+ @Override public void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
+ while (mapLockFile.exists())
+ Thread.sleep(50);
+
+ StringTokenizer wordList = new StringTokenizer(val.toString());
+
+ while (wordList.hasMoreTokens()) {
+ word.set(wordList.nextToken());
+
+ ctx.write(word, one);
+ }
+ }
+ }
+
+ /**
+ * Test Hadoop counters.
+ */
+ public enum TestCounter {
+ COUNTER1, COUNTER2, COUNTER3
+ }
+
+ /**
+ * Test mapper that uses counters.
+ */
+ public static class TestCountingMapper extends TestMapper {
+ /** {@inheritDoc} */
+ @Override public void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
+ super.map(key, val, ctx);
+ ctx.getCounter(TestCounter.COUNTER1).increment(1);
+ }
+ }
+
+ /**
+ * Test combiner that counts invocations.
+ */
+ public static class TestCountingCombiner extends TestReducer {
+ @Override public void reduce(Text key, Iterable<IntWritable> values,
+ Context ctx) throws IOException, InterruptedException {
+ ctx.getCounter(TestCounter.COUNTER1).increment(1);
+ ctx.getCounter(TestCounter.COUNTER2).increment(1);
+
+ int sum = 0;
+ for (IntWritable value : values) {
+ sum += value.get();
+ }
+
+ ctx.write(key, new IntWritable(sum));
+ }
+ }
+
+ /**
+ * Test reducer that counts invocations.
+ */
+ public static class TestCountingReducer extends TestReducer {
+ @Override public void reduce(Text key, Iterable<IntWritable> values,
+ Context ctx) throws IOException, InterruptedException {
+ ctx.getCounter(TestCounter.COUNTER1).increment(1);
+ ctx.getCounter(TestCounter.COUNTER3).increment(1);
+ }
+ }
+
+ /**
+ * Test combiner.
+ */
+ public static class TestCombiner extends Reducer<Text, IntWritable, Text, IntWritable> {
+ // No-op.
+ }
+
+ public static class TestOutputFormat<K, V> extends TextOutputFormat<K, V> {
+ /** {@inheritDoc} */
+ @Override public synchronized OutputCommitter getOutputCommitter(TaskAttemptContext ctx)
+ throws IOException {
+ return new TestOutputCommitter(ctx, (FileOutputCommitter)super.getOutputCommitter(ctx));
+ }
+ }
+
+ /**
+ * Test output committer.
+ */
+ private static class TestOutputCommitter extends FileOutputCommitter {
+ /** Delegate. */
+ private final FileOutputCommitter delegate;
+
+ /**
+ * Constructor.
+ *
+ * @param ctx Task attempt context.
+ * @param delegate Delegate.
+ * @throws IOException If failed.
+ */
+ private TestOutputCommitter(TaskAttemptContext ctx, FileOutputCommitter delegate) throws IOException {
+ super(FileOutputFormat.getOutputPath(ctx), ctx);
+
+ this.delegate = delegate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setupJob(JobContext jobCtx) throws IOException {
+ try {
+ while (setupLockFile.exists())
+ Thread.sleep(50);
+ }
+ catch (InterruptedException ignored) {
+ throw new IOException("Interrupted.");
+ }
+
+ delegate.setupJob(jobCtx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setupTask(TaskAttemptContext taskCtx) throws IOException {
+ delegate.setupTask(taskCtx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean needsTaskCommit(TaskAttemptContext taskCtx) throws IOException {
+ return delegate.needsTaskCommit(taskCtx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void commitTask(TaskAttemptContext taskCtx) throws IOException {
+ delegate.commitTask(taskCtx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void abortTask(TaskAttemptContext taskCtx) throws IOException {
+ delegate.abortTask(taskCtx);
+ }
+ }
+
+ /**
+ * Test reducer.
+ */
+ public static class TestReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
+ /** Writable container for writing sum of word counts. */
+ private IntWritable totalWordCnt = new IntWritable();
+
+ /** {@inheritDoc} */
+ @Override public void reduce(Text key, Iterable<IntWritable> values, Context ctx) throws IOException,
+ InterruptedException {
+ while (reduceLockFile.exists())
+ Thread.sleep(50);
+
+ int wordCnt = 0;
+
+ for (IntWritable value : values)
+ wordCnt += value.get();
+
+ totalWordCnt.set(wordCnt);
+
+ ctx.write(key, totalWordCnt);
+ }
+ }
+}
[08/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java
new file mode 100644
index 0000000..3b4c5c2
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.spi.communication.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+import org.apache.ignite.testframework.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheDistributionMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils.*;
+import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
+
+/**
+ * Tests for IGFS file system handshake.
+ */
+public class IgniteHadoopFileSystemHandshakeSelfTest extends IgfsCommonAbstractTest {
+ /** IP finder. */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** Grid name. */
+ private static final String GRID_NAME = "grid";
+
+ /** IGFS name. */
+ private static final String IGFS_NAME = "igfs";
+
+ /** IGFS path. */
+ private static final IgfsPath PATH = new IgfsPath("/path");
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids(true);
+ }
+
+ /**
+ * Tests for Grid and IGFS having normal names.
+ *
+ * @throws Exception If failed.
+ */
+ public void testHandshake() throws Exception {
+ startUp(false, false);
+
+ checkValid(IGFS_NAME + ":" + GRID_NAME + "@");
+ checkValid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1");
+ checkValid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid(IGFS_NAME + "@");
+ checkInvalid(IGFS_NAME + "@127.0.0.1");
+ checkInvalid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid(":" + GRID_NAME + "@");
+ checkInvalid(":" + GRID_NAME + "@127.0.0.1");
+ checkInvalid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid("");
+ checkInvalid("127.0.0.1");
+ checkInvalid("127.0.0.1:" + DFLT_IPC_PORT);
+ }
+
+ /**
+ * Tests for Grid having {@code null} name and IGFS having normal name.
+ *
+ * @throws Exception If failed.
+ */
+ public void testHandshakeDefaultGrid() throws Exception {
+ startUp(true, false);
+
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@");
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1");
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkValid(IGFS_NAME + "@");
+ checkValid(IGFS_NAME + "@127.0.0.1");
+ checkValid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid(":" + GRID_NAME + "@");
+ checkInvalid(":" + GRID_NAME + "@127.0.0.1");
+ checkInvalid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid("");
+ checkInvalid("127.0.0.1");
+ checkInvalid("127.0.0.1:" + DFLT_IPC_PORT);
+ }
+
+ /**
+ * Tests for Grid having normal name and IGFS having {@code null} name.
+ *
+ * @throws Exception If failed.
+ */
+ public void testHandshakeDefaultIgfs() throws Exception {
+ startUp(false, true);
+
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@");
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1");
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid(IGFS_NAME + "@");
+ checkInvalid(IGFS_NAME + "@127.0.0.1");
+ checkInvalid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkValid(":" + GRID_NAME + "@");
+ checkValid(":" + GRID_NAME + "@127.0.0.1");
+ checkValid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid("");
+ checkInvalid("127.0.0.1");
+ checkInvalid("127.0.0.1:" + DFLT_IPC_PORT);
+ }
+
+ /**
+ * Tests for Grid having {@code null} name and IGFS having {@code null} name.
+ *
+ * @throws Exception If failed.
+ */
+ public void testHandshakeDefaultGridDefaultIgfs() throws Exception {
+ startUp(true, true);
+
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@");
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1");
+ checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid(IGFS_NAME + "@");
+ checkInvalid(IGFS_NAME + "@127.0.0.1");
+ checkInvalid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkInvalid(":" + GRID_NAME + "@");
+ checkInvalid(":" + GRID_NAME + "@127.0.0.1");
+ checkInvalid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT);
+
+ checkValid("");
+ checkValid("127.0.0.1");
+ checkValid("127.0.0.1:" + DFLT_IPC_PORT);
+ }
+
+ /**
+ * Perform startup.
+ *
+ * @param dfltGridName Default Grid name.
+ * @param dfltIgfsName Default IGFS name.
+ * @throws Exception If failed.
+ */
+ private void startUp(boolean dfltGridName, boolean dfltIgfsName) throws Exception {
+ Ignite ignite = G.start(gridConfiguration(dfltGridName, dfltIgfsName));
+
+ IgniteFileSystem igfs = ignite.fileSystem(dfltIgfsName ? null : IGFS_NAME);
+
+ igfs.mkdirs(PATH);
+ }
+
+ /**
+ * Create Grid configuration.
+ *
+ * @param dfltGridName Default Grid name.
+ * @param dfltIgfsName Default IGFS name.
+ * @return Grid configuration.
+ * @throws Exception If failed.
+ */
+ private IgniteConfiguration gridConfiguration(boolean dfltGridName, boolean dfltIgfsName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(dfltGridName ? null : GRID_NAME);
+
+ cfg.setLocalHost("127.0.0.1");
+ cfg.setConnectorConfiguration(null);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(discoSpi);
+
+ TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
+
+ commSpi.setSharedMemoryPort(-1);
+
+ cfg.setCommunicationSpi(commSpi);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
+
+ dataCacheCfg.setName("partitioned");
+ dataCacheCfg.setCacheMode(PARTITIONED);
+ dataCacheCfg.setDistributionMode(PARTITIONED_ONLY);
+ dataCacheCfg.setWriteSynchronizationMode(FULL_SYNC);
+ dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ dataCacheCfg.setBackups(0);
+ dataCacheCfg.setQueryIndexEnabled(false);
+ dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ cfg.setCacheConfiguration(metaCacheCfg, dataCacheCfg);
+
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName(dfltIgfsName ? null : IGFS_NAME);
+ igfsCfg.setPrefetchBlocks(1);
+ igfsCfg.setDefaultMode(PRIMARY);
+ igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", String.valueOf(DFLT_IPC_PORT));
+ }});
+
+ igfsCfg.setManagementPort(-1);
+ igfsCfg.setBlockSize(512 * 1024);
+
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ return cfg;
+ }
+
+ /**
+ * Check valid file system endpoint.
+ *
+ * @param authority Authority.
+ * @throws Exception If failed.
+ */
+ private void checkValid(String authority) throws Exception {
+ FileSystem fs = fileSystem(authority);
+
+ assert fs.exists(new Path(PATH.toString()));
+ }
+
+ /**
+ * Check invalid file system endpoint.
+ *
+ * @param authority Authority.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ private void checkInvalid(final String authority) throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ fileSystem(authority);
+
+ return null;
+ }
+ }, IOException.class, null);
+ }
+
+ /**
+ *
+ *
+ * @param authority Authority.
+ * @return File system.
+ * @throws Exception If failed.
+ */
+ private static FileSystem fileSystem(String authority) throws Exception {
+ return FileSystem.get(new URI("igfs://" + authority + "/"), configuration(authority));
+ }
+
+ /**
+ * Create configuration for test.
+ *
+ * @param authority Authority.
+ * @return Configuration.
+ */
+ private static Configuration configuration(String authority) {
+ Configuration cfg = new Configuration();
+
+ cfg.set("fs.defaultFS", "igfs://" + authority + "/");
+ cfg.set("fs.igfs.impl", org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.class.getName());
+ cfg.set("fs.AbstractFileSystem.igfs.impl",
+ IgniteHadoopFileSystem.class.getName());
+
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ cfg.setBoolean(String.format(PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);
+ cfg.setBoolean(String.format(PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);
+
+ return cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemIpcCacheSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemIpcCacheSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemIpcCacheSelfTest.java
new file mode 100644
index 0000000..135a488
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemIpcCacheSelfTest.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.cache.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.internal.processors.hadoop.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.ipc.shmem.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.spi.communication.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+
+import java.lang.reflect.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.events.EventType.*;
+
+/**
+ * IPC cache test.
+ */
+public class IgniteHadoopFileSystemIpcCacheSelfTest extends IgfsCommonAbstractTest {
+ /** IP finder. */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** Path to test hadoop configuration. */
+ private static final String HADOOP_FS_CFG = "modules/core/src/test/config/hadoop/core-site.xml";
+
+ /** Group size. */
+ public static final int GRP_SIZE = 128;
+
+ /** Started grid counter. */
+ private static int cnt;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+ discoSpi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(discoSpi);
+
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName("igfs");
+ igfsCfg.setManagementPort(FileSystemConfiguration.DFLT_MGMT_PORT + cnt);
+
+ igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
+ put("type", "shmem");
+ put("port", String.valueOf(IpcSharedMemoryServerEndpoint.DFLT_IPC_PORT + cnt));
+ }});
+
+ igfsCfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups.
+
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ cfg.setCacheConfiguration(cacheConfiguration());
+
+ cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
+
+ TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
+
+ commSpi.setSharedMemoryPort(-1);
+
+ cfg.setCommunicationSpi(commSpi);
+
+ cnt++;
+
+ return cfg;
+ }
+
+ /**
+ * Gets cache configuration.
+ *
+ * @return Cache configuration.
+ */
+ private CacheConfiguration[] cacheConfiguration() {
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ return new CacheConfiguration[] {metaCacheCfg, cacheCfg};
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ startGrids(4);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ G.stopAll(true);
+ }
+
+ /**
+ * Test how IPC cache map works.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ public void testIpcCache() throws Exception {
+ Field cacheField = HadoopIgfsIpcIo.class.getDeclaredField("ipcCache");
+
+ cacheField.setAccessible(true);
+
+ Field activeCntField = HadoopIgfsIpcIo.class.getDeclaredField("activeCnt");
+
+ activeCntField.setAccessible(true);
+
+ Map<String, HadoopIgfsIpcIo> cache = (Map<String, HadoopIgfsIpcIo>)cacheField.get(null);
+
+ String name = "igfs:" + getTestGridName(0) + "@";
+
+ Configuration cfg = new Configuration();
+
+ cfg.addResource(U.resolveIgniteUrl(HADOOP_FS_CFG));
+ cfg.setBoolean("fs.igfs.impl.disable.cache", true);
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, name), true);
+
+ // Ensure that existing IO is reused.
+ FileSystem fs1 = FileSystem.get(new URI("igfs://" + name + "/"), cfg);
+
+ assertEquals(1, cache.size());
+
+ HadoopIgfsIpcIo io = null;
+
+ System.out.println("CACHE: " + cache);
+
+ for (String key : cache.keySet()) {
+ if (key.contains("10500")) {
+ io = cache.get(key);
+
+ break;
+ }
+ }
+
+ assert io != null;
+
+ assertEquals(1, ((AtomicInteger)activeCntField.get(io)).get());
+
+ // Ensure that when IO is used by multiple file systems and one of them is closed, IO is not stopped.
+ FileSystem fs2 = FileSystem.get(new URI("igfs://" + name + "/abc"), cfg);
+
+ assertEquals(1, cache.size());
+ assertEquals(2, ((AtomicInteger)activeCntField.get(io)).get());
+
+ fs2.close();
+
+ assertEquals(1, cache.size());
+ assertEquals(1, ((AtomicInteger)activeCntField.get(io)).get());
+
+ Field stopField = HadoopIgfsIpcIo.class.getDeclaredField("stopping");
+
+ stopField.setAccessible(true);
+
+ assert !(Boolean)stopField.get(io);
+
+ // Ensure that IO is stopped when nobody else is need it.
+ fs1.close();
+
+ assert cache.isEmpty();
+
+ assert (Boolean)stopField.get(io);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerSelfTest.java
new file mode 100644
index 0000000..8d43e08
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerSelfTest.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.ignite.internal.igfs.common.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.igfs.common.IgfsLogger.*;
+
+/**
+ * Grid IGFS client logger test.
+ */
+public class IgniteHadoopFileSystemLoggerSelfTest extends IgfsCommonAbstractTest {
+ /** Path string. */
+ private static final String PATH_STR = "/dir1/dir2/file;test";
+
+ /** Path string with escaped semicolons. */
+ private static final String PATH_STR_ESCAPED = PATH_STR.replace(';', '~');
+
+ /** Path. */
+ private static final IgfsPath PATH = new IgfsPath(PATH_STR);
+
+ /** IGFS name. */
+ private static final String IGFS_NAME = "igfs";
+
+ /** Log file path. */
+ private static final String LOG_DIR = U.getIgniteHome();
+
+ /** Endpoint address. */
+ private static final String ENDPOINT = "localhost:10500";
+
+ /** Log file name. */
+ private static final String LOG_FILE = LOG_DIR + File.separator + "igfs-log-" + IGFS_NAME + "-" + U.jvmPid() +
+ ".csv";
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ removeLogs();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ removeLogs();
+ }
+
+ /**
+ * Remove existing logs.
+ *
+ * @throws Exception If failed.
+ */
+ private void removeLogs() throws Exception {
+ File dir = new File(LOG_DIR);
+
+ File[] logs = dir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return name.startsWith("igfs-log-");
+ }
+ });
+
+ for (File log : logs)
+ log.delete();
+ }
+
+ /**
+ * Ensure correct static loggers creation/removal as well as file creation.
+ *
+ * @throws Exception If failed.
+ */
+ public void testCreateDelete() throws Exception {
+ IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ IgfsLogger sameLog0 = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ // Loggers for the same endpoint must be the same object.
+ assert log == sameLog0;
+
+ IgfsLogger otherLog = IgfsLogger.logger("other" + ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ // Logger for another endpoint must be different.
+ assert log != otherLog;
+
+ otherLog.close();
+
+ log.logDelete(PATH, PRIMARY, false);
+
+ log.close();
+
+ File logFile = new File(LOG_FILE);
+
+ // When there are multiple loggers, closing one must not force flushing.
+ assert !logFile.exists();
+
+ IgfsLogger sameLog1 = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ assert sameLog0 == sameLog1;
+
+ sameLog0.close();
+
+ assert !logFile.exists();
+
+ sameLog1.close();
+
+ // When we cloe the last logger, it must flush data to disk.
+ assert logFile.exists();
+
+ logFile.delete();
+
+ IgfsLogger sameLog2 = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ // This time we expect new logger instance to be created.
+ assert sameLog0 != sameLog2;
+
+ sameLog2.close();
+
+ // As we do not add any records to the logger, we do not expect flushing.
+ assert !logFile.exists();
+ }
+
+ /**
+ * Test read operations logging.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLogRead() throws Exception {
+ IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ log.logOpen(1, PATH, PRIMARY, 2, 3L);
+ log.logRandomRead(1, 4L, 5);
+ log.logSeek(1, 6L);
+ log.logSkip(1, 7L);
+ log.logMark(1, 8L);
+ log.logReset(1);
+ log.logCloseIn(1, 9L, 10L, 11);
+
+ log.close();
+
+ checkLog(
+ new SB().a(U.jvmPid() + d() + TYPE_OPEN_IN + d() + PATH_STR_ESCAPED + d() + PRIMARY + d() + 1 + d() + 2 +
+ d() + 3 + d(14)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_RANDOM_READ + d(3) + 1 + d(7) + 4 + d() + 5 + d(8)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_SEEK + d(3) + 1 + d(7) + 6 + d(9)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_SKIP + d(3) + 1 + d(9) + 7 + d(7)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_MARK + d(3) + 1 + d(10) + 8 + d(6)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_RESET + d(3) + 1 + d(16)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_CLOSE_IN + d(3) + 1 + d(11) + 9 + d() + 10 + d() + 11 + d(3)).toString()
+ );
+ }
+
+ /**
+ * Test write operations logging.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLogWrite() throws Exception {
+ IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ log.logCreate(1, PATH, PRIMARY, true, 2, new Integer(3).shortValue(), 4L);
+ log.logAppend(2, PATH, PRIMARY, 8);
+ log.logCloseOut(2, 9L, 10L, 11);
+
+ log.close();
+
+ checkLog(
+ new SB().a(U.jvmPid() + d() + TYPE_OPEN_OUT + d() + PATH_STR_ESCAPED + d() + PRIMARY + d() + 1 + d() +
+ 2 + d(2) + 0 + d() + 1 + d() + 3 + d() + 4 + d(10)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_OPEN_OUT + d() + PATH_STR_ESCAPED + d() + PRIMARY + d() + 2 + d() +
+ 8 + d(2) + 1 + d(13)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_CLOSE_OUT + d(3) + 2 + d(11) + 9 + d() + 10 + d() + 11 + d(3))
+ .toString()
+ );
+ }
+
+ /**
+ * Test miscellaneous operations logging.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("TooBroadScope")
+ public void testLogMisc() throws Exception {
+ IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10);
+
+ String newFile = "/dir3/file.test";
+ String file1 = "/dir3/file1.test";
+ String file2 = "/dir3/file1.test";
+
+ log.logMakeDirectory(PATH, PRIMARY);
+ log.logRename(PATH, PRIMARY, new IgfsPath(newFile));
+ log.logListDirectory(PATH, PRIMARY, new String[] { file1, file2 });
+ log.logDelete(PATH, PRIMARY, false);
+
+ log.close();
+
+ checkLog(
+ new SB().a(U.jvmPid() + d() + TYPE_DIR_MAKE + d() + PATH_STR_ESCAPED + d() + PRIMARY + d(17)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_RENAME + d() + PATH_STR_ESCAPED + d() + PRIMARY + d(15) + newFile +
+ d(2)).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_DIR_LIST + d() + PATH_STR_ESCAPED + d() + PRIMARY + d(17) + file1 +
+ DELIM_FIELD_VAL + file2).toString(),
+ new SB().a(U.jvmPid() + d() + TYPE_DELETE + d(1) + PATH_STR_ESCAPED + d() + PRIMARY + d(16) + 0 +
+ d()).toString()
+ );
+ }
+
+ /**
+ * Create IGFS file with the given path.
+ *
+ * @param path File path.
+ * @return IGFS file instance.
+ */
+ private IgfsFile file(String path) {
+ return new IgfsFileImpl(new IgfsPath(path), new IgfsFileInfo(), 64 * 1024 * 1024);
+ }
+
+ /**
+ * Ensure that log file has only the following lines.
+ *
+ * @param lines Expected lines.
+ */
+ private void checkLog(String... lines) throws Exception {
+ BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(LOG_FILE)));
+
+ List<String> logLines = new ArrayList<>(lines.length);
+
+ String nextLogLine;
+
+ while ((nextLogLine = br.readLine()) != null)
+ logLines.add(nextLogLine);
+
+ U.closeQuiet(br);
+
+ assertEquals(lines.length + 1, logLines.size());
+
+ assertEquals(logLines.get(0), HDR);
+
+ for (int i = 0; i < lines.length; i++) {
+ String logLine = logLines.get(i + 1);
+
+ logLine = logLine.substring(logLine.indexOf(DELIM_FIELD, logLine.indexOf(DELIM_FIELD) + 1) + 1);
+
+ assertEquals(lines[i], logLine);
+ }
+ }
+
+ /**
+ * Return single field delimiter.
+ *
+ * @return Single field delimiter.
+ */
+ private String d() {
+ return d(1);
+ }
+
+ /**
+ * Return a bunch of field delimiters.
+ *
+ * @param cnt Amount of field delimiters.
+ * @return Field delimiters.
+ */
+ private String d(int cnt) {
+ SB buf = new SB();
+
+ for (int i = 0; i < cnt; i++)
+ buf.a(DELIM_FIELD);
+
+ return buf.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerStateSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerStateSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerStateSelfTest.java
new file mode 100644
index 0000000..1f6a204
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerStateSelfTest.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.ignite.*;
+import org.apache.ignite.cache.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.hadoop.fs.v1.*;
+import org.apache.ignite.internal.igfs.common.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+
+import java.lang.reflect.*;
+import java.net.*;
+import java.nio.file.*;
+import java.util.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*;
+
+/**
+ * Ensures that sampling is really turned on/off.
+ */
+public class IgniteHadoopFileSystemLoggerStateSelfTest extends IgfsCommonAbstractTest {
+ /** IGFS. */
+ private IgfsEx igfs;
+
+ /** File system. */
+ private FileSystem fs;
+
+ /** Whether logging is enabled in FS configuration. */
+ private boolean logging;
+
+ /** whether sampling is enabled. */
+ private Boolean sampling;
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ U.closeQuiet(fs);
+
+ igfs = null;
+ fs = null;
+
+ G.stopAll(true);
+
+ logging = false;
+ sampling = null;
+ }
+
+ /**
+ * Startup the grid and instantiate the file system.
+ *
+ * @throws Exception If failed.
+ */
+ private void startUp() throws Exception {
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName("igfs");
+ igfsCfg.setBlockSize(512 * 1024);
+ igfsCfg.setDefaultMode(PRIMARY);
+ igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", "10500");
+ }});
+
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("igfs-grid");
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ cfg.setLocalHost("127.0.0.1");
+ cfg.setConnectorConfiguration(null);
+
+ Ignite g = G.start(cfg);
+
+ igfs = (IgfsEx)g.fileSystem("igfs");
+
+ igfs.globalSampling(sampling);
+
+ fs = fileSystem();
+ }
+
+ /**
+ * When logging is disabled and sampling is not set no-op logger must be used.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLoggingDisabledSamplingNotSet() throws Exception {
+ startUp();
+
+ assert !logEnabled();
+ }
+
+ /**
+ * When logging is enabled and sampling is not set file logger must be used.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLoggingEnabledSamplingNotSet() throws Exception {
+ logging = true;
+
+ startUp();
+
+ assert logEnabled();
+ }
+
+ /**
+ * When logging is disabled and sampling is disabled no-op logger must be used.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLoggingDisabledSamplingDisabled() throws Exception {
+ sampling = false;
+
+ startUp();
+
+ assert !logEnabled();
+ }
+
+ /**
+ * When logging is enabled and sampling is disabled no-op logger must be used.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLoggingEnabledSamplingDisabled() throws Exception {
+ logging = true;
+ sampling = false;
+
+ startUp();
+
+ assert !logEnabled();
+ }
+
+ /**
+ * When logging is disabled and sampling is enabled file logger must be used.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLoggingDisabledSamplingEnabled() throws Exception {
+ sampling = true;
+
+ startUp();
+
+ assert logEnabled();
+ }
+
+ /**
+ * When logging is enabled and sampling is enabled file logger must be used.
+ *
+ * @throws Exception If failed.
+ */
+ public void testLoggingEnabledSamplingEnabled() throws Exception {
+ logging = true;
+ sampling = true;
+
+ startUp();
+
+ assert logEnabled();
+ }
+
+ /**
+ * Ensure sampling change through API causes changes in logging on subsequent client connections.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSamplingChange() throws Exception {
+ // Start with sampling not set.
+ startUp();
+
+ assert !logEnabled();
+
+ fs.close();
+
+ // "Not set" => true transition.
+ igfs.globalSampling(true);
+
+ fs = fileSystem();
+
+ assert logEnabled();
+
+ fs.close();
+
+ // True => "not set" transition.
+ igfs.globalSampling(null);
+
+ fs = fileSystem();
+
+ assert !logEnabled();
+
+ // "Not-set" => false transition.
+ igfs.globalSampling(false);
+
+ fs = fileSystem();
+
+ assert !logEnabled();
+
+ fs.close();
+
+ // False => "not=set" transition.
+ igfs.globalSampling(null);
+
+ fs = fileSystem();
+
+ assert !logEnabled();
+
+ fs.close();
+
+ // True => false transition.
+ igfs.globalSampling(true);
+ igfs.globalSampling(false);
+
+ fs = fileSystem();
+
+ assert !logEnabled();
+
+ fs.close();
+
+ // False => true transition.
+ igfs.globalSampling(true);
+
+ fs = fileSystem();
+
+ assert logEnabled();
+ }
+
+ /**
+ * Ensure that log directory is set to IGFS when client FS connects.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ConstantConditions")
+ public void testLogDirectory() throws Exception {
+ startUp();
+
+ assertEquals(Paths.get(U.getIgniteHome()).normalize().toString(),
+ igfs.clientLogDirectory());
+ }
+
+ /**
+ * Instantiate new file system.
+ *
+ * @return New file system.
+ * @throws Exception If failed.
+ */
+ private IgniteHadoopFileSystem fileSystem() throws Exception {
+ Configuration fsCfg = new Configuration();
+
+ fsCfg.addResource(U.resolveIgniteUrl("modules/core/src/test/config/hadoop/core-site-loopback.xml"));
+
+ fsCfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ if (logging)
+ fsCfg.setBoolean(String.format(PARAM_IGFS_LOG_ENABLED, "igfs:igfs-grid@"), logging);
+
+ fsCfg.setStrings(String.format(PARAM_IGFS_LOG_DIR, "igfs:igfs-grid@"), U.getIgniteHome());
+
+ return (IgniteHadoopFileSystem)FileSystem.get(new URI("igfs://igfs:igfs-grid@/"), fsCfg);
+ }
+
+ /**
+ * Ensure that real logger is used by the file system.
+ *
+ * @return {@code True} in case path is secondary.
+ * @throws Exception If failed.
+ */
+ private boolean logEnabled() throws Exception {
+ assert fs != null;
+
+ Field field = fs.getClass().getDeclaredField("clientLog");
+
+ field.setAccessible(true);
+
+ return ((IgfsLogger)field.get(fs)).isLogEnabled();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackAbstractSelfTest.java
new file mode 100644
index 0000000..e33cde7
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackAbstractSelfTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import java.util.*;
+
+import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test.
+ */
+public abstract class IgniteHadoopFileSystemLoopbackAbstractSelfTest extends
+ IgniteHadoopFileSystemAbstractSelfTest {
+ /**
+ * Constructor.
+ *
+ * @param mode IGFS mode.
+ * @param skipEmbed Skip embedded mode flag.
+ */
+ protected IgniteHadoopFileSystemLoopbackAbstractSelfTest(IgfsMode mode, boolean skipEmbed) {
+ super(mode, skipEmbed, true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Map<String, String> primaryIpcEndpointConfiguration(final String gridName) {
+ return new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", String.valueOf(DFLT_IPC_PORT + getTestGridIndex(gridName)));
+ }};
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java
new file mode 100644
index 0000000..d46001d
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in DUAL_ASYNC mode.
+ */
+public class IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest extends
+ IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest() {
+ super(DUAL_ASYNC, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java
new file mode 100644
index 0000000..fa64734
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in DUAL_SYNC mode.
+ */
+public class IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest
+ extends IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest() {
+ super(DUAL_SYNC, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java
new file mode 100644
index 0000000..9035acd
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in PRIMARY mode.
+ */
+public class IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest
+ extends IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest() {
+ super(PRIMARY, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java
new file mode 100644
index 0000000..8198c52
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in SECONDARY mode.
+ */
+public class IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest extends
+ IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest() {
+ super(PROXY, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java
new file mode 100644
index 0000000..246c516
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in DUAL_ASYNC mode.
+ */
+public class IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest extends
+ IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest() {
+ super(DUAL_ASYNC, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.java
new file mode 100644
index 0000000..b5140af
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in DUAL_SYNC mode.
+ */
+public class IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest
+ extends IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest() {
+ super(DUAL_SYNC, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.java
new file mode 100644
index 0000000..572ac4b
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in PRIMARY mode.
+ */
+public class IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest
+ extends IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest() {
+ super(PRIMARY, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.java
new file mode 100644
index 0000000..006c271
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC loopback self test in SECONDARY mode.
+ */
+public class IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest extends
+ IgniteHadoopFileSystemLoopbackAbstractSelfTest {
+
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest() {
+ super(PROXY, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java
new file mode 100644
index 0000000..3f20070
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.cache.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.hadoop.fs.*;
+import org.apache.ignite.hadoop.fs.v1.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+
+import java.net.*;
+import java.util.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * Ensures correct modes resolution for SECONDARY paths.
+ */
+public class IgniteHadoopFileSystemSecondaryModeSelfTest extends IgfsCommonAbstractTest {
+ /** Path to check. */
+ private static final Path PATH = new Path("/dir");
+
+ /** Pattern matching the path. */
+ private static final String PATTERN_MATCHES = "/dir";
+
+ /** Pattern doesn't matching the path. */
+ private static final String PATTERN_NOT_MATCHES = "/files";
+
+ /** Default IGFS mode. */
+ private IgfsMode mode;
+
+ /** Path modes. */
+ private Map<String, IgfsMode> pathModes;
+
+ /** File system. */
+ private IgniteHadoopFileSystem fs;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ mode = null;
+ pathModes = null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ U.closeQuiet(fs);
+
+ fs = null;
+
+ G.stopAll(true);
+ }
+
+ /**
+ * Perform initial startup.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("NullableProblems")
+ private void startUp() throws Exception {
+ startUpSecondary();
+
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName("igfs");
+ igfsCfg.setBlockSize(512 * 1024);
+ igfsCfg.setDefaultMode(mode);
+ igfsCfg.setPathModes(pathModes);
+ igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", "10500");
+ }});
+
+ igfsCfg.setManagementPort(-1);
+ igfsCfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(
+ "igfs://igfs-secondary:igfs-grid-secondary@127.0.0.1:11500/",
+ "modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml"));
+
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("igfs-grid");
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ cfg.setLocalHost("127.0.0.1");
+
+ G.start(cfg);
+
+ Configuration fsCfg = new Configuration();
+
+ fsCfg.addResource(U.resolveIgniteUrl("modules/core/src/test/config/hadoop/core-site-loopback.xml"));
+
+ fsCfg.setBoolean("fs.igfs.impl.disable.cache", true);
+
+ fs = (IgniteHadoopFileSystem)FileSystem.get(new URI("igfs://igfs:igfs-grid@/"), fsCfg);
+ }
+
+ /**
+ * Startup secondary file system.
+ *
+ * @throws Exception If failed.
+ */
+ private void startUpSecondary() throws Exception {
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("partitioned");
+ igfsCfg.setMetaCacheName("replicated");
+ igfsCfg.setName("igfs-secondary");
+ igfsCfg.setBlockSize(512 * 1024);
+ igfsCfg.setDefaultMode(PRIMARY);
+ igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", "11500");
+ }});
+
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setName("partitioned");
+ cacheCfg.setCacheMode(PARTITIONED);
+ cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ cacheCfg.setBackups(0);
+ cacheCfg.setQueryIndexEnabled(false);
+ cacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("replicated");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("igfs-grid-secondary");
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ cfg.setLocalHost("127.0.0.1");
+
+ G.start(cfg);
+ }
+
+ /**
+ * Check path resolution when secondary mode is not default and there are no other exclusion paths.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSecondaryNotDefaultNoExclusions() throws Exception {
+ mode = PRIMARY;
+
+ startUp();
+
+ assert !secondary(PATH);
+ assert !secondary(PATH);
+ }
+
+ /**
+ * Check path resolution when secondary mode is not default and there is no matching exclusion paths.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSecondaryNotDefaultNonMatchingExclusion() throws Exception {
+ mode = PRIMARY;
+
+ pathModes(F.t(PATTERN_NOT_MATCHES, PROXY));
+
+ startUp();
+
+ assert !secondary(PATH);
+ assert !secondary(PATH);
+ }
+
+ /**
+ * Check path resolution when secondary mode is not default and there is matching exclusion path.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSecondaryNotDefaultMatchingExclusion() throws Exception {
+ mode = PRIMARY;
+
+ pathModes(F.t(PATTERN_NOT_MATCHES, PROXY), F.t(PATTERN_MATCHES, PROXY));
+
+ startUp();
+
+ assert secondary(PATH);
+ assert secondary(PATH);
+ }
+
+ /**
+ * Check path resolution when secondary mode is default and there is no exclusion paths.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSecondaryDefaultNoExclusions() throws Exception {
+ mode = PROXY;
+
+ startUp();
+
+ assert secondary(PATH);
+ assert secondary(PATH);
+ }
+
+ /**
+ * Check path resolution when secondary mode is default and there is no matching exclusion paths.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSecondaryDefaultNonMatchingExclusion() throws Exception {
+ mode = PROXY;
+
+ pathModes(F.t(PATTERN_NOT_MATCHES, PRIMARY));
+
+ startUp();
+
+ assert secondary(PATH);
+ assert secondary(PATH);
+ }
+
+ /**
+ * Check path resolution when secondary mode is default and there is no matching exclusion paths.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSecondaryDefaultMatchingExclusion() throws Exception {
+ mode = PROXY;
+
+ pathModes(F.t(PATTERN_NOT_MATCHES, PRIMARY), F.t(PATTERN_MATCHES, PRIMARY));
+
+ startUp();
+
+ assert !secondary(PATH);
+ assert !secondary(PATH);
+ }
+
+ /**
+ * Set IGFS modes for particular paths.
+ *
+ * @param modes Modes.
+ */
+ @SafeVarargs
+ final void pathModes(IgniteBiTuple<String, IgfsMode>... modes) {
+ assert modes != null;
+
+ pathModes = new LinkedHashMap<>(modes.length, 1.0f);
+
+ for (IgniteBiTuple<String, IgfsMode> mode : modes)
+ pathModes.put(mode.getKey(), mode.getValue());
+ }
+
+ /**
+ * Check whether the given path is threaten as SECONDARY in the file system.
+ *
+ * @param path Path to check.
+ * @return {@code True} in case path is secondary.
+ * @throws Exception If failed.
+ */
+ private boolean secondary(Path path) throws Exception {
+ return fs.mode(path) == PROXY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
new file mode 100644
index 0000000..991045b
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.util.ipc.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.testframework.*;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
+
+/**
+ * IGFS Hadoop file system IPC self test.
+ */
+public abstract class IgniteHadoopFileSystemShmemAbstractSelfTest extends IgniteHadoopFileSystemAbstractSelfTest {
+ /**
+ * Constructor.
+ *
+ * @param mode IGFS mode.
+ * @param skipEmbed Skip embedded mode flag.
+ */
+ protected IgniteHadoopFileSystemShmemAbstractSelfTest(IgfsMode mode, boolean skipEmbed) {
+ super(mode, skipEmbed, false);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Map<String, String> primaryIpcEndpointConfiguration(final String gridName) {
+ return new HashMap<String, String>() {{
+ put("type", "shmem");
+ put("port", String.valueOf(DFLT_IPC_PORT + getTestGridIndex(gridName)));
+ }};
+ }
+
+ /**
+ * Checks correct behaviour in case when we run out of system
+ * resources.
+ *
+ * @throws Exception If error occurred.
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testOutOfResources() throws Exception {
+ final Collection<IpcEndpoint> eps = new LinkedList<>();
+
+ try {
+ IgniteCheckedException e = (IgniteCheckedException)GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @SuppressWarnings("InfiniteLoopStatement")
+ @Override public Object call() throws Exception {
+ while (true) {
+ IpcEndpoint ep = IpcEndpointFactory.connectEndpoint("shmem:10500", log);
+
+ eps.add(ep);
+ }
+ }
+ }, IgniteCheckedException.class, null);
+
+ assertNotNull(e);
+
+ String msg = e.getMessage();
+
+ assertTrue("Invalid exception: " + X.getFullStackTrace(e),
+ msg.contains("(error code: 28)") ||
+ msg.contains("(error code: 24)") ||
+ msg.contains("(error code: 12)"));
+ }
+ finally {
+ for (IpcEndpoint ep : eps)
+ ep.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java
new file mode 100644
index 0000000..ed34398
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in DUAL_ASYNC mode.
+ */
+public class IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest() {
+ super(DUAL_ASYNC, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java
new file mode 100644
index 0000000..4b7bfb1
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in DUAL_SYNC mode.
+ */
+public class IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest() {
+ super(DUAL_SYNC, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.java
new file mode 100644
index 0000000..c1393d3
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in PRIMARY mode.
+ */
+public class IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest() {
+ super(PRIMARY, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.java
new file mode 100644
index 0000000..2c97e7b
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in SECONDARY mode.
+ */
+public class IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest() {
+ super(PROXY, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java
new file mode 100644
index 0000000..6d9973f
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in DUAL_ASYNC mode.
+ */
+public class IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest() {
+ super(DUAL_ASYNC, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.java
new file mode 100644
index 0000000..6ec084d
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in DUAL_SYNC mode.
+ */
+public class IgniteHadoopFileSystemShmemExternalDualSyncSelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemExternalDualSyncSelfTest() {
+ super(DUAL_SYNC, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalPrimarySelfTest.java
new file mode 100644
index 0000000..03f3fee
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalPrimarySelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in PRIMARY mode.
+ */
+public class IgniteHadoopFileSystemShmemExternalPrimarySelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemExternalPrimarySelfTest() {
+ super(PRIMARY, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalSecondarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalSecondarySelfTest.java
new file mode 100644
index 0000000..bc27674
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalSecondarySelfTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * IGFS Hadoop file system IPC shmem self test in SECONDARY mode.
+ */
+public class IgniteHadoopFileSystemShmemExternalSecondarySelfTest
+ extends IgniteHadoopFileSystemShmemAbstractSelfTest {
+ /**
+ * Constructor.
+ */
+ public IgniteHadoopFileSystemShmemExternalSecondarySelfTest() {
+ super(PROXY, true);
+ }
+}
[22/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopExternalTaskExecutor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopExternalTaskExecutor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopExternalTaskExecutor.java
deleted file mode 100644
index 72185c0..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopExternalTaskExecutor.java
+++ /dev/null
@@ -1,960 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.child.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.future.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.lang.*;
-import org.apache.ignite.spi.*;
-import org.jdk8.backport.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.locks.*;
-
-import static org.apache.ignite.internal.processors.hadoop.taskexecutor.GridHadoopTaskState.*;
-
-/**
- * External process registry. Handles external process lifecycle.
- */
-public class GridHadoopExternalTaskExecutor extends GridHadoopTaskExecutorAdapter {
- /** Hadoop context. */
- private GridHadoopContext ctx;
-
- /** */
- private String javaCmd;
-
- /** Logger. */
- private IgniteLogger log;
-
- /** Node process descriptor. */
- private GridHadoopProcessDescriptor nodeDesc;
-
- /** Output base. */
- private File outputBase;
-
- /** Path separator. */
- private String pathSep;
-
- /** Hadoop external communication. */
- private GridHadoopExternalCommunication comm;
-
- /** Starting processes. */
- private final ConcurrentMap<UUID, HadoopProcess> runningProcsByProcId = new ConcurrentHashMap8<>();
-
- /** Starting processes. */
- private final ConcurrentMap<GridHadoopJobId, HadoopProcess> runningProcsByJobId = new ConcurrentHashMap8<>();
-
- /** Busy lock. */
- private final GridSpinReadWriteLock busyLock = new GridSpinReadWriteLock();
-
- /** Job tracker. */
- private GridHadoopJobTracker jobTracker;
-
- /** {@inheritDoc} */
- @Override public void start(GridHadoopContext ctx) throws IgniteCheckedException {
- this.ctx = ctx;
-
- log = ctx.kernalContext().log(GridHadoopExternalTaskExecutor.class);
-
- outputBase = U.resolveWorkDirectory("hadoop", false);
-
- pathSep = System.getProperty("path.separator", U.isWindows() ? ";" : ":");
-
- initJavaCommand();
-
- comm = new GridHadoopExternalCommunication(
- ctx.localNodeId(),
- UUID.randomUUID(),
- ctx.kernalContext().config().getMarshaller(),
- log,
- ctx.kernalContext().getSystemExecutorService(),
- ctx.kernalContext().gridName());
-
- comm.setListener(new MessageListener());
-
- comm.start();
-
- nodeDesc = comm.localProcessDescriptor();
-
- ctx.kernalContext().ports().registerPort(nodeDesc.tcpPort(), IgnitePortProtocol.TCP,
- GridHadoopExternalTaskExecutor.class);
-
- if (nodeDesc.sharedMemoryPort() != -1)
- ctx.kernalContext().ports().registerPort(nodeDesc.sharedMemoryPort(), IgnitePortProtocol.TCP,
- GridHadoopExternalTaskExecutor.class);
-
- jobTracker = ctx.jobTracker();
- }
-
- /** {@inheritDoc} */
- @Override public void stop(boolean cancel) {
- busyLock.writeLock();
-
- try {
- comm.stop();
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to gracefully stop external hadoop communication server (will shutdown anyway)", e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void onJobStateChanged(final GridHadoopJobMetadata meta) {
- final HadoopProcess proc = runningProcsByJobId.get(meta.jobId());
-
- // If we have a local process for this job.
- if (proc != null) {
- if (log.isDebugEnabled())
- log.debug("Updating job information for remote task process [proc=" + proc + ", meta=" + meta + ']');
-
- if (meta.phase() == GridHadoopJobPhase.PHASE_COMPLETE) {
- if (log.isDebugEnabled())
- log.debug("Completed job execution, will terminate child process [jobId=" + meta.jobId() +
- ", proc=" + proc + ']');
-
- runningProcsByJobId.remove(meta.jobId());
- runningProcsByProcId.remove(proc.descriptor().processId());
-
- proc.terminate();
-
- return;
- }
-
- if (proc.initFut.isDone()) {
- if (!proc.initFut.isFailed())
- sendJobInfoUpdate(proc, meta);
- else if (log.isDebugEnabled())
- log.debug("Failed to initialize child process (will skip job state notification) " +
- "[jobId=" + meta.jobId() + ", meta=" + meta + ']');
- }
- else {
- proc.initFut.listenAsync(new CI1<IgniteInternalFuture<IgniteBiTuple<Process, GridHadoopProcessDescriptor>>>() {
- @Override public void apply(IgniteInternalFuture<IgniteBiTuple<Process, GridHadoopProcessDescriptor>> f) {
- try {
- f.get();
-
- sendJobInfoUpdate(proc, meta);
- }
- catch (IgniteCheckedException e) {
- if (log.isDebugEnabled())
- log.debug("Failed to initialize child process (will skip job state notification) " +
- "[jobId=" + meta.jobId() + ", meta=" + meta + ", err=" + e + ']');
- }
-
- }
- });
- }
- }
- else if (ctx.isParticipating(meta)) {
- GridHadoopJob job;
-
- try {
- job = jobTracker.job(meta.jobId(), meta.jobInfo());
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to get job: " + meta.jobId(), e);
-
- return;
- }
-
- startProcess(job, meta.mapReducePlan());
- }
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("ConstantConditions")
- @Override public void run(final GridHadoopJob job, final Collection<GridHadoopTaskInfo> tasks) throws IgniteCheckedException {
- if (!busyLock.tryReadLock()) {
- if (log.isDebugEnabled())
- log.debug("Failed to start hadoop tasks (grid is stopping, will ignore).");
-
- return;
- }
-
- try {
- HadoopProcess proc = runningProcsByJobId.get(job.id());
-
- GridHadoopTaskType taskType = F.first(tasks).type();
-
- if (taskType == GridHadoopTaskType.SETUP || taskType == GridHadoopTaskType.ABORT ||
- taskType == GridHadoopTaskType.COMMIT) {
- if (proc == null || proc.terminated()) {
- runningProcsByJobId.remove(job.id(), proc);
-
- // Start new process for ABORT task since previous processes were killed.
- proc = startProcess(job, jobTracker.plan(job.id()));
-
- if (log.isDebugEnabled())
- log.debug("Starting new process for maintenance task [jobId=" + job.id() +
- ", proc=" + proc + ", taskType=" + taskType + ']');
- }
- }
- else
- assert proc != null : "Missing started process for task execution request: " + job.id() +
- ", tasks=" + tasks;
-
- final HadoopProcess proc0 = proc;
-
- proc.initFut.listenAsync(new CI1<IgniteInternalFuture<IgniteBiTuple<Process, GridHadoopProcessDescriptor>>>() {
- @Override public void apply(
- IgniteInternalFuture<IgniteBiTuple<Process, GridHadoopProcessDescriptor>> f) {
- if (!busyLock.tryReadLock())
- return;
-
- try {
- f.get();
-
- proc0.addTasks(tasks);
-
- if (log.isDebugEnabled())
- log.debug("Sending task execution request to child process [jobId=" + job.id() +
- ", proc=" + proc0 + ", tasks=" + tasks + ']');
-
- sendExecutionRequest(proc0, job, tasks);
- }
- catch (IgniteCheckedException e) {
- notifyTasksFailed(tasks, FAILED, e);
- }
- finally {
- busyLock.readUnlock();
- }
- }
- });
- }
- finally {
- busyLock.readUnlock();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void cancelTasks(GridHadoopJobId jobId) {
- HadoopProcess proc = runningProcsByJobId.get(jobId);
-
- if (proc != null)
- proc.terminate();
- }
-
- /**
- * Sends execution request to remote node.
- *
- * @param proc Process to send request to.
- * @param job Job instance.
- * @param tasks Collection of tasks to execute in started process.
- */
- private void sendExecutionRequest(HadoopProcess proc, GridHadoopJob job, Collection<GridHadoopTaskInfo> tasks)
- throws IgniteCheckedException {
- // Must synchronize since concurrent process crash may happen and will receive onConnectionLost().
- proc.lock();
-
- try {
- if (proc.terminated()) {
- notifyTasksFailed(tasks, CRASHED, null);
-
- return;
- }
-
- GridHadoopTaskExecutionRequest req = new GridHadoopTaskExecutionRequest();
-
- req.jobId(job.id());
- req.jobInfo(job.info());
- req.tasks(tasks);
-
- comm.sendMessage(proc.descriptor(), req);
- }
- finally {
- proc.unlock();
- }
- }
-
- /**
- * @return External task metadata.
- */
- private GridHadoopExternalTaskMetadata buildTaskMeta() {
- GridHadoopExternalTaskMetadata meta = new GridHadoopExternalTaskMetadata();
-
- meta.classpath(Arrays.asList(System.getProperty("java.class.path").split(File.pathSeparator)));
- meta.jvmOptions(Arrays.asList("-Xmx1g", "-ea", "-XX:+UseConcMarkSweepGC", "-XX:+CMSClassUnloadingEnabled",
- "-DIGNITE_HOME=" + U.getIgniteHome()));
-
- return meta;
- }
-
- /**
- * @param tasks Tasks to notify about.
- * @param state Fail state.
- * @param e Optional error.
- */
- private void notifyTasksFailed(Iterable<GridHadoopTaskInfo> tasks, GridHadoopTaskState state, Throwable e) {
- GridHadoopTaskStatus fail = new GridHadoopTaskStatus(state, e);
-
- for (GridHadoopTaskInfo task : tasks)
- jobTracker.onTaskFinished(task, fail);
- }
-
- /**
- * Starts process template that will be ready to execute Hadoop tasks.
- *
- * @param job Job instance.
- * @param plan Map reduce plan.
- */
- private HadoopProcess startProcess(final GridHadoopJob job, final GridHadoopMapReducePlan plan) {
- final UUID childProcId = UUID.randomUUID();
-
- GridHadoopJobId jobId = job.id();
-
- final GridHadoopProcessFuture fut = new GridHadoopProcessFuture(childProcId, jobId, ctx.kernalContext());
-
- final HadoopProcess proc = new HadoopProcess(jobId, fut, plan.reducers(ctx.localNodeId()));
-
- HadoopProcess old = runningProcsByJobId.put(jobId, proc);
-
- assert old == null;
-
- old = runningProcsByProcId.put(childProcId, proc);
-
- assert old == null;
-
- ctx.kernalContext().closure().runLocalSafe(new Runnable() {
- @Override public void run() {
- if (!busyLock.tryReadLock()) {
- fut.onDone(new IgniteCheckedException("Failed to start external process (grid is stopping)."));
-
- return;
- }
-
- try {
- GridHadoopExternalTaskMetadata startMeta = buildTaskMeta();
-
- if (log.isDebugEnabled())
- log.debug("Created hadoop child process metadata for job [job=" + job +
- ", childProcId=" + childProcId + ", taskMeta=" + startMeta + ']');
-
- Process proc = startJavaProcess(childProcId, startMeta, job);
-
- BufferedReader rdr = new BufferedReader(new InputStreamReader(proc.getInputStream()));
-
- String line;
-
- // Read up all the process output.
- while ((line = rdr.readLine()) != null) {
- if (log.isDebugEnabled())
- log.debug("Tracing process output: " + line);
-
- if ("Started".equals(line)) {
- // Process started successfully, it should not write anything more to the output stream.
- if (log.isDebugEnabled())
- log.debug("Successfully started child process [childProcId=" + childProcId +
- ", meta=" + job + ']');
-
- fut.onProcessStarted(proc);
-
- break;
- }
- else if ("Failed".equals(line)) {
- StringBuilder sb = new StringBuilder("Failed to start child process: " + job + "\n");
-
- while ((line = rdr.readLine()) != null)
- sb.append(" ").append(line).append("\n");
-
- // Cut last character.
- sb.setLength(sb.length() - 1);
-
- log.warning(sb.toString());
-
- fut.onDone(new IgniteCheckedException(sb.toString()));
-
- break;
- }
- }
- }
- catch (Throwable e) {
- fut.onDone(new IgniteCheckedException("Failed to initialize child process: " + job, e));
- }
- finally {
- busyLock.readUnlock();
- }
- }
- }, true);
-
- fut.listenAsync(new CI1<IgniteInternalFuture<IgniteBiTuple<Process, GridHadoopProcessDescriptor>>>() {
- @Override public void apply(IgniteInternalFuture<IgniteBiTuple<Process, GridHadoopProcessDescriptor>> f) {
- try {
- // Make sure there were no exceptions.
- f.get();
-
- prepareForJob(proc, job, plan);
- }
- catch (IgniteCheckedException ignore) {
- // Exception is printed in future's onDone() method.
- }
- }
- });
-
- return proc;
- }
-
- /**
- * Checks that java local command is available.
- *
- * @throws IgniteCheckedException If initialization failed.
- */
- private void initJavaCommand() throws IgniteCheckedException {
- String javaHome = System.getProperty("java.home");
-
- if (javaHome == null)
- javaHome = System.getenv("JAVA_HOME");
-
- if (javaHome == null)
- throw new IgniteCheckedException("Failed to locate JAVA_HOME.");
-
- javaCmd = javaHome + File.separator + "bin" + File.separator + (U.isWindows() ? "java.exe" : "java");
-
- try {
- Process proc = new ProcessBuilder(javaCmd, "-version").redirectErrorStream(true).start();
-
- Collection<String> out = readProcessOutput(proc);
-
- int res = proc.waitFor();
-
- if (res != 0)
- throw new IgniteCheckedException("Failed to execute 'java -version' command (process finished with nonzero " +
- "code) [exitCode=" + res + ", javaCmd='" + javaCmd + "', msg=" + F.first(out) + ']');
-
- if (log.isInfoEnabled()) {
- log.info("Will use java for external task execution: ");
-
- for (String s : out)
- log.info(" " + s);
- }
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to check java for external task execution.", e);
- }
- catch (InterruptedException e) {
- Thread.currentThread().interrupt();
-
- throw new IgniteCheckedException("Failed to wait for process completion (thread got interrupted).", e);
- }
- }
-
- /**
- * Reads process output line-by-line.
- *
- * @param proc Process to read output.
- * @return Read lines.
- * @throws IOException If read failed.
- */
- private Collection<String> readProcessOutput(Process proc) throws IOException {
- BufferedReader rdr = new BufferedReader(new InputStreamReader(proc.getInputStream()));
-
- Collection<String> res = new ArrayList<>();
-
- String s;
-
- while ((s = rdr.readLine()) != null)
- res.add(s);
-
- return res;
- }
-
- /**
- * Builds process from metadata.
- *
- * @param childProcId Child process ID.
- * @param startMeta Metadata.
- * @param job Job.
- * @return Started process.
- */
- private Process startJavaProcess(UUID childProcId, GridHadoopExternalTaskMetadata startMeta,
- GridHadoopJob job) throws Exception {
- String outFldr = jobWorkFolder(job.id()) + File.separator + childProcId;
-
- if (log.isDebugEnabled())
- log.debug("Will write process log output to: " + outFldr);
-
- List<String> cmd = new ArrayList<>();
-
- File workDir = U.resolveWorkDirectory("", false);
-
- cmd.add(javaCmd);
- cmd.addAll(startMeta.jvmOptions());
- cmd.add("-cp");
- cmd.add(buildClasspath(startMeta.classpath()));
- cmd.add(GridHadoopExternalProcessStarter.class.getName());
- cmd.add("-cpid");
- cmd.add(String.valueOf(childProcId));
- cmd.add("-ppid");
- cmd.add(String.valueOf(nodeDesc.processId()));
- cmd.add("-nid");
- cmd.add(String.valueOf(nodeDesc.parentNodeId()));
- cmd.add("-addr");
- cmd.add(nodeDesc.address());
- cmd.add("-tport");
- cmd.add(String.valueOf(nodeDesc.tcpPort()));
- cmd.add("-sport");
- cmd.add(String.valueOf(nodeDesc.sharedMemoryPort()));
- cmd.add("-out");
- cmd.add(outFldr);
- cmd.add("-wd");
- cmd.add(workDir.getAbsolutePath());
-
- return new ProcessBuilder(cmd)
- .redirectErrorStream(true)
- .directory(workDir)
- .start();
- }
-
- /**
- * Gets job work folder.
- *
- * @param jobId Job ID.
- * @return Job work folder.
- */
- private String jobWorkFolder(GridHadoopJobId jobId) {
- return outputBase + File.separator + "Job_" + jobId;
- }
-
- /**
- * @param cp Classpath collection.
- * @return Classpath string.
- */
- private String buildClasspath(Collection<String> cp) {
- assert !cp.isEmpty();
-
- StringBuilder sb = new StringBuilder();
-
- for (String s : cp)
- sb.append(s).append(pathSep);
-
- sb.setLength(sb.length() - 1);
-
- return sb.toString();
- }
-
- /**
- * Sends job info update request to remote process.
- *
- * @param proc Process to send request to.
- * @param meta Job metadata.
- */
- private void sendJobInfoUpdate(HadoopProcess proc, GridHadoopJobMetadata meta) {
- Map<Integer, GridHadoopProcessDescriptor> rdcAddrs = meta.reducersAddresses();
-
- int rdcNum = meta.mapReducePlan().reducers();
-
- GridHadoopProcessDescriptor[] addrs = null;
-
- if (rdcAddrs != null && rdcAddrs.size() == rdcNum) {
- addrs = new GridHadoopProcessDescriptor[rdcNum];
-
- for (int i = 0; i < rdcNum; i++) {
- GridHadoopProcessDescriptor desc = rdcAddrs.get(i);
-
- assert desc != null : "Missing reducing address [meta=" + meta + ", rdc=" + i + ']';
-
- addrs[i] = desc;
- }
- }
-
- try {
- comm.sendMessage(proc.descriptor(), new GridHadoopJobInfoUpdateRequest(proc.jobId, meta.phase(), addrs));
- }
- catch (IgniteCheckedException e) {
- if (!proc.terminated()) {
- log.error("Failed to send job state update message to remote child process (will kill the process) " +
- "[jobId=" + proc.jobId + ", meta=" + meta + ']', e);
-
- proc.terminate();
- }
- }
- }
-
- /**
- * Sends prepare request to remote process.
- *
- * @param proc Process to send request to.
- * @param job Job.
- * @param plan Map reduce plan.
- */
- private void prepareForJob(HadoopProcess proc, GridHadoopJob job, GridHadoopMapReducePlan plan) {
- try {
- comm.sendMessage(proc.descriptor(), new GridHadoopPrepareForJobRequest(job.id(), job.info(),
- plan.reducers(), plan.reducers(ctx.localNodeId())));
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to send job prepare request to remote process [proc=" + proc + ", job=" + job +
- ", plan=" + plan + ']', e);
-
- proc.terminate();
- }
- }
-
- /**
- * Processes task finished message.
- *
- * @param desc Remote process descriptor.
- * @param taskMsg Task finished message.
- */
- private void processTaskFinishedMessage(GridHadoopProcessDescriptor desc, GridHadoopTaskFinishedMessage taskMsg) {
- HadoopProcess proc = runningProcsByProcId.get(desc.processId());
-
- if (proc != null)
- proc.removeTask(taskMsg.taskInfo());
-
- jobTracker.onTaskFinished(taskMsg.taskInfo(), taskMsg.status());
- }
-
- /**
- *
- */
- private class MessageListener implements GridHadoopMessageListener {
- /** {@inheritDoc} */
- @Override public void onMessageReceived(GridHadoopProcessDescriptor desc, GridHadoopMessage msg) {
- if (!busyLock.tryReadLock())
- return;
-
- try {
- if (msg instanceof GridHadoopProcessStartedAck) {
- HadoopProcess proc = runningProcsByProcId.get(desc.processId());
-
- assert proc != null : "Missing child process for processId: " + desc;
-
- GridHadoopProcessFuture fut = proc.initFut;
-
- if (fut != null)
- fut.onReplyReceived(desc);
- // Safety.
- else
- log.warning("Failed to find process start future (will ignore): " + desc);
- }
- else if (msg instanceof GridHadoopTaskFinishedMessage) {
- GridHadoopTaskFinishedMessage taskMsg = (GridHadoopTaskFinishedMessage)msg;
-
- processTaskFinishedMessage(desc, taskMsg);
- }
- else
- log.warning("Unexpected message received by node [desc=" + desc + ", msg=" + msg + ']');
- }
- finally {
- busyLock.readUnlock();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void onConnectionLost(GridHadoopProcessDescriptor desc) {
- if (!busyLock.tryReadLock())
- return;
-
- try {
- if (desc == null) {
- U.warn(log, "Handshake failed.");
-
- return;
- }
-
- // Notify job tracker about failed tasks.
- HadoopProcess proc = runningProcsByProcId.get(desc.processId());
-
- if (proc != null) {
- Collection<GridHadoopTaskInfo> tasks = proc.tasks();
-
- if (!F.isEmpty(tasks)) {
- log.warning("Lost connection with alive process (will terminate): " + desc);
-
- GridHadoopTaskStatus status = new GridHadoopTaskStatus(CRASHED,
- new IgniteCheckedException("Failed to run tasks (external process finished unexpectedly): " + desc));
-
- for (GridHadoopTaskInfo info : tasks)
- jobTracker.onTaskFinished(info, status);
-
- runningProcsByJobId.remove(proc.jobId(), proc);
- }
-
- // Safety.
- proc.terminate();
- }
- }
- finally {
- busyLock.readUnlock();
- }
- }
- }
-
- /**
- * Hadoop process.
- */
- private static class HadoopProcess extends ReentrantLock {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Job ID. */
- private final GridHadoopJobId jobId;
-
- /** Process. */
- private Process proc;
-
- /** Init future. Completes when process is ready to receive messages. */
- private final GridHadoopProcessFuture initFut;
-
- /** Process descriptor. */
- private GridHadoopProcessDescriptor procDesc;
-
- /** Reducers planned for this process. */
- private Collection<Integer> reducers;
-
- /** Tasks. */
- private final Collection<GridHadoopTaskInfo> tasks = new ConcurrentLinkedDeque8<>();
-
- /** Terminated flag. */
- private volatile boolean terminated;
-
- /**
- * @param jobId Job ID.
- * @param initFut Init future.
- */
- private HadoopProcess(GridHadoopJobId jobId, GridHadoopProcessFuture initFut,
- int[] reducers) {
- this.jobId = jobId;
- this.initFut = initFut;
-
- if (!F.isEmpty(reducers)) {
- this.reducers = new ArrayList<>(reducers.length);
-
- for (int r : reducers)
- this.reducers.add(r);
- }
- }
-
- /**
- * @return Communication process descriptor.
- */
- private GridHadoopProcessDescriptor descriptor() {
- return procDesc;
- }
-
- /**
- * @return Job ID.
- */
- public GridHadoopJobId jobId() {
- return jobId;
- }
-
- /**
- * Initialized callback.
- *
- * @param proc Java process representation.
- * @param procDesc Process descriptor.
- */
- private void onInitialized(Process proc, GridHadoopProcessDescriptor procDesc) {
- this.proc = proc;
- this.procDesc = procDesc;
- }
-
- /**
- * Terminates process (kills it).
- */
- private void terminate() {
- // Guard against concurrent message sending.
- lock();
-
- try {
- terminated = true;
-
- if (!initFut.isDone())
- initFut.listenAsync(new CI1<IgniteInternalFuture<IgniteBiTuple<Process, GridHadoopProcessDescriptor>>>() {
- @Override public void apply(
- IgniteInternalFuture<IgniteBiTuple<Process, GridHadoopProcessDescriptor>> f) {
- proc.destroy();
- }
- });
- else
- proc.destroy();
- }
- finally {
- unlock();
- }
- }
-
- /**
- * @return Terminated flag.
- */
- private boolean terminated() {
- return terminated;
- }
-
- /**
- * Sets process tasks.
- *
- * @param tasks Tasks to set.
- */
- private void addTasks(Collection<GridHadoopTaskInfo> tasks) {
- this.tasks.addAll(tasks);
- }
-
- /**
- * Removes task when it was completed.
- *
- * @param task Task to remove.
- */
- private void removeTask(GridHadoopTaskInfo task) {
- if (tasks != null)
- tasks.remove(task);
- }
-
- /**
- * @return Collection of tasks.
- */
- private Collection<GridHadoopTaskInfo> tasks() {
- return tasks;
- }
-
- /**
- * @return Planned reducers.
- */
- private Collection<Integer> reducers() {
- return reducers;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(HadoopProcess.class, this);
- }
- }
-
- /**
- *
- */
- private class GridHadoopProcessFuture extends GridFutureAdapter<IgniteBiTuple<Process, GridHadoopProcessDescriptor>> {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Child process ID. */
- private UUID childProcId;
-
- /** Job ID. */
- private GridHadoopJobId jobId;
-
- /** Process descriptor. */
- private GridHadoopProcessDescriptor desc;
-
- /** Running process. */
- private Process proc;
-
- /** Process started flag. */
- private volatile boolean procStarted;
-
- /** Reply received flag. */
- private volatile boolean replyReceived;
-
- /** Logger. */
- private final IgniteLogger log = GridHadoopExternalTaskExecutor.this.log;
-
- /**
- * Empty constructor.
- */
- public GridHadoopProcessFuture() {
- // No-op.
- }
-
- /**
- * @param ctx Kernal context.
- */
- private GridHadoopProcessFuture(UUID childProcId, GridHadoopJobId jobId, GridKernalContext ctx) {
- super(ctx);
-
- this.childProcId = childProcId;
- this.jobId = jobId;
- }
-
- /**
- * Process started callback.
- */
- public void onProcessStarted(Process proc) {
- this.proc = proc;
-
- procStarted = true;
-
- if (procStarted && replyReceived)
- onDone(F.t(proc, desc));
- }
-
- /**
- * Reply received callback.
- */
- public void onReplyReceived(GridHadoopProcessDescriptor desc) {
- assert childProcId.equals(desc.processId());
-
- this.desc = desc;
-
- replyReceived = true;
-
- if (procStarted && replyReceived)
- onDone(F.t(proc, desc));
- }
-
- /** {@inheritDoc} */
- @Override public boolean onDone(@Nullable IgniteBiTuple<Process, GridHadoopProcessDescriptor> res,
- @Nullable Throwable err) {
- if (err == null) {
- HadoopProcess proc = runningProcsByProcId.get(childProcId);
-
- assert proc != null;
-
- assert proc.initFut == this;
-
- proc.onInitialized(res.get1(), res.get2());
-
- if (!F.isEmpty(proc.reducers()))
- jobTracker.onExternalMappersInitialized(jobId, proc.reducers(), desc);
- }
- else {
- // Clean up since init failed.
- runningProcsByJobId.remove(jobId);
- runningProcsByProcId.remove(childProcId);
- }
-
- if (super.onDone(res, err)) {
- if (err == null) {
- if (log.isDebugEnabled())
- log.debug("Initialized child process for external task execution [jobId=" + jobId +
- ", desc=" + desc + ", initTime=" + duration() + ']');
- }
- else
- U.error(log, "Failed to initialize child process for external task execution [jobId=" + jobId +
- ", desc=" + desc + ']', err);
-
- return true;
- }
-
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopExternalTaskMetadata.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopExternalTaskMetadata.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopExternalTaskMetadata.java
deleted file mode 100644
index 39606bc..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopExternalTaskMetadata.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
-
-import org.apache.ignite.internal.util.tostring.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.util.*;
-
-/**
- * External task metadata (classpath, JVM options) needed to start external process execution.
- */
-public class GridHadoopExternalTaskMetadata {
- /** Process classpath. */
- private Collection<String> classpath;
-
- /** JVM options. */
- @GridToStringInclude
- private Collection<String> jvmOpts;
-
- /**
- * @return JVM Options.
- */
- public Collection<String> jvmOptions() {
- return jvmOpts;
- }
-
- /**
- * @param jvmOpts JVM options.
- */
- public void jvmOptions(Collection<String> jvmOpts) {
- this.jvmOpts = jvmOpts;
- }
-
- /**
- * @return Classpath.
- */
- public Collection<String> classpath() {
- return classpath;
- }
-
- /**
- * @param classpath Classpath.
- */
- public void classpath(Collection<String> classpath) {
- this.classpath = classpath;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopExternalTaskMetadata.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopJobInfoUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopJobInfoUpdateRequest.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopJobInfoUpdateRequest.java
deleted file mode 100644
index 2a7c7a8..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopJobInfoUpdateRequest.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.util.tostring.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-
-/**
- * Job info update request.
- */
-public class GridHadoopJobInfoUpdateRequest implements GridHadoopMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Job ID. */
- @GridToStringInclude
- private GridHadoopJobId jobId;
-
- /** Job phase. */
- @GridToStringInclude
- private GridHadoopJobPhase jobPhase;
-
- /** Reducers addresses. */
- @GridToStringInclude
- private GridHadoopProcessDescriptor[] reducersAddrs;
-
- /**
- * Constructor required by {@link Externalizable}.
- */
- public GridHadoopJobInfoUpdateRequest() {
- // No-op.
- }
-
- /**
- * @param jobId Job ID.
- * @param jobPhase Job phase.
- * @param reducersAddrs Reducers addresses.
- */
- public GridHadoopJobInfoUpdateRequest(GridHadoopJobId jobId, GridHadoopJobPhase jobPhase,
- GridHadoopProcessDescriptor[] reducersAddrs) {
- assert jobId != null;
-
- this.jobId = jobId;
- this.jobPhase = jobPhase;
- this.reducersAddrs = reducersAddrs;
- }
-
- /**
- * @return Job ID.
- */
- public GridHadoopJobId jobId() {
- return jobId;
- }
-
- /**
- * @return Job phase.
- */
- public GridHadoopJobPhase jobPhase() {
- return jobPhase;
- }
-
- /**
- * @return Reducers addresses.
- */
- public GridHadoopProcessDescriptor[] reducersAddresses() {
- return reducersAddrs;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- jobId.writeExternal(out);
-
- out.writeObject(jobPhase);
- U.writeArray(out, reducersAddrs);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- jobId = new GridHadoopJobId();
- jobId.readExternal(in);
-
- jobPhase = (GridHadoopJobPhase)in.readObject();
- reducersAddrs = (GridHadoopProcessDescriptor[])U.readArray(in);
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopJobInfoUpdateRequest.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopPrepareForJobRequest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopPrepareForJobRequest.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopPrepareForJobRequest.java
deleted file mode 100644
index 3a55d19..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopPrepareForJobRequest.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.util.tostring.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-
-/**
- * Child process initialization request.
- */
-public class GridHadoopPrepareForJobRequest implements GridHadoopMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Job ID. */
- @GridToStringInclude
- private GridHadoopJobId jobId;
-
- /** Job info. */
- @GridToStringInclude
- private GridHadoopJobInfo jobInfo;
-
- /** Total amount of reducers in the job. */
- @GridToStringInclude
- private int totalReducersCnt;
-
- /** Reducers to be executed on current node. */
- @GridToStringInclude
- private int[] locReducers;
-
- /**
- * Constructor required by {@link Externalizable}.
- */
- public GridHadoopPrepareForJobRequest() {
- // No-op.
- }
-
- /**
- * @param jobId Job ID.
- * @param jobInfo Job info.
- * @param totalReducersCnt Number of reducers in the job.
- * @param locReducers Reducers to be executed on current node.
- */
- public GridHadoopPrepareForJobRequest(GridHadoopJobId jobId, GridHadoopJobInfo jobInfo, int totalReducersCnt,
- int[] locReducers) {
- assert jobId != null;
-
- this.jobId = jobId;
- this.jobInfo = jobInfo;
- this.totalReducersCnt = totalReducersCnt;
- this.locReducers = locReducers;
- }
-
- /**
- * @return Job info.
- */
- public GridHadoopJobInfo jobInfo() {
- return jobInfo;
- }
-
- /**
- * @return Job ID.
- */
- public GridHadoopJobId jobId() {
- return jobId;
- }
-
- /**
- * @return Reducers to be executed on current node.
- */
- public int[] localReducers() {
- return locReducers;
- }
-
- /**
- * @return Number of reducers in job.
- */
- public int totalReducerCount() {
- return totalReducersCnt;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- jobId.writeExternal(out);
-
- out.writeObject(jobInfo);
- out.writeInt(totalReducersCnt);
-
- U.writeIntArray(out, locReducers);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- jobId = new GridHadoopJobId();
- jobId.readExternal(in);
-
- jobInfo = (GridHadoopJobInfo)in.readObject();
- totalReducersCnt = in.readInt();
-
- locReducers = U.readIntArray(in);
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopPrepareForJobRequest.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopProcessDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopProcessDescriptor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopProcessDescriptor.java
deleted file mode 100644
index 7fc8858..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopProcessDescriptor.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
-
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Process descriptor used to identify process for which task is running.
- */
-public class GridHadoopProcessDescriptor implements Serializable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Parent node ID. */
- private UUID parentNodeId;
-
- /** Process ID. */
- private UUID procId;
-
- /** Address. */
- private String addr;
-
- /** TCP port. */
- private int tcpPort;
-
- /** Shared memory port. */
- private int shmemPort;
-
- /**
- * @param parentNodeId Parent node ID.
- * @param procId Process ID.
- */
- public GridHadoopProcessDescriptor(UUID parentNodeId, UUID procId) {
- this.parentNodeId = parentNodeId;
- this.procId = procId;
- }
-
- /**
- * Gets process ID.
- *
- * @return Process ID.
- */
- public UUID processId() {
- return procId;
- }
-
- /**
- * Gets parent node ID.
- *
- * @return Parent node ID.
- */
- public UUID parentNodeId() {
- return parentNodeId;
- }
-
- /**
- * Gets host address.
- *
- * @return Host address.
- */
- public String address() {
- return addr;
- }
-
- /**
- * Sets host address.
- *
- * @param addr Host address.
- */
- public void address(String addr) {
- this.addr = addr;
- }
-
- /**
- * @return Shared memory port.
- */
- public int sharedMemoryPort() {
- return shmemPort;
- }
-
- /**
- * Sets shared memory port.
- *
- * @param shmemPort Shared memory port.
- */
- public void sharedMemoryPort(int shmemPort) {
- this.shmemPort = shmemPort;
- }
-
- /**
- * @return TCP port.
- */
- public int tcpPort() {
- return tcpPort;
- }
-
- /**
- * Sets TCP port.
- *
- * @param tcpPort TCP port.
- */
- public void tcpPort(int tcpPort) {
- this.tcpPort = tcpPort;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (!(o instanceof GridHadoopProcessDescriptor))
- return false;
-
- GridHadoopProcessDescriptor that = (GridHadoopProcessDescriptor)o;
-
- return parentNodeId.equals(that.parentNodeId) && procId.equals(that.procId);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int result = parentNodeId.hashCode();
-
- result = 31 * result + procId.hashCode();
-
- return result;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopProcessDescriptor.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopProcessStartedAck.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopProcessStartedAck.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopProcessStartedAck.java
deleted file mode 100644
index 679da6c..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopProcessStartedAck.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
-
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-
-/**
- * Process started message.
- */
-public class GridHadoopProcessStartedAck implements GridHadoopMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopProcessStartedAck.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopTaskExecutionRequest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopTaskExecutionRequest.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopTaskExecutionRequest.java
deleted file mode 100644
index 9f11e0e..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopTaskExecutionRequest.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.util.tostring.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Message sent from node to child process to start task(s) execution.
- */
-public class GridHadoopTaskExecutionRequest implements GridHadoopMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Job ID. */
- @GridToStringInclude
- private GridHadoopJobId jobId;
-
- /** Job info. */
- @GridToStringInclude
- private GridHadoopJobInfo jobInfo;
-
- /** Mappers. */
- @GridToStringInclude
- private Collection<GridHadoopTaskInfo> tasks;
-
- /**
- * @return Job ID.
- */
- public GridHadoopJobId jobId() {
- return jobId;
- }
-
- /**
- * @param jobId Job ID.
- */
- public void jobId(GridHadoopJobId jobId) {
- this.jobId = jobId;
- }
-
- /**
- * @return Jon info.
- */
- public GridHadoopJobInfo jobInfo() {
- return jobInfo;
- }
-
- /**
- * @param jobInfo Job info.
- */
- public void jobInfo(GridHadoopJobInfo jobInfo) {
- this.jobInfo = jobInfo;
- }
-
- /**
- * @return Tasks.
- */
- public Collection<GridHadoopTaskInfo> tasks() {
- return tasks;
- }
-
- /**
- * @param tasks Tasks.
- */
- public void tasks(Collection<GridHadoopTaskInfo> tasks) {
- this.tasks = tasks;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopTaskExecutionRequest.class, this);
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- jobId.writeExternal(out);
-
- out.writeObject(jobInfo);
- U.writeCollection(out, tasks);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- jobId = new GridHadoopJobId();
- jobId.readExternal(in);
-
- jobInfo = (GridHadoopJobInfo)in.readObject();
- tasks = U.readCollection(in);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopTaskFinishedMessage.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopTaskFinishedMessage.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopTaskFinishedMessage.java
deleted file mode 100644
index f69abaf..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/GridHadoopTaskFinishedMessage.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor.external;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.message.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-
-/**
- * Task finished message. Sent when local task finishes execution.
- */
-public class GridHadoopTaskFinishedMessage implements GridHadoopMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Finished task info. */
- private GridHadoopTaskInfo taskInfo;
-
- /** Task finish status. */
- private GridHadoopTaskStatus status;
-
- /**
- * Constructor required by {@link Externalizable}.
- */
- public GridHadoopTaskFinishedMessage() {
- // No-op.
- }
-
- /**
- * @param taskInfo Finished task info.
- * @param status Task finish status.
- */
- public GridHadoopTaskFinishedMessage(GridHadoopTaskInfo taskInfo, GridHadoopTaskStatus status) {
- assert taskInfo != null;
- assert status != null;
-
- this.taskInfo = taskInfo;
- this.status = status;
- }
-
- /**
- * @return Finished task info.
- */
- public GridHadoopTaskInfo taskInfo() {
- return taskInfo;
- }
-
- /**
- * @return Task finish status.
- */
- public GridHadoopTaskStatus status() {
- return status;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopTaskFinishedMessage.class, this);
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- taskInfo.writeExternal(out);
- status.writeExternal(out);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- taskInfo = new GridHadoopTaskInfo();
- taskInfo.readExternal(in);
-
- status = new GridHadoopTaskStatus();
- status.readExternal(in);
- }
-}
[43/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/IgfsReader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsReader.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsReader.java
deleted file mode 100644
index fff8ca3..0000000
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsReader.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import java.io.*;
-
-/**
- * The simplest data input interface to read from secondary file system in dual modes.
- */
-public interface IgfsReader extends Closeable {
- /**
- * Read up to the specified number of bytes, from a given position within a file, and return the number of bytes
- * read.
- *
- * @param pos Position in the input stream to seek.
- * @param buf Buffer into which data is read.
- * @param off Offset in the buffer from which stream data should be written.
- * @param len The number of bytes to read.
- * @return Total number of bytes read into the buffer, or -1 if there is no more data (EOF).
- * @throws IOException In case of any exception.
- */
- public int read(long pos, byte[] buf, int off, int len) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java
index bf2636f..d7fd866 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java
@@ -32,7 +32,7 @@ import java.io.*;
*/
public abstract class IgfsInputStreamJobAdapter extends IgfsJobAdapter {
/** {@inheritDoc} */
- @Override public final Object execute(IgniteFs igfs, IgfsFileRange range, IgfsInputStream in)
+ @Override public final Object execute(IgniteFileSystem igfs, IgfsFileRange range, IgfsInputStream in)
throws IgniteException, IOException {
in.seek(range.start());
@@ -48,5 +48,5 @@ public abstract class IgfsInputStreamJobAdapter extends IgfsJobAdapter {
* @throws IgniteException If execution failed.
* @throws IOException If IO exception encountered while working with stream.
*/
- public abstract Object execute(IgniteFs igfs, IgfsRangeInputStream in) throws IgniteException, IOException;
+ public abstract Object execute(IgniteFileSystem igfs, IgfsRangeInputStream in) throws IgniteException, IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java
index d12fc79..0fdc0f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java
@@ -26,7 +26,7 @@ import java.io.*;
* Defines executable unit for {@link IgfsTask}. Before this job is executed, it is assigned one of the
* ranges provided by the {@link IgfsRecordResolver} passed to one of the {@code IgniteFs.execute(...)} methods.
* <p>
- * {@link #execute(org.apache.ignite.IgniteFs, IgfsFileRange, org.apache.ignite.igfs.IgfsInputStream)} method is given {@link IgfsFileRange} this
+ * {@link #execute(org.apache.ignite.IgniteFileSystem, IgfsFileRange, org.apache.ignite.igfs.IgfsInputStream)} method is given {@link IgfsFileRange} this
* job is expected to operate on, and already opened {@link org.apache.ignite.igfs.IgfsInputStream} for the file this range belongs to.
* <p>
* Note that provided input stream has position already adjusted to range start. However, it will not
@@ -52,7 +52,7 @@ public interface IgfsJob {
* @throws IgniteException If execution failed.
* @throws IOException If file system operation resulted in IO exception.
*/
- public Object execute(IgniteFs igfs, IgfsFileRange range, IgfsInputStream in) throws IgniteException,
+ public Object execute(IgniteFileSystem igfs, IgfsFileRange range, IgfsInputStream in) throws IgniteException,
IOException;
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java
index 52afeda..cc1d73f 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java
@@ -51,6 +51,6 @@ public interface IgfsRecordResolver extends Serializable {
* @throws IgniteException If resolve failed.
* @throws IOException If resolve failed.
*/
- @Nullable public IgfsFileRange resolveRecords(IgniteFs fs, IgfsInputStream stream,
+ @Nullable public IgfsFileRange resolveRecords(IgniteFileSystem fs, IgfsInputStream stream,
IgfsFileRange suggestedRecord) throws IgniteException, IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java
index e42b012..9936140 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java
@@ -37,7 +37,7 @@ import java.util.*;
* <p>
* Each file participating in IGFS task is split into {@link IgfsFileRange}s first. Normally range is a number of
* consequent bytes located on a single node (see {@code IgfssGroupDataBlocksKeyMapper}). In case maximum range size
- * is provided (either through {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} or {@code IgniteFs.execute()}
+ * is provided (either through {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()} or {@code IgniteFs.execute()}
* argument), then ranges could be further divided into smaller chunks.
* <p>
* Once file is split into ranges, each range is passed to {@code IgfsTask.createJob()} method in order to create a
@@ -88,7 +88,7 @@ public abstract class IgfsTask<T, R> extends ComputeTaskAdapter<IgfsTaskArgs<T>,
assert ignite != null;
assert args != null;
- IgniteFs fs = ignite.fileSystem(args.igfsName());
+ IgniteFileSystem fs = ignite.fileSystem(args.igfsName());
IgfsProcessorAdapter igfsProc = ((IgniteKernal) ignite).context().igfs();
Map<ComputeJob, ClusterNode> splitMap = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTaskArgs.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTaskArgs.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTaskArgs.java
index 7db26ec..5ef5352 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTaskArgs.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTaskArgs.java
@@ -33,7 +33,7 @@ import java.util.*;
* <li>{@link IgfsRecordResolver} for that task</li>
* <li>Flag indicating whether to skip non-existent file paths or throw an exception</li>
* <li>User-defined task argument</li>
- * <li>Maximum file range length for that task (see {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()})</li>
+ * <li>Maximum file range length for that task (see {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()})</li>
* </ul>
*/
public interface IgfsTaskArgs<T> {
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java
index c1c15a5..c9ed821 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java
@@ -20,7 +20,6 @@ package org.apache.ignite.igfs.mapreduce.records;
import org.apache.ignite.*;
import org.apache.ignite.igfs.*;
import org.apache.ignite.igfs.mapreduce.*;
-import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.util.tostring.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
@@ -78,7 +77,7 @@ public class IgfsByteDelimiterRecordResolver implements IgfsRecordResolver, Exte
}
/** {@inheritDoc} */
- @Override public IgfsFileRange resolveRecords(IgniteFs fs, IgfsInputStream stream,
+ @Override public IgfsFileRange resolveRecords(IgniteFileSystem fs, IgfsInputStream stream,
IgfsFileRange suggestedRecord) throws IgniteException, IOException {
long suggestedStart = suggestedRecord.start();
long suggestedEnd = suggestedStart + suggestedRecord.length();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java
index fe22627..bcc8f69 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java
@@ -52,7 +52,7 @@ public class IgfsFixedLengthRecordResolver implements IgfsRecordResolver, Extern
}
/** {@inheritDoc} */
- @Override public IgfsFileRange resolveRecords(IgniteFs fs, IgfsInputStream stream,
+ @Override public IgfsFileRange resolveRecords(IgniteFileSystem fs, IgfsInputStream stream,
IgfsFileRange suggestedRecord)
throws IgniteException, IOException {
long suggestedEnd = suggestedRecord.start() + suggestedRecord.length();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
new file mode 100644
index 0000000..089a8e3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs.secondary;
+
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Secondary file system interface.
+ */
+public interface IgfsSecondaryFileSystem {
+ /**
+ * Checks if the specified path exists.
+ *
+ * @param path Path to check for existence.
+ * @return {@code True} if such file exists, otherwise - {@code false}.
+ * @throws IgniteException In case of error.
+ */
+ public boolean exists(IgfsPath path);
+
+ /**
+ * Updates file information for the specified path. Existent properties, not listed in the passed collection,
+ * will not be affected. Other properties will be added or overwritten. Passed properties with {@code null} values
+ * will be removed from the stored properties or ignored if they don't exist in the file info.
+ * <p>
+ * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties will be propagated
+ * to the secondary file system:
+ * <ul>
+ * <li>{@code usrName} - file owner name;</li>
+ * <li>{@code grpName} - file owner group;</li>
+ * <li>{@code permission} - Unix-style string representing file permissions.</li>
+ * </ul>
+ *
+ * @param path File path to set properties for.
+ * @param props Properties to update.
+ * @return File information for specified path or {@code null} if such path does not exist.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteException;
+
+ /**
+ * Renames/moves a file.
+ * <p>
+ * You are free to rename/move data files as you wish, but directories can be only renamed.
+ * You cannot move the directory between different parent directories.
+ * <p>
+ * Examples:
+ * <ul>
+ * <li>"/work/file.txt" => "/home/project/Presentation Scenario.txt"</li>
+ * <li>"/work" => "/work-2012.bkp"</li>
+ * <li>"/work" => "<strike>/backups/work</strike>" - such operation is restricted for directories.</li>
+ * </ul>
+ *
+ * @param src Source file path to rename.
+ * @param dest Destination file path. If destination path is a directory, then source file will be placed
+ * into destination directory with original name.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If source file doesn't exist.
+ */
+ public void rename(IgfsPath src, IgfsPath dest) throws IgniteException;
+
+ /**
+ * Deletes file.
+ *
+ * @param path File path to delete.
+ * @param recursive Delete non-empty directories recursively.
+ * @return {@code True} in case of success, {@code false} otherwise.
+ * @throws IgniteException In case of error.
+ */
+ public boolean delete(IgfsPath path, boolean recursive) throws IgniteException;
+
+ /**
+ * Creates directories under specified path.
+ *
+ * @param path Path of directories chain to create.
+ * @throws IgniteException In case of error.
+ */
+ public void mkdirs(IgfsPath path) throws IgniteException;
+
+ /**
+ * Creates directories under specified path with the specified properties.
+ *
+ * @param path Path of directories chain to create.
+ * @param props Metadata properties to set on created directories.
+ * @throws IgniteException In case of error.
+ */
+ public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) throws IgniteException;
+
+ /**
+ * Lists file paths under the specified path.
+ *
+ * @param path Path to list files under.
+ * @return List of files under the specified path.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If path doesn't exist.
+ */
+ public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteException;
+
+ /**
+ * Lists files under the specified path.
+ *
+ * @param path Path to list files under.
+ * @return List of files under the specified path.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If path doesn't exist.
+ */
+ public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteException;
+
+ /**
+ * Opens a file for reading.
+ *
+ * @param path File path to read.
+ * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
+ * @return File input stream to read data from.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If path doesn't exist.
+ */
+ public IgfsSecondaryFileSystemPositionedReadable open(IgfsPath path, int bufSize) throws IgniteException;
+
+ /**
+ * Creates a file and opens it for writing.
+ *
+ * @param path File path to create.
+ * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
+ * @return File output stream to write data to.
+ * @throws IgniteException In case of error.
+ */
+ public OutputStream create(IgfsPath path, boolean overwrite) throws IgniteException;
+
+ /**
+ * Creates a file and opens it for writing.
+ *
+ * @param path File path to create.
+ * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
+ * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
+ * @param replication Replication factor.
+ * @param blockSize Block size.
+ * @param props File properties to set.
+ * @return File output stream to write data to.
+ * @throws IgniteException In case of error.
+ */
+ public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication, long blockSize,
+ @Nullable Map<String, String> props) throws IgniteException;
+
+ /**
+ * Opens an output stream to an existing file for appending data.
+ *
+ * @param path File path to append.
+ * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
+ * @param create Create file if it doesn't exist yet.
+ * @param props File properties to set only in case it file was just created.
+ * @return File output stream to append data to.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
+ */
+ public OutputStream append(IgfsPath path, int bufSize, boolean create, @Nullable Map<String, String> props)
+ throws IgniteException;
+
+ /**
+ * Gets file information for the specified path.
+ *
+ * @param path Path to get information for.
+ * @return File information for specified path or {@code null} if such path does not exist.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsFile info(IgfsPath path) throws IgniteException;
+
+ /**
+ * Gets used space in bytes.
+ *
+ * @return Used space in bytes.
+ * @throws IgniteException In case of error.
+ */
+ public long usedSpaceSize() throws IgniteException;
+
+ /**
+ * Gets the implementation specific properties of file system.
+ *
+ * @return Map of properties.
+ */
+ public Map<String,String> properties();
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemPositionedReadable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemPositionedReadable.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemPositionedReadable.java
new file mode 100644
index 0000000..3d36236
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemPositionedReadable.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs.secondary;
+
+import java.io.*;
+
+/**
+ * The simplest data input interface to read from secondary file system.
+ */
+public interface IgfsSecondaryFileSystemPositionedReadable extends Closeable {
+ /**
+ * Read up to the specified number of bytes, from a given position within a file, and return the number of bytes
+ * read.
+ *
+ * @param pos Position in the input stream to seek.
+ * @param buf Buffer into which data is read.
+ * @param off Offset in the buffer from which stream data should be written.
+ * @param len The number of bytes to read.
+ * @return Total number of bytes read into the buffer, or -1 if there is no more data (EOF).
+ * @throws IOException In case of any exception.
+ */
+ public int read(long pos, byte[] buf, int off, int len) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/igfs/secondary/package.html
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/package.html b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/package.html
new file mode 100644
index 0000000..8bd668d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/package.html
@@ -0,0 +1,24 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<body>
+ <!-- Package description. -->
+ Contains APIs for IGFS secondary file system.
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index cb9ffa1..30ba883 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -282,7 +282,7 @@ public interface GridKernalContext extends Iterable<GridComponent> {
*
* @return Hadoop processor.
*/
- public IgniteHadoopProcessorAdapter hadoop();
+ public HadoopProcessorAdapter hadoop();
/**
* Gets utility cache pool.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 756c16a..e80df0b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -230,7 +230,7 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
/** */
@GridToStringExclude
- private IgniteHadoopProcessorAdapter hadoopProc;
+ private HadoopProcessorAdapter hadoopProc;
/** */
@GridToStringExclude
@@ -456,8 +456,8 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
streamProc = (GridStreamProcessor)comp;
else if (comp instanceof GridContinuousProcessor)
contProc = (GridContinuousProcessor)comp;
- else if (comp instanceof IgniteHadoopProcessorAdapter)
- hadoopProc = (IgniteHadoopProcessorAdapter)comp;
+ else if (comp instanceof HadoopProcessorAdapter)
+ hadoopProc = (HadoopProcessorAdapter)comp;
else if (comp instanceof GridPortableProcessor)
portableProc = (GridPortableProcessor)comp;
else if (comp instanceof IgnitePluginProcessor)
@@ -680,7 +680,7 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
}
/** {@inheritDoc} */
- @Override public IgniteHadoopProcessorAdapter hadoop() {
+ @Override public HadoopProcessorAdapter hadoop() {
return hadoopProc;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java
index d0e487a..0e5c1cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java
@@ -35,8 +35,8 @@ public enum IgniteComponentType {
/** Hadoop. */
HADOOP(
- "org.apache.ignite.internal.processors.hadoop.IgniteHadoopNoopProcessor",
- "org.apache.ignite.internal.processors.hadoop.IgniteHadoopProcessor",
+ "org.apache.ignite.internal.processors.hadoop.HadoopNoopProcessor",
+ "org.apache.ignite.internal.processors.hadoop.HadoopProcessor",
"ignite-hadoop"
),
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java
index 3c35a08..3ede8d5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java
@@ -117,14 +117,14 @@ public interface IgniteEx extends Ignite {
* @param name IGFS name.
* @return IGFS.
*/
- @Nullable public IgniteFs igfsx(@Nullable String name);
+ @Nullable public IgniteFileSystem igfsx(@Nullable String name);
/**
* Get Hadoop facade.
*
* @return Hadoop.
*/
- public GridHadoop hadoop();
+ public Hadoop hadoop();
/** {@inheritDoc} */
@Override IgniteClusterEx cluster();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index cc4efa6..c6ea165 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -704,7 +704,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
// Starts lifecycle aware components.
U.startLifecycleAware(lifecycleAwares(cfg));
- addHelper(ctx, IGFS_HELPER.create(F.isEmpty(cfg.getIgfsConfiguration())));
+ addHelper(ctx, IGFS_HELPER.create(F.isEmpty(cfg.getFileSystemConfiguration())));
startProcessor(ctx, new IgnitePluginProcessor(ctx, cfg), attrs);
@@ -756,7 +756,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
startProcessor(ctx, new GridRestProcessor(ctx), attrs);
startProcessor(ctx, new GridDataLoaderProcessor(ctx), attrs);
startProcessor(ctx, new GridStreamProcessor(ctx), attrs);
- startProcessor(ctx, (GridProcessor) IGFS.create(ctx, F.isEmpty(cfg.getIgfsConfiguration())), attrs);
+ startProcessor(ctx, (GridProcessor) IGFS.create(ctx, F.isEmpty(cfg.getFileSystemConfiguration())), attrs);
startProcessor(ctx, new GridContinuousProcessor(ctx), attrs);
startProcessor(ctx, (GridProcessor)(cfg.isPeerClassLoadingEnabled() ?
IgniteComponentType.HADOOP.create(ctx, true): // No-op when peer class loading is enabled.
@@ -2349,11 +2349,11 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/** {@inheritDoc} */
- @Override public IgniteFs fileSystem(String name) {
+ @Override public IgniteFileSystem fileSystem(String name) {
guard();
try{
- IgniteFs fs = ctx.igfs().igfs(name);
+ IgniteFileSystem fs = ctx.igfs().igfs(name);
if (fs == null)
throw new IllegalArgumentException("IGFS is not configured: " + name);
@@ -2366,7 +2366,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/** {@inheritDoc} */
- @Nullable @Override public IgniteFs igfsx(@Nullable String name) {
+ @Nullable @Override public IgniteFileSystem igfsx(@Nullable String name) {
guard();
try {
@@ -2378,7 +2378,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/** {@inheritDoc} */
- @Override public Collection<IgniteFs> fileSystems() {
+ @Override public Collection<IgniteFileSystem> fileSystems() {
guard();
try {
@@ -2390,7 +2390,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/** {@inheritDoc} */
- @Override public GridHadoop hadoop() {
+ @Override public Hadoop hadoop() {
guard();
try {
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index cb2efbf..2e8cfc1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -1606,15 +1606,15 @@ public class IgnitionEx {
if (myCfg.getPeerClassLoadingLocalClassPathExclude() == null)
myCfg.setPeerClassLoadingLocalClassPathExclude(EMPTY_STR_ARR);
- IgfsConfiguration[] igfsCfgs = myCfg.getIgfsConfiguration();
+ FileSystemConfiguration[] igfsCfgs = myCfg.getFileSystemConfiguration();
if (igfsCfgs != null) {
- IgfsConfiguration[] clone = igfsCfgs.clone();
+ FileSystemConfiguration[] clone = igfsCfgs.clone();
for (int i = 0; i < igfsCfgs.length; i++)
- clone[i] = new IgfsConfiguration(igfsCfgs[i]);
+ clone[i] = new FileSystemConfiguration(igfsCfgs[i]);
- myCfg.setIgfsConfiguration(clone);
+ myCfg.setFileSystemConfiguration(clone);
}
StreamerConfiguration[] streamerCfgs = myCfg.getStreamerConfiguration();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 12ea535..fe88012 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -236,10 +236,10 @@ public abstract class GridCacheAdapter<K, V> implements GridCache<K, V>,
mxBean = new CacheMetricsMXBeanImpl(this);
- IgfsConfiguration[] igfsCfgs = gridCfg.getIgfsConfiguration();
+ FileSystemConfiguration[] igfsCfgs = gridCfg.getFileSystemConfiguration();
if (igfsCfgs != null) {
- for (IgfsConfiguration igfsCfg : igfsCfgs) {
+ for (FileSystemConfiguration igfsCfg : igfsCfgs) {
if (F.eq(ctx.name(), igfsCfg.getDataCacheName())) {
if (!ctx.isNear()) {
igfsDataCache = true;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 3b2ca47..72c2b49 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -550,10 +550,10 @@ public class GridCacheProcessor extends GridProcessorAdapter {
maxPreloadOrder = validatePreloadOrder(ctx.config().getCacheConfiguration());
// Internal caches which should not be returned to user.
- IgfsConfiguration[] igfsCfgs = ctx.grid().configuration().getIgfsConfiguration();
+ FileSystemConfiguration[] igfsCfgs = ctx.grid().configuration().getFileSystemConfiguration();
if (igfsCfgs != null) {
- for (IgfsConfiguration igfsCfg : igfsCfgs) {
+ for (FileSystemConfiguration igfsCfg : igfsCfgs) {
sysCaches.add(igfsCfg.getMetaCacheName());
sysCaches.add(igfsCfg.getDataCacheName());
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 1e67907..b07c14e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -1594,10 +1594,10 @@ public class GridCacheUtils {
* @return {@code True} in this is IGFS data or meta cache.
*/
public static boolean isIgfsCache(IgniteConfiguration cfg, @Nullable String cacheName) {
- IgfsConfiguration[] igfsCfgs = cfg.getIgfsConfiguration();
+ FileSystemConfiguration[] igfsCfgs = cfg.getFileSystemConfiguration();
if (igfsCfgs != null) {
- for (IgfsConfiguration igfsCfg : igfsCfgs) {
+ for (FileSystemConfiguration igfsCfg : igfsCfgs) {
// IGFS config probably has not been validated yet => possible NPE, so we check for null.
if (igfsCfg != null &&
(F.eq(cacheName, igfsCfg.getDataCacheName()) || F.eq(cacheName, igfsCfg.getMetaCacheName())))
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoop.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoop.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoop.java
deleted file mode 100644
index c262d48..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoop.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.*;
-import org.jetbrains.annotations.*;
-
-/**
- * Hadoop facade providing access to Ignite Hadoop features.
- */
-public interface GridHadoop {
- /**
- * Gets Hadoop module configuration.
- *
- * @return Hadoop module configuration.
- */
- public GridHadoopConfiguration configuration();
-
- /**
- * Generate next job ID.
- *
- * @return Next job ID.
- */
- public GridHadoopJobId nextJobId();
-
- /**
- * Submits job to job tracker.
- *
- * @param jobId Job ID to submit.
- * @param jobInfo Job info to submit.
- * @return Execution future.
- */
- public IgniteInternalFuture<?> submit(GridHadoopJobId jobId, GridHadoopJobInfo jobInfo);
-
- /**
- * Gets Hadoop job execution status.
- *
- * @param jobId Job ID to get status for.
- * @return Job execution status or {@code null} in case job with the given ID is not found.
- * @throws IgniteCheckedException If failed.
- */
- @Nullable public GridHadoopJobStatus status(GridHadoopJobId jobId) throws IgniteCheckedException;
-
- /**
- * Returns job counters.
- *
- * @param jobId Job ID to get counters for.
- * @return Job counters object.
- * @throws IgniteCheckedException If failed.
- */
- public GridHadoopCounters counters(GridHadoopJobId jobId) throws IgniteCheckedException;
-
- /**
- * Gets Hadoop finish future for particular job.
- *
- * @param jobId Job ID.
- * @return Job finish future or {@code null} in case job with the given ID is not found.
- * @throws IgniteCheckedException If failed.
- */
- @Nullable public IgniteInternalFuture<?> finishFuture(GridHadoopJobId jobId) throws IgniteCheckedException;
-
- /**
- * Kills job.
- *
- * @param jobId Job ID.
- * @return {@code True} if job was killed.
- * @throws IgniteCheckedException If failed.
- */
- public boolean kill(GridHadoopJobId jobId) throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopConfiguration.java
deleted file mode 100644
index f66b95a..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopConfiguration.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-/**
- * Hadoop configuration.
- */
-public class GridHadoopConfiguration {
- /** Default finished job info time-to-live. */
- public static final long DFLT_FINISHED_JOB_INFO_TTL = 10_000;
-
- /** Default value for external execution flag. */
- public static final boolean DFLT_EXTERNAL_EXECUTION = false;
-
- /** Default value for the max parallel tasks. */
- public static final int DFLT_MAX_PARALLEL_TASKS = Runtime.getRuntime().availableProcessors();
-
- /** Default value for the max task queue size. */
- public static final int DFLT_MAX_TASK_QUEUE_SIZE = 1000;
-
- /** Map reduce planner. */
- private GridHadoopMapReducePlanner planner;
-
- /** */
- private boolean extExecution = DFLT_EXTERNAL_EXECUTION;
-
- /** Finished job info TTL. */
- private long finishedJobInfoTtl = DFLT_FINISHED_JOB_INFO_TTL;
-
- /** */
- private int maxParallelTasks = DFLT_MAX_PARALLEL_TASKS;
-
- /** */
- private int maxTaskQueueSize = DFLT_MAX_TASK_QUEUE_SIZE;
-
- /**
- * Default constructor.
- */
- public GridHadoopConfiguration() {
- // No-op.
- }
-
- /**
- * Copy constructor.
- *
- * @param cfg Configuration to copy.
- */
- public GridHadoopConfiguration(GridHadoopConfiguration cfg) {
- // Preserve alphabetic order.
- extExecution = cfg.isExternalExecution();
- finishedJobInfoTtl = cfg.getFinishedJobInfoTtl();
- planner = cfg.getMapReducePlanner();
- maxParallelTasks = cfg.getMaxParallelTasks();
- maxTaskQueueSize = cfg.getMaxTaskQueueSize();
- }
-
- /**
- * Gets max number of local tasks that may be executed in parallel.
- *
- * @return Max number of local tasks that may be executed in parallel.
- */
- public int getMaxParallelTasks() {
- return maxParallelTasks;
- }
-
- /**
- * Sets max number of local tasks that may be executed in parallel.
- *
- * @param maxParallelTasks Max number of local tasks that may be executed in parallel.
- */
- public void setMaxParallelTasks(int maxParallelTasks) {
- this.maxParallelTasks = maxParallelTasks;
- }
-
- /**
- * Gets max task queue size.
- *
- * @return Max task queue size.
- */
- public int getMaxTaskQueueSize() {
- return maxTaskQueueSize;
- }
-
- /**
- * Sets max task queue size.
- *
- * @param maxTaskQueueSize Max task queue size.
- */
- public void setMaxTaskQueueSize(int maxTaskQueueSize) {
- this.maxTaskQueueSize = maxTaskQueueSize;
- }
-
- /**
- * Gets finished job info time-to-live in milliseconds.
- *
- * @return Finished job info time-to-live.
- */
- public long getFinishedJobInfoTtl() {
- return finishedJobInfoTtl;
- }
-
- /**
- * Sets finished job info time-to-live.
- *
- * @param finishedJobInfoTtl Finished job info time-to-live.
- */
- public void setFinishedJobInfoTtl(long finishedJobInfoTtl) {
- this.finishedJobInfoTtl = finishedJobInfoTtl;
- }
-
- /**
- * Gets external task execution flag. If {@code true}, hadoop job tasks will be executed in an external
- * (relative to node) process.
- *
- * @return {@code True} if external execution.
- */
- public boolean isExternalExecution() {
- return extExecution;
- }
-
- /**
- * Sets external task execution flag.
- *
- * @param extExecution {@code True} if tasks should be executed in an external process.
- * @see #isExternalExecution()
- */
- public void setExternalExecution(boolean extExecution) {
- this.extExecution = extExecution;
- }
-
- /**
- * Gets Hadoop map-reduce planner, a component which defines job execution plan based on job
- * configuration and current grid topology.
- *
- * @return Map-reduce planner.
- */
- public GridHadoopMapReducePlanner getMapReducePlanner() {
- return planner;
- }
-
- /**
- * Sets Hadoop map-reduce planner, a component which defines job execution plan based on job
- * configuration and current grid topology.
- *
- * @param planner Map-reduce planner.
- */
- public void setMapReducePlanner(GridHadoopMapReducePlanner planner) {
- this.planner = planner;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopConfiguration.class, this, super.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounter.java
deleted file mode 100644
index 83902dd..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounter.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-/**
- * Hadoop counter.
- */
-public interface GridHadoopCounter {
- /**
- * Gets name.
- *
- * @return Name of the counter.
- */
- public String name();
-
- /**
- * Gets counter group.
- *
- * @return Counter group's name.
- */
- public String group();
-
- /**
- * Merge the given counter to this counter.
- *
- * @param cntr Counter to merge into this counter.
- */
- public void merge(GridHadoopCounter cntr);
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounterWriter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounterWriter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounterWriter.java
deleted file mode 100644
index af72e69..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounterWriter.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-
-/**
- * The object that writes some system counters to some storage for each running job. This operation is a part of
- * whole statistics collection process.
- */
-public interface GridHadoopCounterWriter {
- /**
- * Writes counters of given job to some statistics storage.
- *
- * @param jobInfo Job info.
- * @param jobId Job id.
- * @param cntrs Counters.
- * @throws IgniteCheckedException If failed.
- */
- public void write(GridHadoopJobInfo jobInfo, GridHadoopJobId jobId, GridHadoopCounters cntrs) throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounters.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounters.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounters.java
deleted file mode 100644
index 91eb8a1..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounters.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import java.util.*;
-
-/**
- * Counters store.
- */
-public interface GridHadoopCounters {
- /**
- * Returns counter for the specified group and counter name. Creates new if it does not exist.
- *
- * @param grp Counter group name.
- * @param name Counter name.
- * @param cls Class for new instance creation if it's needed.
- * @return The counter that was found or added or {@code null} if create is false.
- */
- <T extends GridHadoopCounter> T counter(String grp, String name, Class<T> cls);
-
- /**
- * Returns all existing counters.
- *
- * @return Collection of counters.
- */
- Collection<GridHadoopCounter> all();
-
- /**
- * Merges all counters from another store with existing counters.
- *
- * @param other Counters to merge with.
- */
- void merge(GridHadoopCounters other);
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopFileBlock.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopFileBlock.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopFileBlock.java
deleted file mode 100644
index fae111a..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopFileBlock.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.internal.util.tostring.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-
-/**
- * Hadoop file block.
- */
-public class GridHadoopFileBlock extends GridHadoopInputSplit {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- @GridToStringInclude
- protected URI file;
-
- /** */
- @GridToStringInclude
- protected long start;
-
- /** */
- @GridToStringInclude
- protected long len;
-
- /**
- * Creates new file block.
- */
- public GridHadoopFileBlock() {
- // No-op.
- }
-
- /**
- * Creates new file block.
- *
- * @param hosts List of hosts where the block resides.
- * @param file File URI.
- * @param start Start position of the block in the file.
- * @param len Length of the block.
- */
- public GridHadoopFileBlock(String[] hosts, URI file, long start, long len) {
- A.notNull(hosts, "hosts", file, "file");
-
- this.hosts = hosts;
- this.file = file;
- this.start = start;
- this.len = len;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeObject(file());
- out.writeLong(start());
- out.writeLong(length());
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- file = (URI)in.readObject();
- start = in.readLong();
- len = in.readLong();
- }
-
- /**
- * @return Length.
- */
- public long length() {
- return len;
- }
-
- /**
- * @param len New length.
- */
- public void length(long len) {
- this.len = len;
- }
-
- /**
- * @return Start.
- */
- public long start() {
- return start;
- }
-
- /**
- * @param start New start.
- */
- public void start(long start) {
- this.start = start;
- }
-
- /**
- * @return File.
- */
- public URI file() {
- return file;
- }
-
- /**
- * @param file New file.
- */
- public void file(URI file) {
- this.file = file;
- }
-
- /**
- * @param hosts New hosts.
- */
- public void hosts(String[] hosts) {
- A.notNull(hosts, "hosts");
-
- this.hosts = hosts;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (!(o instanceof GridHadoopFileBlock))
- return false;
-
- GridHadoopFileBlock that = (GridHadoopFileBlock)o;
-
- return len == that.len && start == that.start && file.equals(that.file);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = file.hashCode();
-
- res = 31 * res + (int)(start ^ (start >>> 32));
- res = 31 * res + (int)(len ^ (len >>> 32));
-
- return res;
- }
-
- /** {@inheritDoc} */
- public String toString() {
- return S.toString(GridHadoopFileBlock.class, this, "hosts", Arrays.toString(hosts));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopInputSplit.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopInputSplit.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopInputSplit.java
deleted file mode 100644
index e68a6f5..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopInputSplit.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import java.io.*;
-
-/**
- * Abstract fragment of an input data source.
- */
-public abstract class GridHadoopInputSplit implements Externalizable {
- /** */
- protected String[] hosts;
-
- /**
- * Array of hosts where this input split resides.
- *
- * @return Hosts.
- */
- public String[] hosts() {
- assert hosts != null;
-
- return hosts;
- }
-
- /**
- * This method must be implemented for purpose of internal implementation.
- *
- * @param obj Another object.
- * @return {@code true} If objects are equal.
- */
- @Override public abstract boolean equals(Object obj);
-
- /**
- * This method must be implemented for purpose of internal implementation.
- *
- * @return Hash code of the object.
- */
- @Override public abstract int hashCode();
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJob.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJob.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJob.java
deleted file mode 100644
index f7ea105..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJob.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-
-import java.util.*;
-
-/**
- * Hadoop job.
- */
-public interface GridHadoopJob {
- /**
- * Gets job ID.
- *
- * @return Job ID.
- */
- public GridHadoopJobId id();
-
- /**
- * Gets job information.
- *
- * @return Job information.
- */
- public GridHadoopJobInfo info();
-
- /**
- * Gets collection of input splits for this job.
- *
- * @return Input splits.
- */
- public Collection<GridHadoopInputSplit> input() throws IgniteCheckedException;
-
- /**
- * Returns context for task execution.
- *
- * @param info Task info.
- * @return Task Context.
- * @throws IgniteCheckedException If failed.
- */
- public GridHadoopTaskContext getTaskContext(GridHadoopTaskInfo info) throws IgniteCheckedException;
-
- /**
- * Does all the needed initialization for the job. Will be called on each node where tasks for this job must
- * be executed.
- * <p>
- * If job is running in external mode this method will be called on instance in Ignite node with parameter
- * {@code false} and on instance in external process with parameter {@code true}.
- *
- * @param external If {@code true} then this job instance resides in external process.
- * @param locNodeId Local node ID.
- * @throws IgniteCheckedException If failed.
- */
- public void initialize(boolean external, UUID locNodeId) throws IgniteCheckedException;
-
- /**
- * Release all the resources.
- * <p>
- * If job is running in external mode this method will be called on instance in Ignite node with parameter
- * {@code false} and on instance in external process with parameter {@code true}.
- *
- * @param external If {@code true} then this job instance resides in external process.
- * @throws IgniteCheckedException If failed.
- */
- public void dispose(boolean external) throws IgniteCheckedException;
-
- /**
- * Prepare local environment for the task.
- *
- * @param info Task info.
- * @throws IgniteCheckedException If failed.
- */
- public void prepareTaskEnvironment(GridHadoopTaskInfo info) throws IgniteCheckedException;
-
- /**
- * Cleans up local environment of the task.
- *
- * @param info Task info.
- * @throws IgniteCheckedException If failed.
- */
- public void cleanupTaskEnvironment(GridHadoopTaskInfo info) throws IgniteCheckedException;
-
- /**
- * Cleans up the job staging directory.
- */
- void cleanupStagingDirectory();
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobId.java
deleted file mode 100644
index ffc2057..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobId.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.internal.processors.cache.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Job ID.
- */
-public class GridHadoopJobId implements GridCacheInternal, Externalizable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private UUID nodeId;
-
- /** */
- private int jobId;
-
- /**
- * For {@link Externalizable}.
- */
- public GridHadoopJobId() {
- // No-op.
- }
-
- /**
- * @param nodeId Node ID.
- * @param jobId Job ID.
- */
- public GridHadoopJobId(UUID nodeId, int jobId) {
- this.nodeId = nodeId;
- this.jobId = jobId;
- }
-
- public UUID globalId() {
- return nodeId;
- }
-
- public int localId() {
- return jobId;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- U.writeUuid(out, nodeId);
- out.writeInt(jobId);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- nodeId = U.readUuid(in);
- jobId = in.readInt();
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- GridHadoopJobId that = (GridHadoopJobId) o;
-
- if (jobId != that.jobId)
- return false;
-
- if (!nodeId.equals(that.nodeId))
- return false;
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return 31 * nodeId.hashCode() + jobId;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return nodeId + "_" + jobId;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobInfo.java
deleted file mode 100644
index 9a891f4..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobInfo.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * Compact job description.
- */
-public interface GridHadoopJobInfo extends Serializable {
- /**
- * Gets optional configuration property for the job.
- *
- * @param name Property name.
- * @return Value or {@code null} if none.
- */
- @Nullable public String property(String name);
-
- /**
- * Checks whether job has combiner.
- *
- * @return {@code true} If job has combiner.
- */
- public boolean hasCombiner();
-
- /**
- * Checks whether job has reducer.
- * Actual number of reducers will be in {@link GridHadoopMapReducePlan#reducers()}.
- *
- * @return Number of reducer.
- */
- public boolean hasReducer();
-
- /**
- * Creates new job instance for the given ID.
- * {@link GridHadoopJobInfo} is reusable for multiple jobs while {@link GridHadoopJob} is for one job execution.
- * This method will be called once for the same ID on one node, though it can be called on the same host
- * multiple times from different processes (in case of multiple nodes on the same host or external execution).
- *
- * @param jobId Job ID.
- * @param log Logger.
- * @return Job.
- * @throws IgniteCheckedException If failed.
- */
- GridHadoopJob createJob(GridHadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException;
-
- /**
- * @return Number of reducers configured for job.
- */
- public int reducers();
-
- /**
- * Gets job name.
- *
- * @return Job name.
- */
- public String jobName();
-
- /**
- * Gets user name.
- *
- * @return User name.
- */
- public String user();
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobPhase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobPhase.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobPhase.java
deleted file mode 100644
index cc122bb..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobPhase.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-/**
- * Job run phase.
- */
-public enum GridHadoopJobPhase {
- /** Job is running setup task. */
- PHASE_SETUP,
-
- /** Job is running map and combine tasks. */
- PHASE_MAP,
-
- /** Job has finished all map tasks and running reduce tasks. */
- PHASE_REDUCE,
-
- /** Job is stopping due to exception during any of the phases. */
- PHASE_CANCELLING,
-
- /** Job has finished execution. */
- PHASE_COMPLETE
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobProperty.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobProperty.java
deleted file mode 100644
index 0ece051..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobProperty.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.jetbrains.annotations.*;
-
-/**
- * Enumeration of optional properties supported by Ignite for Apache Hadoop.
- */
-public enum GridHadoopJobProperty {
- /**
- * Initial size for hashmap which stores output of mapper and will be used as input of combiner.
- * <p>
- * Setting it right allows to avoid rehashing.
- */
- COMBINER_HASHMAP_SIZE,
-
- /**
- * Initial size for hashmap which stores output of mapper or combiner and will be used as input of reducer.
- * <p>
- * Setting it right allows to avoid rehashing.
- */
- PARTITION_HASHMAP_SIZE,
-
- /**
- * Specifies number of concurrently running mappers for external execution mode.
- * <p>
- * If not specified, defaults to {@code Runtime.getRuntime().availableProcessors()}.
- */
- EXTERNAL_CONCURRENT_MAPPERS,
-
- /**
- * Specifies number of concurrently running reducers for external execution mode.
- * <p>
- * If not specified, defaults to {@code Runtime.getRuntime().availableProcessors()}.
- */
- EXTERNAL_CONCURRENT_REDUCERS,
-
- /**
- * Delay in milliseconds after which Ignite server will reply job status.
- */
- JOB_STATUS_POLL_DELAY,
-
- /**
- * Size in bytes of single memory page which will be allocated for data structures in shuffle.
- * <p>
- * By default is {@code 32 * 1024}.
- */
- SHUFFLE_OFFHEAP_PAGE_SIZE,
-
- /**
- * If set to {@code true} then input for combiner will not be sorted by key.
- * Internally hash-map will be used instead of sorted one, so {@link Object#equals(Object)}
- * and {@link Object#hashCode()} methods of key must be implemented consistently with
- * comparator for that type. Grouping comparator is not supported if this setting is {@code true}.
- * <p>
- * By default is {@code false}.
- */
- SHUFFLE_COMBINER_NO_SORTING,
-
- /**
- * If set to {@code true} then input for reducer will not be sorted by key.
- * Internally hash-map will be used instead of sorted one, so {@link Object#equals(Object)}
- * and {@link Object#hashCode()} methods of key must be implemented consistently with
- * comparator for that type. Grouping comparator is not supported if this setting is {@code true}.
- * <p>
- * By default is {@code false}.
- */
- SHUFFLE_REDUCER_NO_SORTING;
-
- /** */
- private final String ptyName;
-
- /**
- *
- */
- GridHadoopJobProperty() {
- ptyName = "ignite." + name().toLowerCase().replace('_', '.');
- }
-
- /**
- * @return Property name.
- */
- public String propertyName() {
- return ptyName;
- }
-
- /**
- * @param jobInfo Job info.
- * @param pty Property.
- * @param dflt Default value.
- * @return Property value.
- */
- public static String get(GridHadoopJobInfo jobInfo, GridHadoopJobProperty pty, @Nullable String dflt) {
- String res = jobInfo.property(pty.propertyName());
-
- return res == null ? dflt : res;
- }
-
- /**
- * @param jobInfo Job info.
- * @param pty Property.
- * @param dflt Default value.
- * @return Property value.
- */
- public static int get(GridHadoopJobInfo jobInfo, GridHadoopJobProperty pty, int dflt) {
- String res = jobInfo.property(pty.propertyName());
-
- return res == null ? dflt : Integer.parseInt(res);
- }
-
- /**
- * @param jobInfo Job info.
- * @param pty Property.
- * @param dflt Default value.
- * @return Property value.
- */
- public static boolean get(GridHadoopJobInfo jobInfo, GridHadoopJobProperty pty, boolean dflt) {
- String res = jobInfo.property(pty.propertyName());
-
- return res == null ? dflt : Boolean.parseBoolean(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobStatus.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobStatus.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobStatus.java
deleted file mode 100644
index 02ea883..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobStatus.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-
-/**
- * Hadoop job status.
- */
-public class GridHadoopJobStatus implements Externalizable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Job ID. */
- private GridHadoopJobId jobId;
-
- /** Job name. */
- private String jobName;
-
- /** User. */
- private String usr;
-
- /** Pending mappers count. */
- private int pendingMapperCnt;
-
- /** Pending reducers count. */
- private int pendingReducerCnt;
-
- /** Total mappers count. */
- private int totalMapperCnt;
-
- /** Total reducers count. */
- private int totalReducerCnt;
- /** Phase. */
- private GridHadoopJobPhase jobPhase;
-
- /** */
- private boolean failed;
-
- /** Version. */
- private long ver;
-
- /**
- * {@link Externalizable} support.
- */
- public GridHadoopJobStatus() {
- // No-op.
- }
-
- /**
- * Constructor.
- *
- * @param jobId Job ID.
- * @param jobName Job name.
- * @param usr User.
- * @param pendingMapperCnt Pending mappers count.
- * @param pendingReducerCnt Pending reducers count.
- * @param totalMapperCnt Total mappers count.
- * @param totalReducerCnt Total reducers count.
- * @param jobPhase Job phase.
- * @param failed Failed.
- * @param ver Version.
- */
- public GridHadoopJobStatus(
- GridHadoopJobId jobId,
- String jobName,
- String usr,
- int pendingMapperCnt,
- int pendingReducerCnt,
- int totalMapperCnt,
- int totalReducerCnt,
- GridHadoopJobPhase jobPhase,
- boolean failed,
- long ver
- ) {
- this.jobId = jobId;
- this.jobName = jobName;
- this.usr = usr;
- this.pendingMapperCnt = pendingMapperCnt;
- this.pendingReducerCnt = pendingReducerCnt;
- this.totalMapperCnt = totalMapperCnt;
- this.totalReducerCnt = totalReducerCnt;
- this.jobPhase = jobPhase;
- this.failed = failed;
- this.ver = ver;
- }
-
- /**
- * @return Job ID.
- */
- public GridHadoopJobId jobId() {
- return jobId;
- }
-
- /**
- * @return Job name.
- */
- public String jobName() {
- return jobName;
- }
-
- /**
- * @return User.
- */
- public String user() {
- return usr;
- }
-
- /**
- * @return Pending mappers count.
- */
- public int pendingMapperCnt() {
- return pendingMapperCnt;
- }
-
- /**
- * @return Pending reducers count.
- */
- public int pendingReducerCnt() {
- return pendingReducerCnt;
- }
-
- /**
- * @return Total mappers count.
- */
- public int totalMapperCnt() {
- return totalMapperCnt;
- }
-
- /**
- * @return Total reducers count.
- */
- public int totalReducerCnt() {
- return totalReducerCnt;
- }
-
- /**
- * @return Version.
- */
- public long version() {
- return ver;
- }
-
- /**
- * @return Job phase.
- */
- public GridHadoopJobPhase jobPhase() {
- return jobPhase;
- }
-
- /**
- * @return {@code true} If the job failed.
- */
- public boolean isFailed() {
- return failed;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopJobStatus.class, this);
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeObject(jobId);
- U.writeString(out, jobName);
- U.writeString(out, usr);
- out.writeInt(pendingMapperCnt);
- out.writeInt(pendingReducerCnt);
- out.writeInt(totalMapperCnt);
- out.writeInt(totalReducerCnt);
- out.writeObject(jobPhase);
- out.writeBoolean(failed);
- out.writeLong(ver);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- jobId = (GridHadoopJobId)in.readObject();
- jobName = U.readString(in);
- usr = U.readString(in);
- pendingMapperCnt = in.readInt();
- pendingReducerCnt = in.readInt();
- totalMapperCnt = in.readInt();
- totalReducerCnt = in.readInt();
- jobPhase = (GridHadoopJobPhase)in.readObject();
- failed = in.readBoolean();
- ver = in.readLong();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReducePlan.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReducePlan.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReducePlan.java
deleted file mode 100644
index 2fd5160..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReducePlan.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Map-reduce job execution plan.
- */
-public interface GridHadoopMapReducePlan extends Serializable {
- /**
- * Gets collection of file blocks for which mappers should be executed.
- *
- * @param nodeId Node ID to check.
- * @return Collection of file blocks or {@code null} if no mappers should be executed on given node.
- */
- @Nullable public Collection<GridHadoopInputSplit> mappers(UUID nodeId);
-
- /**
- * Gets reducer IDs that should be started on given node.
- *
- * @param nodeId Node ID to check.
- * @return Array of reducer IDs.
- */
- @Nullable public int[] reducers(UUID nodeId);
-
- /**
- * Gets collection of all node IDs involved in map part of job execution.
- *
- * @return Collection of node IDs.
- */
- public Collection<UUID> mapperNodeIds();
-
- /**
- * Gets collection of all node IDs involved in reduce part of job execution.
- *
- * @return Collection of node IDs.
- */
- public Collection<UUID> reducerNodeIds();
-
- /**
- * Gets overall number of mappers for the job.
- *
- * @return Number of mappers.
- */
- public int mappers();
-
- /**
- * Gets overall number of reducers for the job.
- *
- * @return Number of reducers.
- */
- public int reducers();
-
- /**
- * Gets node ID for reducer.
- *
- * @param reducer Reducer.
- * @return Node ID.
- */
- public UUID nodeForReducer(int reducer);
-}
[34/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopStreamDelegate.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopStreamDelegate.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopStreamDelegate.java
deleted file mode 100644
index 9aaab4c..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopStreamDelegate.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-/**
- * IGFS Hadoop stream descriptor.
- */
-public class IgfsHadoopStreamDelegate {
- /** RPC handler. */
- private final IgfsHadoopEx hadoop;
-
- /** Target. */
- private final Object target;
-
- /** Optional stream length. */
- private final long len;
-
- /**
- * Constructor.
- *
- * @param target Target.
- */
- public IgfsHadoopStreamDelegate(IgfsHadoopEx hadoop, Object target) {
- this(hadoop, target, -1);
- }
-
- /**
- * Constructor.
- *
- * @param target Target.
- * @param len Optional length.
- */
- public IgfsHadoopStreamDelegate(IgfsHadoopEx hadoop, Object target, long len) {
- assert hadoop != null;
- assert target != null;
-
- this.hadoop = hadoop;
- this.target = target;
- this.len = len;
- }
-
- /**
- * @return RPC handler.
- */
- public IgfsHadoopEx hadoop() {
- return hadoop;
- }
-
- /**
- * @return Stream target.
- */
- @SuppressWarnings("unchecked")
- public <T> T target() {
- return (T) target;
- }
-
- /**
- * @return Length.
- */
- public long length() {
- return len;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return System.identityHashCode(target);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object obj) {
- return obj != null && obj instanceof IgfsHadoopStreamDelegate &&
- target == ((IgfsHadoopStreamDelegate)obj).target;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(IgfsHadoopStreamDelegate.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopStreamEventListener.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopStreamEventListener.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopStreamEventListener.java
deleted file mode 100644
index 20d7f2a..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopStreamEventListener.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.ignite.*;
-
-/**
- * IGFS input stream event listener.
- */
-public interface IgfsHadoopStreamEventListener {
- /**
- * Callback invoked when the stream is being closed.
- *
- * @throws IgniteCheckedException If failed.
- */
- public void onClose() throws IgniteCheckedException;
-
- /**
- * Callback invoked when remote error occurs.
- *
- * @param errMsg Error message.
- */
- public void onError(String errMsg);
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopUtils.java
deleted file mode 100644
index bd96e60..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopUtils.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * Utility constants and methods for IGFS Hadoop file system.
- */
-public class IgfsHadoopUtils {
- /** Parameter name for endpoint no embed mode flag. */
- public static final String PARAM_IGFS_ENDPOINT_NO_EMBED = "fs.igfs.%s.endpoint.no_embed";
-
- /** Parameter name for endpoint no shared memory flag. */
- public static final String PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM = "fs.igfs.%s.endpoint.no_local_shmem";
-
- /** Parameter name for endpoint no local TCP flag. */
- public static final String PARAM_IGFS_ENDPOINT_NO_LOCAL_TCP = "fs.igfs.%s.endpoint.no_local_tcp";
-
- /**
- * Get string parameter.
- *
- * @param cfg Configuration.
- * @param name Parameter name.
- * @param authority Authority.
- * @param dflt Default value.
- * @return String value.
- */
- public static String parameter(Configuration cfg, String name, String authority, String dflt) {
- return cfg.get(String.format(name, authority != null ? authority : ""), dflt);
- }
-
- /**
- * Get integer parameter.
- *
- * @param cfg Configuration.
- * @param name Parameter name.
- * @param authority Authority.
- * @param dflt Default value.
- * @return Integer value.
- * @throws IOException In case of parse exception.
- */
- public static int parameter(Configuration cfg, String name, String authority, int dflt) throws IOException {
- String name0 = String.format(name, authority != null ? authority : "");
-
- try {
- return cfg.getInt(name0, dflt);
- }
- catch (NumberFormatException ignore) {
- throw new IOException("Failed to parse parameter value to integer: " + name0);
- }
- }
-
- /**
- * Get boolean parameter.
- *
- * @param cfg Configuration.
- * @param name Parameter name.
- * @param authority Authority.
- * @param dflt Default value.
- * @return Boolean value.
- */
- public static boolean parameter(Configuration cfg, String name, String authority, boolean dflt) {
- return cfg.getBoolean(String.format(name, authority != null ? authority : ""), dflt);
- }
-
- /**
- * Cast Ignite exception to appropriate IO exception.
- *
- * @param e Exception to cast.
- * @return Casted exception.
- */
- public static IOException cast(IgniteCheckedException e) {
- return cast(e, null);
- }
-
- /**
- * Cast Ignite exception to appropriate IO exception.
- *
- * @param e Exception to cast.
- * @param path Path for exceptions.
- * @return Casted exception.
- */
- @SuppressWarnings("unchecked")
- public static IOException cast(IgniteCheckedException e, @Nullable String path) {
- assert e != null;
-
- // First check for any nested IOException; if exists - re-throw it.
- if (e.hasCause(IOException.class))
- return e.getCause(IOException.class);
- else if (e.hasCause(IgfsFileNotFoundException.class))
- return new FileNotFoundException(path); // TODO: Or PathNotFoundException?
- else if (e.hasCause(IgfsParentNotDirectoryException.class))
- return new ParentNotDirectoryException(path);
- else if (path != null && e.hasCause(IgfsDirectoryNotEmptyException.class))
- return new PathIsNotEmptyDirectoryException(path);
- else if (path != null && e.hasCause(IgfsPathAlreadyExistsException.class))
- return new PathExistsException(path);
- else
- return new IOException(e);
- }
-
- /**
- * Constructor.
- */
- private IgfsHadoopUtils() {
- // No-op.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopWrapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopWrapper.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopWrapper.java
deleted file mode 100644
index 5586e72..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopWrapper.java
+++ /dev/null
@@ -1,511 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.commons.logging.*;
-import org.apache.hadoop.conf.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.atomic.*;
-
-import static org.apache.ignite.internal.igfs.hadoop.IgfsHadoopEndpoint.*;
-import static org.apache.ignite.internal.igfs.hadoop.IgfsHadoopUtils.*;
-
-/**
- * Wrapper for IGFS server.
- */
-public class IgfsHadoopWrapper implements IgfsHadoop {
- /** Delegate. */
- private final AtomicReference<Delegate> delegateRef = new AtomicReference<>();
-
- /** Authority. */
- private final String authority;
-
- /** Connection string. */
- private final IgfsHadoopEndpoint endpoint;
-
- /** Log directory. */
- private final String logDir;
-
- /** Configuration. */
- private final Configuration conf;
-
- /** Logger. */
- private final Log log;
-
- /**
- * Constructor.
- *
- * @param authority Authority (connection string).
- * @param logDir Log directory for server.
- * @param conf Configuration.
- * @param log Current logger.
- */
- public IgfsHadoopWrapper(String authority, String logDir, Configuration conf, Log log) throws IOException {
- try {
- this.authority = authority;
- this.endpoint = new IgfsHadoopEndpoint(authority);
- this.logDir = logDir;
- this.conf = conf;
- this.log = log;
- }
- catch (IgniteCheckedException e) {
- throw new IOException("Failed to parse endpoint: " + authority, e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHandshakeResponse handshake(String logDir) throws IOException {
- return withReconnectHandling(new FileSystemClosure<IgfsHandshakeResponse>() {
- @Override public IgfsHandshakeResponse apply(IgfsHadoopEx hadoop,
- IgfsHandshakeResponse hndResp) {
- return hndResp;
- }
- });
- }
-
- /** {@inheritDoc} */
- @Override public void close(boolean force) {
- Delegate delegate = delegateRef.get();
-
- if (delegate != null && delegateRef.compareAndSet(delegate, null))
- delegate.close(force);
- }
-
- /** {@inheritDoc} */
- @Override public IgfsFile info(final IgfsPath path) throws IOException {
- return withReconnectHandling(new FileSystemClosure<IgfsFile>() {
- @Override public IgfsFile apply(IgfsHadoopEx hadoop, IgfsHandshakeResponse hndResp)
- throws IgniteCheckedException, IOException {
- return hadoop.info(path);
- }
- }, path);
- }
-
- /** {@inheritDoc} */
- @Override public IgfsFile update(final IgfsPath path, final Map<String, String> props) throws IOException {
- return withReconnectHandling(new FileSystemClosure<IgfsFile>() {
- @Override public IgfsFile apply(IgfsHadoopEx hadoop, IgfsHandshakeResponse hndResp)
- throws IgniteCheckedException, IOException {
- return hadoop.update(path, props);
- }
- }, path);
- }
-
- /** {@inheritDoc} */
- @Override public Boolean setTimes(final IgfsPath path, final long accessTime, final long modificationTime)
- throws IOException {
- return withReconnectHandling(new FileSystemClosure<Boolean>() {
- @Override public Boolean apply(IgfsHadoopEx hadoop, IgfsHandshakeResponse hndResp)
- throws IgniteCheckedException, IOException {
- return hadoop.setTimes(path, accessTime, modificationTime);
- }
- }, path);
- }
-
- /** {@inheritDoc} */
- @Override public Boolean rename(final IgfsPath src, final IgfsPath dest) throws IOException {
- return withReconnectHandling(new FileSystemClosure<Boolean>() {
- @Override public Boolean apply(IgfsHadoopEx hadoop, IgfsHandshakeResponse hndResp)
- throws IgniteCheckedException, IOException {
- return hadoop.rename(src, dest);
- }
- }, src);
- }
-
- /** {@inheritDoc} */
- @Override public Boolean delete(final IgfsPath path, final boolean recursive) throws IOException {
- return withReconnectHandling(new FileSystemClosure<Boolean>() {
- @Override public Boolean apply(IgfsHadoopEx hadoop, IgfsHandshakeResponse hndResp)
- throws IgniteCheckedException, IOException {
- return hadoop.delete(path, recursive);
- }
- }, path);
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsBlockLocation> affinity(final IgfsPath path, final long start,
- final long len) throws IOException {
- return withReconnectHandling(new FileSystemClosure<Collection<IgfsBlockLocation>>() {
- @Override public Collection<IgfsBlockLocation> apply(IgfsHadoopEx hadoop,
- IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
- return hadoop.affinity(path, start, len);
- }
- }, path);
- }
-
- /** {@inheritDoc} */
- @Override public IgfsPathSummary contentSummary(final IgfsPath path) throws IOException {
- return withReconnectHandling(new FileSystemClosure<IgfsPathSummary>() {
- @Override public IgfsPathSummary apply(IgfsHadoopEx hadoop, IgfsHandshakeResponse hndResp)
- throws IgniteCheckedException, IOException {
- return hadoop.contentSummary(path);
- }
- }, path);
- }
-
- /** {@inheritDoc} */
- @Override public Boolean mkdirs(final IgfsPath path, final Map<String, String> props) throws IOException {
- return withReconnectHandling(new FileSystemClosure<Boolean>() {
- @Override public Boolean apply(IgfsHadoopEx hadoop, IgfsHandshakeResponse hndResp)
- throws IgniteCheckedException, IOException {
- return hadoop.mkdirs(path, props);
- }
- }, path);
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsFile> listFiles(final IgfsPath path) throws IOException {
- return withReconnectHandling(new FileSystemClosure<Collection<IgfsFile>>() {
- @Override public Collection<IgfsFile> apply(IgfsHadoopEx hadoop,
- IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
- return hadoop.listFiles(path);
- }
- }, path);
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsPath> listPaths(final IgfsPath path) throws IOException {
- return withReconnectHandling(new FileSystemClosure<Collection<IgfsPath>>() {
- @Override public Collection<IgfsPath> apply(IgfsHadoopEx hadoop,
- IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
- return hadoop.listPaths(path);
- }
- }, path);
- }
-
- /** {@inheritDoc} */
- @Override public IgfsStatus fsStatus() throws IOException {
- return withReconnectHandling(new FileSystemClosure<IgfsStatus>() {
- @Override public IgfsStatus apply(IgfsHadoopEx hadoop, IgfsHandshakeResponse hndResp)
- throws IgniteCheckedException, IOException {
- return hadoop.fsStatus();
- }
- });
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHadoopStreamDelegate open(final IgfsPath path) throws IOException {
- return withReconnectHandling(new FileSystemClosure<IgfsHadoopStreamDelegate>() {
- @Override public IgfsHadoopStreamDelegate apply(IgfsHadoopEx hadoop,
- IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
- return hadoop.open(path);
- }
- }, path);
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHadoopStreamDelegate open(final IgfsPath path, final int seqReadsBeforePrefetch)
- throws IOException {
- return withReconnectHandling(new FileSystemClosure<IgfsHadoopStreamDelegate>() {
- @Override public IgfsHadoopStreamDelegate apply(IgfsHadoopEx hadoop,
- IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
- return hadoop.open(path, seqReadsBeforePrefetch);
- }
- }, path);
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHadoopStreamDelegate create(final IgfsPath path, final boolean overwrite,
- final boolean colocate, final int replication, final long blockSize, @Nullable final Map<String, String> props)
- throws IOException {
- return withReconnectHandling(new FileSystemClosure<IgfsHadoopStreamDelegate>() {
- @Override public IgfsHadoopStreamDelegate apply(IgfsHadoopEx hadoop,
- IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
- return hadoop.create(path, overwrite, colocate, replication, blockSize, props);
- }
- }, path);
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHadoopStreamDelegate append(final IgfsPath path, final boolean create,
- @Nullable final Map<String, String> props) throws IOException {
- return withReconnectHandling(new FileSystemClosure<IgfsHadoopStreamDelegate>() {
- @Override public IgfsHadoopStreamDelegate apply(IgfsHadoopEx hadoop,
- IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException {
- return hadoop.append(path, create, props);
- }
- }, path);
- }
-
- /**
- * Execute closure which is not path-specific.
- *
- * @param clo Closure.
- * @return Result.
- * @throws IOException If failed.
- */
- private <T> T withReconnectHandling(FileSystemClosure<T> clo) throws IOException {
- return withReconnectHandling(clo, null);
- }
-
- /**
- * Execute closure.
- *
- * @param clo Closure.
- * @param path Path for exceptions.
- * @return Result.
- * @throws IOException If failed.
- */
- private <T> T withReconnectHandling(final FileSystemClosure<T> clo, @Nullable IgfsPath path)
- throws IOException {
- Exception err = null;
-
- for (int i = 0; i < 2; i++) {
- Delegate curDelegate = null;
-
- boolean close = false;
- boolean force = false;
-
- try {
- curDelegate = delegate();
-
- assert curDelegate != null;
-
- close = curDelegate.doomed;
-
- return clo.apply(curDelegate.hadoop, curDelegate.hndResp);
- }
- catch (IgfsHadoopCommunicationException e) {
- if (curDelegate != null && !curDelegate.doomed) {
- // Try getting rid fo faulty delegate ASAP.
- delegateRef.compareAndSet(curDelegate, null);
-
- close = true;
- force = true;
- }
-
- if (log.isDebugEnabled())
- log.debug("Failed to send message to a server: " + e);
-
- err = e;
- }
- catch (IgniteCheckedException e) {
- throw IgfsHadoopUtils.cast(e, path != null ? path.toString() : null);
- }
- finally {
- if (close) {
- assert curDelegate != null;
-
- curDelegate.close(force);
- }
- }
- }
-
- throw new IOException("Failed to communicate with IGFS.", err);
- }
-
- /**
- * Get delegate creating it if needed.
- *
- * @return Delegate.
- */
- private Delegate delegate() throws IgfsHadoopCommunicationException {
- Exception err = null;
-
- // 1. If delegate is set, return it immediately.
- Delegate curDelegate = delegateRef.get();
-
- if (curDelegate != null)
- return curDelegate;
-
- // 2. Guess that we are in the same VM.
- if (!parameter(conf, PARAM_IGFS_ENDPOINT_NO_EMBED, authority, false)) {
- IgfsEx igfs = null;
-
- if (endpoint.grid() == null) {
- try {
- Ignite ignite = G.ignite();
-
- igfs = (IgfsEx)ignite.fileSystem(endpoint.igfs());
- }
- catch (Exception e) {
- err = e;
- }
- }
- else {
- for (Ignite ignite : G.allGrids()) {
- try {
- igfs = (IgfsEx)ignite.fileSystem(endpoint.igfs());
-
- break;
- }
- catch (Exception e) {
- err = e;
- }
- }
- }
-
- if (igfs != null) {
- IgfsHadoopEx hadoop = null;
-
- try {
- hadoop = new IgfsHadoopInProc(igfs, log);
-
- curDelegate = new Delegate(hadoop, hadoop.handshake(logDir));
- }
- catch (IOException | IgniteCheckedException e) {
- if (e instanceof IgfsHadoopCommunicationException)
- hadoop.close(true);
-
- if (log.isDebugEnabled())
- log.debug("Failed to connect to in-proc IGFS, fallback to IPC mode.", e);
-
- err = e;
- }
- }
- }
-
- // 3. Try connecting using shmem.
- if (!parameter(conf, PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority, false)) {
- if (curDelegate == null && !U.isWindows()) {
- IgfsHadoopEx hadoop = null;
-
- try {
- hadoop = new IgfsHadoopOutProc(endpoint.port(), endpoint.grid(), endpoint.igfs(), log);
-
- curDelegate = new Delegate(hadoop, hadoop.handshake(logDir));
- }
- catch (IOException | IgniteCheckedException e) {
- if (e instanceof IgfsHadoopCommunicationException)
- hadoop.close(true);
-
- if (log.isDebugEnabled())
- log.debug("Failed to connect to out-proc local IGFS using shmem.", e);
-
- err = e;
- }
- }
- }
-
- // 4. Try local TCP connection.
- boolean skipLocTcp = parameter(conf, PARAM_IGFS_ENDPOINT_NO_LOCAL_TCP, authority, false);
-
- if (!skipLocTcp) {
- if (curDelegate == null) {
- IgfsHadoopEx hadoop = null;
-
- try {
- hadoop = new IgfsHadoopOutProc(LOCALHOST, endpoint.port(), endpoint.grid(), endpoint.igfs(),
- log);
-
- curDelegate = new Delegate(hadoop, hadoop.handshake(logDir));
- }
- catch (IOException | IgniteCheckedException e) {
- if (e instanceof IgfsHadoopCommunicationException)
- hadoop.close(true);
-
- if (log.isDebugEnabled())
- log.debug("Failed to connect to out-proc local IGFS using TCP.", e);
-
- err = e;
- }
- }
- }
-
- // 5. Try remote TCP connection.
- if (curDelegate == null && (skipLocTcp || !F.eq(LOCALHOST, endpoint.host()))) {
- IgfsHadoopEx hadoop = null;
-
- try {
- hadoop = new IgfsHadoopOutProc(endpoint.host(), endpoint.port(), endpoint.grid(), endpoint.igfs(), log);
-
- curDelegate = new Delegate(hadoop, hadoop.handshake(logDir));
- }
- catch (IOException | IgniteCheckedException e) {
- if (e instanceof IgfsHadoopCommunicationException)
- hadoop.close(true);
-
- if (log.isDebugEnabled())
- log.debug("Failed to connect to out-proc remote IGFS using TCP.", e);
-
- err = e;
- }
- }
-
- if (curDelegate != null) {
- if (!delegateRef.compareAndSet(null, curDelegate))
- curDelegate.doomed = true;
-
- return curDelegate;
- }
- else
- throw new IgfsHadoopCommunicationException("Failed to connect to IGFS: " + endpoint, err);
- }
-
- /**
- * File system operation closure.
- */
- private static interface FileSystemClosure<T> {
- /**
- * Call closure body.
- *
- * @param hadoop RPC handler.
- * @param hndResp Handshake response.
- * @return Result.
- * @throws IgniteCheckedException If failed.
- * @throws IOException If failed.
- */
- public T apply(IgfsHadoopEx hadoop, IgfsHandshakeResponse hndResp) throws IgniteCheckedException, IOException;
- }
-
- /**
- * Delegate.
- */
- private static class Delegate {
- /** RPC handler. */
- private final IgfsHadoopEx hadoop;
-
- /** Handshake request. */
- private final IgfsHandshakeResponse hndResp;
-
- /** Close guard. */
- private final AtomicBoolean closeGuard = new AtomicBoolean();
-
- /** Whether this delegate must be closed at the end of the next invocation. */
- private boolean doomed;
-
- /**
- * Constructor.
- *
- * @param hadoop Hadoop.
- * @param hndResp Handshake response.
- */
- private Delegate(IgfsHadoopEx hadoop, IgfsHandshakeResponse hndResp) {
- this.hadoop = hadoop;
- this.hndResp = hndResp;
- }
-
- /**
- * Close underlying RPC handler.
- *
- * @param force Force flag.
- */
- private void close(boolean force) {
- if (closeGuard.compareAndSet(false, true))
- hadoop.close(force);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/package.html
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/package.html b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/package.html
deleted file mode 100644
index ec380f2..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/package.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<body>
- <!-- Package description. -->
- Contains IGFS client classes.
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/package.html
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/package.html b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/package.html
deleted file mode 100644
index 4b070d3..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/package.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<body>
- <!-- Package description. -->
- Contains IGFS client and common classes.
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopClassLoader.java
deleted file mode 100644
index bc4c0bb..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopClassLoader.java
+++ /dev/null
@@ -1,552 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.jdk8.backport.*;
-import org.jetbrains.annotations.*;
-import org.objectweb.asm.*;
-import org.objectweb.asm.commons.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-import java.util.concurrent.atomic.*;
-
-/**
- * Class loader allowing explicitly load classes without delegation to parent class loader.
- * Also supports class parsing for finding dependencies which contain transitive dependencies
- * unavailable for parent.
- */
-public class GridHadoopClassLoader extends URLClassLoader {
- /**
- * We are very parallel capable.
- */
- static {
- registerAsParallelCapable();
- }
-
- /** */
- private static final URLClassLoader APP_CLS_LDR = (URLClassLoader)GridHadoopClassLoader.class.getClassLoader();
-
- /** */
- private static final Collection<URL> appJars = F.asList(APP_CLS_LDR.getURLs());
-
- /** */
- private static volatile Collection<URL> hadoopJars;
-
- /** */
- private static final Map<String, Boolean> cache = new ConcurrentHashMap8<>();
-
- /** */
- private static final Map<String, byte[]> bytesCache = new ConcurrentHashMap8<>();
-
- /**
- * @param urls Urls.
- */
- public GridHadoopClassLoader(URL[] urls) {
- super(addHadoopUrls(urls), APP_CLS_LDR);
-
- assert !(getParent() instanceof GridHadoopClassLoader);
- }
-
- /**
- * Need to parse only Ignite Hadoop and IGFS classes.
- *
- * @param cls Class name.
- * @return {@code true} if we need to check this class.
- */
- private static boolean isIgfsHadoop(String cls) {
- String ignitePackagePrefix = "org.apache.ignite";
- int len = ignitePackagePrefix.length();
-
- return cls.startsWith(ignitePackagePrefix) && (cls.indexOf("igfs.", len) != -1 || cls.indexOf(".fs.", len) != -1 || cls.indexOf("hadoop.", len) != -1);
- }
-
- /**
- * @param cls Class name.
- * @return {@code true} If this is Hadoop class.
- */
- private static boolean isHadoop(String cls) {
- return cls.startsWith("org.apache.hadoop.");
- }
-
- /** {@inheritDoc} */
- @Override protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
- try {
- if (isHadoop(name)) { // Always load Hadoop classes explicitly, since Hadoop can be available in App classpath.
- if (name.endsWith(".util.ShutdownHookManager")) // Dirty hack to get rid of Hadoop shutdown hooks.
- return loadFromBytes(name, GridHadoopShutdownHookManager.class.getName());
- else if (name.endsWith(".util.NativeCodeLoader"))
- return loadFromBytes(name, GridHadoopNativeCodeLoader.class.getName());
-
- return loadClassExplicitly(name, resolve);
- }
-
- if (isIgfsHadoop(name)) { // For Ignite Hadoop and IGFS classes we have to check if they depend on Hadoop.
- Boolean hasDeps = cache.get(name);
-
- if (hasDeps == null) {
- hasDeps = hasExternalDependencies(name, new HashSet<String>());
-
- cache.put(name, hasDeps);
- }
-
- if (hasDeps)
- return loadClassExplicitly(name, resolve);
- }
-
- return super.loadClass(name, resolve);
- }
- catch (NoClassDefFoundError | ClassNotFoundException e) {
- throw new ClassNotFoundException("Failed to load class: " + name, e);
- }
- }
-
- /**
- * @param name Name.
- * @param replace Replacement.
- * @return Class.
- */
- private Class<?> loadFromBytes(final String name, final String replace) {
- synchronized (getClassLoadingLock(name)) {
- // First, check if the class has already been loaded
- Class c = findLoadedClass(name);
-
- if (c != null)
- return c;
-
- byte[] bytes = bytesCache.get(name);
-
- if (bytes == null) {
- InputStream in = loadClassBytes(getParent(), replace);
-
- ClassReader rdr;
-
- try {
- rdr = new ClassReader(in);
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- ClassWriter w = new ClassWriter(Opcodes.ASM4);
-
- rdr.accept(new RemappingClassAdapter(w, new Remapper() {
- /** */
- String replaceType = replace.replace('.', '/');
-
- /** */
- String nameType = name.replace('.', '/');
-
- @Override public String map(String type) {
- if (type.equals(replaceType))
- return nameType;
-
- return type;
- }
- }), ClassReader.EXPAND_FRAMES);
-
- bytes = w.toByteArray();
-
- bytesCache.put(name, bytes);
- }
-
- return defineClass(name, bytes, 0, bytes.length);
- }
- }
-
- /**
- * @param name Class name.
- * @param resolve Resolve class.
- * @return Class.
- * @throws ClassNotFoundException If failed.
- */
- private Class<?> loadClassExplicitly(String name, boolean resolve) throws ClassNotFoundException {
- synchronized (getClassLoadingLock(name)) {
- // First, check if the class has already been loaded
- Class c = findLoadedClass(name);
-
- if (c == null) {
- long t1 = System.nanoTime();
-
- c = findClass(name);
-
- // this is the defining class loader; record the stats
- sun.misc.PerfCounter.getFindClassTime().addElapsedTimeFrom(t1);
- sun.misc.PerfCounter.getFindClasses().increment();
- }
-
- if (resolve)
- resolveClass(c);
-
- return c;
- }
- }
-
- /**
- * @param ldr Loader.
- * @param clsName Class.
- * @return Input stream.
- */
- @Nullable private InputStream loadClassBytes(ClassLoader ldr, String clsName) {
- return ldr.getResourceAsStream(clsName.replace('.', '/') + ".class");
- }
-
- /**
- * @param clsName Class name.
- * @return {@code true} If the class has external dependencies.
- */
- boolean hasExternalDependencies(final String clsName, final Set<String> visited) {
- if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea sucks, so filtering explicitly as external.
- return true;
-
- // Try to get from parent to check if the type accessible.
- InputStream in = loadClassBytes(getParent(), clsName);
-
- if (in == null) // The class is external itself, it must be loaded from this class loader.
- return true;
-
- if (!isIgfsHadoop(clsName)) // Other classes should not have external dependencies.
- return false;
-
- final ClassReader rdr;
-
- try {
- rdr = new ClassReader(in);
- }
- catch (IOException e) {
- throw new RuntimeException("Failed to read class: " + clsName, e);
- }
-
- visited.add(clsName);
-
- final AtomicBoolean hasDeps = new AtomicBoolean();
-
- rdr.accept(new ClassVisitor(Opcodes.ASM4) {
- AnnotationVisitor av = new AnnotationVisitor(Opcodes.ASM4) {
- // TODO
- };
-
- FieldVisitor fv = new FieldVisitor(Opcodes.ASM4) {
- @Override public AnnotationVisitor visitAnnotation(String desc, boolean b) {
- onType(desc);
-
- return av;
- }
- };
-
- MethodVisitor mv = new MethodVisitor(Opcodes.ASM4) {
- @Override public AnnotationVisitor visitAnnotation(String desc, boolean b) {
- onType(desc);
-
- return av;
- }
-
- @Override public AnnotationVisitor visitParameterAnnotation(int i, String desc, boolean b) {
- onType(desc);
-
- return av;
- }
-
- @Override public AnnotationVisitor visitAnnotationDefault() {
- return av;
- }
-
- @Override public void visitFieldInsn(int i, String owner, String name, String desc) {
- onType(owner);
- onType(desc);
- }
-
- @Override public void visitFrame(int i, int i2, Object[] locTypes, int i3, Object[] stackTypes) {
- for (Object o : locTypes) {
- if (o instanceof String)
- onType((String)o);
- }
-
- for (Object o : stackTypes) {
- if (o instanceof String)
- onType((String)o);
- }
- }
-
- @Override public void visitLocalVariable(String name, String desc, String signature, Label lb,
- Label lb2, int i) {
- onType(desc);
- }
-
- @Override public void visitMethodInsn(int i, String owner, String name, String desc) {
- onType(owner);
- }
-
- @Override public void visitMultiANewArrayInsn(String desc, int dim) {
- onType(desc);
- }
-
- @Override public void visitTryCatchBlock(Label lb, Label lb2, Label lb3, String e) {
- onType(e);
- }
- };
-
- void onClass(String depCls) {
- assert validateClassName(depCls) : depCls;
-
- if (depCls.startsWith("java.")) // Filter out platform classes.
- return;
-
- if (visited.contains(depCls))
- return;
-
- Boolean res = cache.get(depCls);
-
- if (res == Boolean.TRUE || (res == null && hasExternalDependencies(depCls, visited)))
- hasDeps.set(true);
- }
-
- void onType(String type) {
- if (type == null)
- return;
-
- int off = 0;
-
- while (type.charAt(off) == '[')
- off++; // Handle arrays.
-
- if (off != 0)
- type = type.substring(off);
-
- if (type.length() == 1)
- return; // Get rid of primitives.
-
- if (type.charAt(type.length() - 1) == ';') {
- assert type.charAt(0) == 'L' : type;
-
- type = type.substring(1, type.length() - 1);
- }
-
- type = type.replace('/', '.');
-
- onClass(type);
- }
-
- @Override public void visit(int i, int i2, String name, String signature, String superName,
- String[] ifaces) {
- onType(superName);
-
- if (ifaces != null) {
- for (String iface : ifaces)
- onType(iface);
- }
- }
-
- @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
- onType(desc);
-
- return av;
- }
-
- @Override public void visitInnerClass(String name, String outerName, String innerName, int i) {
- onType(name);
- }
-
- @Override public FieldVisitor visitField(int i, String name, String desc, String signature, Object val) {
- onType(desc);
-
- return fv;
- }
-
- @Override public MethodVisitor visitMethod(int i, String name, String desc, String signature,
- String[] exceptions) {
- if (exceptions != null) {
- for (String e : exceptions)
- onType(e);
- }
-
- return mv;
- }
- }, 0);
-
- if (hasDeps.get()) // We already know that we have dependencies, no need to check parent.
- return true;
-
- // Here we are known to not have any dependencies but possibly we have a parent which have them.
- int idx = clsName.lastIndexOf('$');
-
- if (idx == -1) // No parent class.
- return false;
-
- String parentCls = clsName.substring(0, idx);
-
- if (visited.contains(parentCls))
- return false;
-
- Boolean res = cache.get(parentCls);
-
- if (res == null)
- res = hasExternalDependencies(parentCls, visited);
-
- return res;
- }
-
- /**
- * @param name Class name.
- * @return {@code true} If this is a valid class name.
- */
- private static boolean validateClassName(String name) {
- int len = name.length();
-
- if (len <= 1)
- return false;
-
- if (!Character.isJavaIdentifierStart(name.charAt(0)))
- return false;
-
- boolean hasDot = false;
-
- for (int i = 1; i < len; i++) {
- char c = name.charAt(i);
-
- if (c == '.')
- hasDot = true;
- else if (!Character.isJavaIdentifierPart(c))
- return false;
- }
-
- return hasDot;
- }
-
- /**
- * @param name Variable name.
- * @param dflt Default.
- * @return Value.
- */
- private static String getEnv(String name, String dflt) {
- String res = System.getProperty(name);
-
- if (F.isEmpty(res))
- res = System.getenv(name);
-
- return F.isEmpty(res) ? dflt : res;
- }
-
- /**
- * @param res Result.
- * @param dir Directory.
- * @param startsWith Starts with prefix.
- * @throws MalformedURLException If failed.
- */
- private static void addUrls(Collection<URL> res, File dir, final String startsWith) throws Exception {
- File[] files = dir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return startsWith == null || name.startsWith(startsWith);
- }
- });
-
- if (files == null)
- throw new IOException("Path is not a directory: " + dir);
-
- for (File file : files)
- res.add(file.toURI().toURL());
- }
-
- /**
- * @param urls URLs.
- * @return URLs.
- */
- private static URL[] addHadoopUrls(URL[] urls) {
- Collection<URL> hadoopJars;
-
- try {
- hadoopJars = hadoopUrls();
- }
- catch (IgniteCheckedException e) {
- throw new RuntimeException(e);
- }
-
- ArrayList<URL> list = new ArrayList<>(hadoopJars.size() + appJars.size() + (urls == null ? 0 : urls.length));
-
- list.addAll(appJars);
- list.addAll(hadoopJars);
-
- if (!F.isEmpty(urls))
- list.addAll(F.asList(urls));
-
- return list.toArray(new URL[list.size()]);
- }
-
- /**
- * @return HADOOP_HOME Variable.
- */
- @Nullable public static String hadoopHome() {
- return getEnv("HADOOP_PREFIX", getEnv("HADOOP_HOME", null));
- }
-
- /**
- * @return Collection of jar URLs.
- * @throws IgniteCheckedException If failed.
- */
- public static Collection<URL> hadoopUrls() throws IgniteCheckedException {
- Collection<URL> hadoopUrls = hadoopJars;
-
- if (hadoopUrls != null)
- return hadoopUrls;
-
- synchronized (GridHadoopClassLoader.class) {
- hadoopUrls = hadoopJars;
-
- if (hadoopUrls != null)
- return hadoopUrls;
-
- hadoopUrls = new ArrayList<>();
-
- String hadoopPrefix = hadoopHome();
-
- if (F.isEmpty(hadoopPrefix))
- throw new IgniteCheckedException("Failed resolve Hadoop installation location. Either HADOOP_PREFIX or " +
- "HADOOP_HOME environment variables must be set.");
-
- String commonHome = getEnv("HADOOP_COMMON_HOME", hadoopPrefix + "/share/hadoop/common");
- String hdfsHome = getEnv("HADOOP_HDFS_HOME", hadoopPrefix + "/share/hadoop/hdfs");
- String mapredHome = getEnv("HADOOP_MAPRED_HOME", hadoopPrefix + "/share/hadoop/mapreduce");
-
- try {
- addUrls(hadoopUrls, new File(commonHome + "/lib"), null);
- addUrls(hadoopUrls, new File(hdfsHome + "/lib"), null);
- addUrls(hadoopUrls, new File(mapredHome + "/lib"), null);
-
- addUrls(hadoopUrls, new File(hdfsHome), "hadoop-hdfs-");
-
- addUrls(hadoopUrls, new File(commonHome), "hadoop-common-");
- addUrls(hadoopUrls, new File(commonHome), "hadoop-auth-");
- addUrls(hadoopUrls, new File(commonHome + "/lib"), "hadoop-auth-");
-
- addUrls(hadoopUrls, new File(mapredHome), "hadoop-mapreduce-client-common");
- addUrls(hadoopUrls, new File(mapredHome), "hadoop-mapreduce-client-core");
- }
- catch (Exception e) {
- throw new IgniteCheckedException(e);
- }
-
- hadoopJars = hadoopUrls;
-
- return hadoopUrls;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopComponent.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopComponent.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopComponent.java
deleted file mode 100644
index 337bfe9..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopComponent.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-
-/**
- * Abstract class for all hadoop components.
- */
-public abstract class GridHadoopComponent {
- /** Hadoop context. */
- protected GridHadoopContext ctx;
-
- /** Logger. */
- protected IgniteLogger log;
-
- /**
- * @param ctx Hadoop context.
- */
- public void start(GridHadoopContext ctx) throws IgniteCheckedException {
- this.ctx = ctx;
-
- log = ctx.kernalContext().log(getClass());
- }
-
- /**
- * Stops manager.
- */
- public void stop(boolean cancel) {
- // No-op.
- }
-
- /**
- * Callback invoked when all grid components are started.
- */
- public void onKernalStart() throws IgniteCheckedException {
- // No-op.
- }
-
- /**
- * Callback invoked before all grid components are stopped.
- */
- public void onKernalStop(boolean cancel) {
- // No-op.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopContext.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopContext.java
deleted file mode 100644
index 3160e3d..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopContext.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.cluster.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
-import org.apache.ignite.internal.processors.hadoop.shuffle.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.util.*;
-
-/**
- * Hadoop accelerator context.
- */
-public class GridHadoopContext {
- /** Kernal context. */
- private GridKernalContext ctx;
-
- /** Hadoop configuration. */
- private GridHadoopConfiguration cfg;
-
- /** Job tracker. */
- private GridHadoopJobTracker jobTracker;
-
- /** External task executor. */
- private GridHadoopTaskExecutorAdapter taskExecutor;
-
- /** */
- private GridHadoopShuffle shuffle;
-
- /** Managers list. */
- private List<GridHadoopComponent> components = new ArrayList<>();
-
- /**
- * @param ctx Kernal context.
- */
- public GridHadoopContext(
- GridKernalContext ctx,
- GridHadoopConfiguration cfg,
- GridHadoopJobTracker jobTracker,
- GridHadoopTaskExecutorAdapter taskExecutor,
- GridHadoopShuffle shuffle
- ) {
- this.ctx = ctx;
- this.cfg = cfg;
-
- this.jobTracker = add(jobTracker);
- this.taskExecutor = add(taskExecutor);
- this.shuffle = add(shuffle);
- }
-
- /**
- * Gets list of managers.
- *
- * @return List of managers.
- */
- public List<GridHadoopComponent> components() {
- return components;
- }
-
- /**
- * Gets kernal context.
- *
- * @return Grid kernal context instance.
- */
- public GridKernalContext kernalContext() {
- return ctx;
- }
-
- /**
- * Gets Hadoop configuration.
- *
- * @return Hadoop configuration.
- */
- public GridHadoopConfiguration configuration() {
- return cfg;
- }
-
- /**
- * Gets local node ID. Shortcut for {@code kernalContext().localNodeId()}.
- *
- * @return Local node ID.
- */
- public UUID localNodeId() {
- return ctx.localNodeId();
- }
-
- /**
- * Gets local node order.
- *
- * @return Local node order.
- */
- public long localNodeOrder() {
- assert ctx.discovery() != null;
-
- return ctx.discovery().localNode().order();
- }
-
- /**
- * @return Hadoop-enabled nodes.
- */
- public Collection<ClusterNode> nodes() {
- return ctx.discovery().cacheNodes(CU.SYS_CACHE_HADOOP_MR, ctx.discovery().topologyVersion());
- }
-
- /**
- * @return {@code True} if
- */
- public boolean jobUpdateLeader() {
- long minOrder = Long.MAX_VALUE;
- ClusterNode minOrderNode = null;
-
- for (ClusterNode node : nodes()) {
- if (node.order() < minOrder) {
- minOrder = node.order();
- minOrderNode = node;
- }
- }
-
- assert minOrderNode != null;
-
- return localNodeId().equals(minOrderNode.id());
- }
-
- /**
- * @param meta Job metadata.
- * @return {@code true} If local node is participating in job execution.
- */
- public boolean isParticipating(GridHadoopJobMetadata meta) {
- UUID locNodeId = localNodeId();
-
- if (locNodeId.equals(meta.submitNodeId()))
- return true;
-
- GridHadoopMapReducePlan plan = meta.mapReducePlan();
-
- return plan.mapperNodeIds().contains(locNodeId) || plan.reducerNodeIds().contains(locNodeId) || jobUpdateLeader();
- }
-
- /**
- * @return Jon tracker instance.
- */
- public GridHadoopJobTracker jobTracker() {
- return jobTracker;
- }
-
- /**
- * @return Task executor.
- */
- public GridHadoopTaskExecutorAdapter taskExecutor() {
- return taskExecutor;
- }
-
- /**
- * @return Shuffle.
- */
- public GridHadoopShuffle shuffle() {
- return shuffle;
- }
-
- /**
- * @return Map-reduce planner.
- */
- public GridHadoopMapReducePlanner planner() {
- return cfg.getMapReducePlanner();
- }
-
- /**
- * Adds component.
- *
- * @param c Component to add.
- * @return Added manager.
- */
- private <C extends GridHadoopComponent> C add(C c) {
- components.add(c);
-
- return c;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopDefaultJobInfo.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopDefaultJobInfo.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopDefaultJobInfo.java
deleted file mode 100644
index 555c573..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopDefaultJobInfo.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.lang.reflect.*;
-import java.util.*;
-
-/**
- * Hadoop job info based on default Hadoop configuration.
- */
-public class GridHadoopDefaultJobInfo implements GridHadoopJobInfo, Externalizable {
- /** */
- private static final long serialVersionUID = 5489900236464999951L;
-
- /** {@code true} If job has combiner. */
- private boolean hasCombiner;
-
- /** Number of reducers configured for job. */
- private int numReduces;
-
- /** Configuration. */
- private Map<String,String> props = new HashMap<>();
-
- /** Job name. */
- private String jobName;
-
- /** User name. */
- private String user;
-
- /** */
- private static volatile Class<?> jobCls;
-
- /**
- * Default constructor required by {@link Externalizable}.
- */
- public GridHadoopDefaultJobInfo() {
- // No-op.
- }
-
- /**
- * Constructor.
- *
- * @param jobName Job name.
- * @param user User name.
- * @param hasCombiner {@code true} If job has combiner.
- * @param numReduces Number of reducers configured for job.
- * @param props All other properties of the job.
- */
- public GridHadoopDefaultJobInfo(String jobName, String user, boolean hasCombiner, int numReduces,
- Map<String, String> props) {
- this.jobName = jobName;
- this.user = user;
- this.hasCombiner = hasCombiner;
- this.numReduces = numReduces;
- this.props = props;
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public String property(String name) {
- return props.get(name);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopJob createJob(GridHadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException {
- try {
- Class<?> jobCls0 = jobCls;
-
- if (jobCls0 == null) { // It is enough to have only one class loader with only Hadoop classes.
- synchronized (GridHadoopDefaultJobInfo.class) {
- if ((jobCls0 = jobCls) == null) {
- GridHadoopClassLoader ldr = new GridHadoopClassLoader(null);
-
- jobCls = jobCls0 = ldr.loadClass(GridHadoopV2Job.class.getName());
- }
- }
- }
-
- Constructor<?> constructor = jobCls0.getConstructor(GridHadoopJobId.class, GridHadoopDefaultJobInfo.class,
- IgniteLogger.class);
-
- return (GridHadoopJob)constructor.newInstance(jobId, this, log);
- }
- // NB: java.lang.NoClassDefFoundError may be thrown from Class#getConstructor() call.
- catch (Throwable t) {
- throw new IgniteCheckedException(t);
- }
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasCombiner() {
- return hasCombiner;
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasReducer() {
- return reducers() > 0;
- }
-
- /** {@inheritDoc} */
- @Override public int reducers() {
- return numReduces;
- }
-
- /** {@inheritDoc} */
- @Override public String jobName() {
- return jobName;
- }
-
- /** {@inheritDoc} */
- @Override public String user() {
- return user;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- U.writeString(out, jobName);
- U.writeString(out, user);
-
- out.writeBoolean(hasCombiner);
- out.writeInt(numReduces);
-
- U.writeStringMap(out, props);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- jobName = U.readString(in);
- user = U.readString(in);
-
- hasCombiner = in.readBoolean();
- numReduces = in.readInt();
-
- props = U.readStringMap(in);
- }
-
- /**
- * @return Properties of the job.
- */
- public Map<String, String> properties() {
- return props;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopImpl.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopImpl.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopImpl.java
deleted file mode 100644
index 55e3690..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopImpl.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.util.*;
-import org.jetbrains.annotations.*;
-
-/**
- * Hadoop facade implementation.
- */
-public class GridHadoopImpl implements GridHadoop {
- /** Hadoop processor. */
- private final IgniteHadoopProcessor proc;
-
- /** Busy lock. */
- private final GridSpinBusyLock busyLock = new GridSpinBusyLock();
-
- /**
- * Constructor.
- *
- * @param proc Hadoop processor.
- */
- GridHadoopImpl(IgniteHadoopProcessor proc) {
- this.proc = proc;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopConfiguration configuration() {
- return proc.config();
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopJobId nextJobId() {
- if (busyLock.enterBusy()) {
- try {
- return proc.nextJobId();
- }
- finally {
- busyLock.leaveBusy();
- }
- }
- else
- throw new IllegalStateException("Failed to get next job ID (grid is stopping).");
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<?> submit(GridHadoopJobId jobId, GridHadoopJobInfo jobInfo) {
- if (busyLock.enterBusy()) {
- try {
- return proc.submit(jobId, jobInfo);
- }
- finally {
- busyLock.leaveBusy();
- }
- }
- else
- throw new IllegalStateException("Failed to submit job (grid is stopping).");
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public GridHadoopJobStatus status(GridHadoopJobId jobId) throws IgniteCheckedException {
- if (busyLock.enterBusy()) {
- try {
- return proc.status(jobId);
- }
- finally {
- busyLock.leaveBusy();
- }
- }
- else
- throw new IllegalStateException("Failed to get job status (grid is stopping).");
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public GridHadoopCounters counters(GridHadoopJobId jobId) throws IgniteCheckedException {
- if (busyLock.enterBusy()) {
- try {
- return proc.counters(jobId);
- }
- finally {
- busyLock.leaveBusy();
- }
- }
- else
- throw new IllegalStateException("Failed to get job counters (grid is stopping).");
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public IgniteInternalFuture<?> finishFuture(GridHadoopJobId jobId) throws IgniteCheckedException {
- if (busyLock.enterBusy()) {
- try {
- return proc.finishFuture(jobId);
- }
- finally {
- busyLock.leaveBusy();
- }
- }
- else
- throw new IllegalStateException("Failed to get job finish future (grid is stopping).");
- }
-
- /** {@inheritDoc} */
- @Override public boolean kill(GridHadoopJobId jobId) throws IgniteCheckedException {
- if (busyLock.enterBusy()) {
- try {
- return proc.kill(jobId);
- }
- finally {
- busyLock.leaveBusy();
- }
- }
- else
- throw new IllegalStateException("Failed to kill job (grid is stopping).");
- }
-}
[38/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
new file mode 100644
index 0000000..93c0df4
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -0,0 +1,1237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.fs.v1;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.*;
+import org.apache.hadoop.hdfs.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.util.*;
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.igfs.common.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.configuration.FileSystemConfiguration.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*;
+import static org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils.*;
+import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
+
+/**
+ * {@code IGFS} Hadoop 1.x file system driver over file system API. To use
+ * {@code IGFS} as Hadoop file system, you should configure this class
+ * in Hadoop's {@code core-site.xml} as follows:
+ * <pre name="code" class="xml">
+ * <property>
+ * <name>fs.default.name</name>
+ * <value>igfs://ipc</value>
+ * </property>
+ *
+ * <property>
+ * <name>fs.igfs.impl</name>
+ * <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value>
+ * </property>
+ * </pre>
+ * You should also add Ignite JAR and all libraries to Hadoop classpath. To
+ * do this, add following lines to {@code conf/hadoop-env.sh} script in Hadoop
+ * distribution:
+ * <pre name="code" class="bash">
+ * export IGNITE_HOME=/path/to/Ignite/distribution
+ * export HADOOP_CLASSPATH=$IGNITE_HOME/ignite*.jar
+ *
+ * for f in $IGNITE_HOME/libs/*.jar; do
+ * export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f;
+ * done
+ * </pre>
+ * <h1 class="header">Data vs Clients Nodes</h1>
+ * Hadoop needs to use its FileSystem remotely from client nodes as well as directly on
+ * data nodes. Client nodes are responsible for basic file system operations as well as
+ * accessing data nodes remotely. Usually, client nodes are started together
+ * with {@code job-submitter} or {@code job-scheduler} processes, while data nodes are usually
+ * started together with Hadoop {@code task-tracker} processes.
+ * <p>
+ * For sample client and data node configuration refer to {@code config/hadoop/default-config-client.xml}
+ * and {@code config/hadoop/default-config.xml} configuration files in Ignite installation.
+ */
+public class IgniteHadoopFileSystem extends FileSystem {
+ /** Internal property to indicate management connection. */
+ public static final String IGFS_MANAGEMENT = "fs.igfs.management.connection";
+
+ /** Empty array of file block locations. */
+ private static final BlockLocation[] EMPTY_BLOCK_LOCATIONS = new BlockLocation[0];
+
+ /** Empty array of file statuses. */
+ public static final FileStatus[] EMPTY_FILE_STATUS = new FileStatus[0];
+
+ /** Ensures that close routine is invoked at most once. */
+ private final AtomicBoolean closeGuard = new AtomicBoolean();
+
+ /** Grid remote client. */
+ private HadoopIgfsWrapper rmtClient;
+
+ /** User name for each thread. */
+ private final ThreadLocal<String> userName = new ThreadLocal<String>(){
+ /** {@inheritDoc} */
+ @Override protected String initialValue() {
+ return DFLT_USER_NAME;
+ }
+ };
+
+ /** Working directory for each thread. */
+ private final ThreadLocal<Path> workingDir = new ThreadLocal<Path>(){
+ /** {@inheritDoc} */
+ @Override protected Path initialValue() {
+ return getHomeDirectory();
+ }
+ };
+
+ /** Default replication factor. */
+ private short dfltReplication;
+
+ /** Base file system uri. */
+ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
+ private URI uri;
+
+ /** Authority. */
+ private String uriAuthority;
+
+ /** Client logger. */
+ private IgfsLogger clientLog;
+
+ /** Secondary URI string. */
+ private URI secondaryUri;
+
+ /** IGFS mode resolver. */
+ private IgfsModeResolver modeRslvr;
+
+ /** Secondary file system instance. */
+ private FileSystem secondaryFs;
+
+ /** Management connection flag. */
+ private boolean mgmt;
+
+ /** Whether custom sequential reads before prefetch value is provided. */
+ private boolean seqReadsBeforePrefetchOverride;
+
+ /** IGFS group block size. */
+ private long igfsGrpBlockSize;
+
+ /** Flag that controls whether file writes should be colocated. */
+ private boolean colocateFileWrites;
+
+ /** Prefer local writes. */
+ private boolean preferLocFileWrites;
+
+ /** Custom-provided sequential reads before prefetch. */
+ private int seqReadsBeforePrefetch;
+
+ /** The cache was disabled when the instance was creating. */
+ private boolean cacheEnabled;
+
+ /** {@inheritDoc} */
+ @Override public URI getUri() {
+ if (uri == null)
+ throw new IllegalStateException("URI is null (was IgniteHadoopFileSystem properly initialized?).");
+
+ return uri;
+ }
+
+ /**
+ * Enter busy state.
+ *
+ * @throws IOException If file system is stopped.
+ */
+ private void enterBusy() throws IOException {
+ if (closeGuard.get())
+ throw new IOException("File system is stopped.");
+ }
+
+ /**
+ * Leave busy state.
+ */
+ private void leaveBusy() {
+ // No-op.
+ }
+
+ /**
+ * Public setter that can be used by direct users of FS or Visor.
+ *
+ * @param colocateFileWrites Whether all ongoing file writes should be colocated.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void colocateFileWrites(boolean colocateFileWrites) {
+ this.colocateFileWrites = colocateFileWrites;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("ConstantConditions")
+ @Override public void initialize(URI name, Configuration cfg) throws IOException {
+ enterBusy();
+
+ try {
+ if (rmtClient != null)
+ throw new IOException("File system is already initialized: " + rmtClient);
+
+ A.notNull(name, "name");
+ A.notNull(cfg, "cfg");
+
+ super.initialize(name, cfg);
+
+ setConf(cfg);
+
+ String disableCacheName = String.format("fs.%s.impl.disable.cache", name.getScheme());
+
+ cacheEnabled = !cfg.getBoolean(disableCacheName, false);
+
+ mgmt = cfg.getBoolean(IGFS_MANAGEMENT, false);
+
+ if (!IGFS_SCHEME.equals(name.getScheme()))
+ throw new IOException("Illegal file system URI [expected=" + IGFS_SCHEME +
+ "://[name]/[optional_path], actual=" + name + ']');
+
+ uri = name;
+
+ uriAuthority = uri.getAuthority();
+
+ setUser(cfg.get(MRJobConfig.USER_NAME, DFLT_USER_NAME));
+
+ // Override sequential reads before prefetch if needed.
+ seqReadsBeforePrefetch = parameter(cfg, PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH, uriAuthority, 0);
+
+ if (seqReadsBeforePrefetch > 0)
+ seqReadsBeforePrefetchOverride = true;
+
+ // In Ignite replication factor is controlled by data cache affinity.
+ // We use replication factor to force the whole file to be stored on local node.
+ dfltReplication = (short)cfg.getInt("dfs.replication", 3);
+
+ // Get file colocation control flag.
+ colocateFileWrites = parameter(cfg, PARAM_IGFS_COLOCATED_WRITES, uriAuthority, false);
+ preferLocFileWrites = cfg.getBoolean(PARAM_IGFS_PREFER_LOCAL_WRITES, false);
+
+ // Get log directory.
+ String logDirCfg = parameter(cfg, PARAM_IGFS_LOG_DIR, uriAuthority, DFLT_IGFS_LOG_DIR);
+
+ File logDirFile = U.resolveIgnitePath(logDirCfg);
+
+ String logDir = logDirFile != null ? logDirFile.getAbsolutePath() : null;
+
+ rmtClient = new HadoopIgfsWrapper(uriAuthority, logDir, cfg, LOG);
+
+ // Handshake.
+ IgfsHandshakeResponse handshake = rmtClient.handshake(logDir);
+
+ igfsGrpBlockSize = handshake.blockSize();
+
+ IgfsPaths paths = handshake.secondaryPaths();
+
+ // Initialize client logger.
+ Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);
+
+ if (handshake.sampling() != null ? handshake.sampling() : logEnabled) {
+ // Initiate client logger.
+ if (logDir == null)
+ throw new IOException("Failed to resolve log directory: " + logDirCfg);
+
+ Integer batchSize = parameter(cfg, PARAM_IGFS_LOG_BATCH_SIZE, uriAuthority, DFLT_IGFS_LOG_BATCH_SIZE);
+
+ clientLog = IgfsLogger.logger(uriAuthority, handshake.igfsName(), logDir, batchSize);
+ }
+ else
+ clientLog = IgfsLogger.disabledLogger();
+
+ modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes());
+
+ boolean initSecondary = paths.defaultMode() == PROXY;
+
+ if (!initSecondary && paths.pathModes() != null && !paths.pathModes().isEmpty()) {
+ for (T2<IgfsPath, IgfsMode> pathMode : paths.pathModes()) {
+ IgfsMode mode = pathMode.getValue();
+
+ if (mode == PROXY) {
+ initSecondary = true;
+
+ break;
+ }
+ }
+ }
+
+ if (initSecondary) {
+ Map<String, String> props = paths.properties();
+
+ String secUri = props.get(SECONDARY_FS_URI);
+ String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
+
+ try {
+ SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
+
+ secondaryFs = secProvider.createFileSystem();
+ secondaryUri = secProvider.uri();
+ }
+ catch (IOException e) {
+ if (!mgmt)
+ throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
+ else
+ LOG.warn("Visor failed to create secondary file system (operations on paths with PROXY mode " +
+ "will have no effect): " + e.getMessage());
+ }
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void checkPath(Path path) {
+ URI uri = path.toUri();
+
+ if (uri.isAbsolute()) {
+ if (!F.eq(uri.getScheme(), IGFS_SCHEME))
+ throw new InvalidPathException("Wrong path scheme [expected=" + IGFS_SCHEME + ", actual=" +
+ uri.getAuthority() + ']');
+
+ if (!F.eq(uri.getAuthority(), uriAuthority))
+ throw new InvalidPathException("Wrong path authority [expected=" + uriAuthority + ", actual=" +
+ uri.getAuthority() + ']');
+ }
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("deprecation")
+ @Override public short getDefaultReplication() {
+ return dfltReplication;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void finalize() throws Throwable {
+ super.finalize();
+
+ close0();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IOException {
+ if (cacheEnabled && get(getUri(), getConf()) == this)
+ return;
+
+ close0();
+ }
+
+ /**
+ * Closes file system.
+ *
+ * @throws IOException If failed.
+ */
+ private void close0() throws IOException {
+ if (closeGuard.compareAndSet(false, true)) {
+ if (LOG.isDebugEnabled())
+ LOG.debug("File system closed [uri=" + uri + ", endpoint=" + uriAuthority + ']');
+
+ if (rmtClient == null)
+ return;
+
+ super.close();
+
+ rmtClient.close(false);
+
+ if (clientLog.isLogEnabled())
+ clientLog.close();
+
+ if (secondaryFs != null)
+ U.closeQuiet(secondaryFs);
+
+ // Reset initialized resources.
+ uri = null;
+ rmtClient = null;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setTimes(Path p, long mtime, long atime) throws IOException {
+ enterBusy();
+
+ try {
+ A.notNull(p, "p");
+
+ if (mode(p) == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ // No-op for management connection.
+ return;
+ }
+
+ secondaryFs.setTimes(toSecondary(p), mtime, atime);
+ }
+ else {
+ IgfsPath path = convert(p);
+
+ rmtClient.setTimes(path, atime, mtime);
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setPermission(Path p, FsPermission perm) throws IOException {
+ enterBusy();
+
+ try {
+ A.notNull(p, "p");
+
+ if (mode(p) == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ // No-op for management connection.
+ return;
+ }
+
+ secondaryFs.setPermission(toSecondary(p), perm);
+ }
+ else if (rmtClient.update(convert(p), permission(perm)) == null) {
+ throw new IOException("Failed to set file permission (file not found?)" +
+ " [path=" + p + ", perm=" + perm + ']');
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setOwner(Path p, String username, String grpName) throws IOException {
+ A.notNull(p, "p");
+ A.notNull(username, "username");
+ A.notNull(grpName, "grpName");
+
+ enterBusy();
+
+ try {
+ if (mode(p) == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ // No-op for management connection.
+ return;
+ }
+
+ secondaryFs.setOwner(toSecondary(p), username, grpName);
+ }
+ else if (rmtClient.update(convert(p), F.asMap(PROP_USER_NAME, username, PROP_GROUP_NAME, grpName)) == null)
+ throw new IOException("Failed to set file permission (file not found?)" +
+ " [path=" + p + ", userName=" + username + ", groupName=" + grpName + ']');
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public FSDataInputStream open(Path f, int bufSize) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ try {
+ IgfsPath path = convert(f);
+ IgfsMode mode = mode(path);
+
+ if (mode == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ throw new IOException("Failed to open file (secondary file system is not initialized): " + f);
+ }
+
+ FSDataInputStream is = secondaryFs.open(toSecondary(f), bufSize);
+
+ if (clientLog.isLogEnabled()) {
+ // At this point we do not know file size, so we perform additional request to remote FS to get it.
+ FileStatus status = secondaryFs.getFileStatus(toSecondary(f));
+
+ long size = status != null ? status.getLen() : -1;
+
+ long logId = IgfsLogger.nextId();
+
+ clientLog.logOpen(logId, path, PROXY, bufSize, size);
+
+ return new FSDataInputStream(new HadoopIgfsProxyInputStream(is, clientLog, logId));
+ }
+ else
+ return is;
+ }
+ else {
+ HadoopIgfsStreamDelegate stream = seqReadsBeforePrefetchOverride ?
+ rmtClient.open(path, seqReadsBeforePrefetch) : rmtClient.open(path);
+
+ long logId = -1;
+
+ if (clientLog.isLogEnabled()) {
+ logId = IgfsLogger.nextId();
+
+ clientLog.logOpen(logId, path, mode, bufSize, stream.length());
+ }
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Opening input stream [thread=" + Thread.currentThread().getName() + ", path=" + path +
+ ", bufSize=" + bufSize + ']');
+
+ HadoopIgfsInputStream igfsIn = new HadoopIgfsInputStream(stream, stream.length(),
+ bufSize, LOG, clientLog, logId);
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Opened input stream [path=" + path + ", delegate=" + stream + ']');
+
+ return new FSDataInputStream(igfsIn);
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("deprecation")
+ @Override public FSDataOutputStream create(Path f, FsPermission perm, boolean overwrite, int bufSize,
+ short replication, long blockSize, Progressable progress) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ OutputStream out = null;
+
+ try {
+ IgfsPath path = convert(f);
+ IgfsMode mode = mode(path);
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Opening output stream in create [thread=" + Thread.currentThread().getName() + "path=" +
+ path + ", overwrite=" + overwrite + ", bufSize=" + bufSize + ']');
+
+ if (mode == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ throw new IOException("Failed to create file (secondary file system is not initialized): " + f);
+ }
+
+ FSDataOutputStream os =
+ secondaryFs.create(toSecondary(f), perm, overwrite, bufSize, replication, blockSize, progress);
+
+ if (clientLog.isLogEnabled()) {
+ long logId = IgfsLogger.nextId();
+
+ clientLog.logCreate(logId, path, PROXY, overwrite, bufSize, replication, blockSize);
+
+ return new FSDataOutputStream(new HadoopIgfsProxyOutputStream(os, clientLog, logId));
+ }
+ else
+ return os;
+ }
+ else {
+ // Create stream and close it in the 'finally' section if any sequential operation failed.
+ HadoopIgfsStreamDelegate stream = rmtClient.create(path, overwrite, colocateFileWrites,
+ replication, blockSize, F.asMap(PROP_PERMISSION, toString(perm),
+ PROP_PREFER_LOCAL_WRITES, Boolean.toString(preferLocFileWrites)));
+
+ assert stream != null;
+
+ long logId = -1;
+
+ if (clientLog.isLogEnabled()) {
+ logId = IgfsLogger.nextId();
+
+ clientLog.logCreate(logId, path, mode, overwrite, bufSize, replication, blockSize);
+ }
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Opened output stream in create [path=" + path + ", delegate=" + stream + ']');
+
+ HadoopIgfsOutputStream igfsOut = new HadoopIgfsOutputStream(stream, LOG, clientLog,
+ logId);
+
+ bufSize = Math.max(64 * 1024, bufSize);
+
+ out = new BufferedOutputStream(igfsOut, bufSize);
+
+ FSDataOutputStream res = new FSDataOutputStream(out, null, 0);
+
+ // Mark stream created successfully.
+ out = null;
+
+ return res;
+ }
+ }
+ finally {
+ // Close if failed during stream creation.
+ if (out != null)
+ U.closeQuiet(out);
+
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("deprecation")
+ @Override public FSDataOutputStream append(Path f, int bufSize, Progressable progress) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ try {
+ IgfsPath path = convert(f);
+ IgfsMode mode = mode(path);
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Opening output stream in append [thread=" + Thread.currentThread().getName() +
+ ", path=" + path + ", bufSize=" + bufSize + ']');
+
+ if (mode == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ throw new IOException("Failed to append file (secondary file system is not initialized): " + f);
+ }
+
+ FSDataOutputStream os = secondaryFs.append(toSecondary(f), bufSize, progress);
+
+ if (clientLog.isLogEnabled()) {
+ long logId = IgfsLogger.nextId();
+
+ clientLog.logAppend(logId, path, PROXY, bufSize); // Don't have stream ID.
+
+ return new FSDataOutputStream(new HadoopIgfsProxyOutputStream(os, clientLog, logId));
+ }
+ else
+ return os;
+ }
+ else {
+ HadoopIgfsStreamDelegate stream = rmtClient.append(path, false, null);
+
+ assert stream != null;
+
+ long logId = -1;
+
+ if (clientLog.isLogEnabled()) {
+ logId = IgfsLogger.nextId();
+
+ clientLog.logAppend(logId, path, mode, bufSize);
+ }
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Opened output stream in append [path=" + path + ", delegate=" + stream + ']');
+
+ HadoopIgfsOutputStream igfsOut = new HadoopIgfsOutputStream(stream, LOG, clientLog,
+ logId);
+
+ bufSize = Math.max(64 * 1024, bufSize);
+
+ BufferedOutputStream out = new BufferedOutputStream(igfsOut, bufSize);
+
+ return new FSDataOutputStream(out, null, 0);
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public boolean rename(Path src, Path dst) throws IOException {
+ A.notNull(src, "src");
+ A.notNull(dst, "dst");
+
+ enterBusy();
+
+ try {
+ IgfsPath srcPath = convert(src);
+ IgfsPath dstPath = convert(dst);
+ IgfsMode mode = mode(srcPath);
+
+ if (mode == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ return false;
+ }
+
+ if (clientLog.isLogEnabled())
+ clientLog.logRename(srcPath, PROXY, dstPath);
+
+ return secondaryFs.rename(toSecondary(src), toSecondary(dst));
+ }
+ else {
+ // Will throw exception if failed.
+ rmtClient.rename(srcPath, dstPath);
+
+ if (clientLog.isLogEnabled())
+ clientLog.logRename(srcPath, mode, dstPath);
+
+ return true;
+ }
+ }
+ catch (IOException e) {
+ // Intentionally ignore IGFS exceptions here to follow Hadoop contract.
+ if (F.eq(IOException.class, e.getClass()) && (e.getCause() == null ||
+ !X.hasCause(e.getCause(), IgfsException.class)))
+ throw e;
+ else
+ return false;
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("deprecation")
+ @Override public boolean delete(Path f) throws IOException {
+ return delete(f, false);
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public boolean delete(Path f, boolean recursive) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ try {
+ IgfsPath path = convert(f);
+ IgfsMode mode = mode(path);
+
+ if (mode == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ return false;
+ }
+
+ if (clientLog.isLogEnabled())
+ clientLog.logDelete(path, PROXY, recursive);
+
+ return secondaryFs.delete(toSecondary(f), recursive);
+ }
+ else {
+ // Will throw exception if delete failed.
+ boolean res = rmtClient.delete(path, recursive);
+
+ if (clientLog.isLogEnabled())
+ clientLog.logDelete(path, mode, recursive);
+
+ return res;
+ }
+ }
+ catch (IOException e) {
+ // Intentionally ignore IGFS exceptions here to follow Hadoop contract.
+ if (F.eq(IOException.class, e.getClass()) && (e.getCause() == null ||
+ !X.hasCause(e.getCause(), IgfsException.class)))
+ throw e;
+ else
+ return false;
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileStatus[] listStatus(Path f) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ try {
+ IgfsPath path = convert(f);
+ IgfsMode mode = mode(path);
+
+ if (mode == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ return EMPTY_FILE_STATUS;
+ }
+
+ FileStatus[] arr = secondaryFs.listStatus(toSecondary(f));
+
+ if (arr == null)
+ throw new FileNotFoundException("File " + f + " does not exist.");
+
+ for (int i = 0; i < arr.length; i++)
+ arr[i] = toPrimary(arr[i]);
+
+ if (clientLog.isLogEnabled()) {
+ String[] fileArr = new String[arr.length];
+
+ for (int i = 0; i < arr.length; i++)
+ fileArr[i] = arr[i].getPath().toString();
+
+ clientLog.logListDirectory(path, PROXY, fileArr);
+ }
+
+ return arr;
+ }
+ else {
+ Collection<IgfsFile> list = rmtClient.listFiles(path);
+
+ if (list == null)
+ throw new FileNotFoundException("File " + f + " does not exist.");
+
+ List<IgfsFile> files = new ArrayList<>(list);
+
+ FileStatus[] arr = new FileStatus[files.size()];
+
+ for (int i = 0; i < arr.length; i++)
+ arr[i] = convert(files.get(i));
+
+ if (clientLog.isLogEnabled()) {
+ String[] fileArr = new String[arr.length];
+
+ for (int i = 0; i < arr.length; i++)
+ fileArr[i] = arr[i].getPath().toString();
+
+ clientLog.logListDirectory(path, mode, fileArr);
+ }
+
+ return arr;
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Path getHomeDirectory() {
+ Path path = new Path("/user/" + userName.get());
+
+ return path.makeQualified(getUri(), null);
+ }
+
+ /**
+ * Set user name and default working directory for current thread.
+ *
+ * @param userName User name.
+ */
+ public void setUser(String userName) {
+ this.userName.set(userName);
+
+ setWorkingDirectory(null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setWorkingDirectory(Path newPath) {
+ if (newPath == null) {
+ Path homeDir = getHomeDirectory();
+
+ if (secondaryFs != null)
+ secondaryFs.setWorkingDirectory(toSecondary(homeDir));
+
+ workingDir.set(homeDir);
+ }
+ else {
+ Path fixedNewPath = fixRelativePart(newPath);
+
+ String res = fixedNewPath.toUri().getPath();
+
+ if (!DFSUtil.isValidName(res))
+ throw new IllegalArgumentException("Invalid DFS directory name " + res);
+
+ if (secondaryFs != null)
+ secondaryFs.setWorkingDirectory(toSecondary(fixedNewPath));
+
+ workingDir.set(fixedNewPath);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Path getWorkingDirectory() {
+ return workingDir.get();
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public boolean mkdirs(Path f, FsPermission perm) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ try {
+ IgfsPath path = convert(f);
+ IgfsMode mode = mode(path);
+
+ if (mode == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ return false;
+ }
+
+ if (clientLog.isLogEnabled())
+ clientLog.logMakeDirectory(path, PROXY);
+
+ return secondaryFs.mkdirs(toSecondary(f), perm);
+ }
+ else {
+ boolean mkdirRes = rmtClient.mkdirs(path, permission(perm));
+
+ if (clientLog.isLogEnabled())
+ clientLog.logMakeDirectory(path, mode);
+
+ return mkdirRes;
+ }
+ }
+ catch (IOException e) {
+ // Intentionally ignore IGFS exceptions here to follow Hadoop contract.
+ if (F.eq(IOException.class, e.getClass()) && (e.getCause() == null ||
+ !X.hasCause(e.getCause(), IgfsException.class)))
+ throw e;
+ else
+ return false;
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileStatus getFileStatus(Path f) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ try {
+ if (mode(f) == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ throw new IOException("Failed to get file status (secondary file system is not initialized): " + f);
+ }
+
+ return toPrimary(secondaryFs.getFileStatus(toSecondary(f)));
+ }
+ else {
+ IgfsFile info = rmtClient.info(convert(f));
+
+ if (info == null)
+ throw new FileNotFoundException("File not found: " + f);
+
+ return convert(info);
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public ContentSummary getContentSummary(Path f) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ try {
+ if (mode(f) == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ throw new IOException("Failed to get content summary (secondary file system is not initialized): " +
+ f);
+ }
+
+ return secondaryFs.getContentSummary(toSecondary(f));
+ }
+ else {
+ IgfsPathSummary sum = rmtClient.contentSummary(convert(f));
+
+ return new ContentSummary(sum.totalLength(), sum.filesCount(), sum.directoriesCount(),
+ -1, sum.totalLength(), rmtClient.fsStatus().spaceTotal());
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public BlockLocation[] getFileBlockLocations(FileStatus status, long start, long len) throws IOException {
+ A.notNull(status, "status");
+
+ enterBusy();
+
+ try {
+ IgfsPath path = convert(status.getPath());
+
+ if (mode(status.getPath()) == PROXY) {
+ if (secondaryFs == null) {
+ assert mgmt;
+
+ return EMPTY_BLOCK_LOCATIONS;
+ }
+
+ Path secPath = toSecondary(status.getPath());
+
+ return secondaryFs.getFileBlockLocations(secondaryFs.getFileStatus(secPath), start, len);
+ }
+ else {
+ long now = System.currentTimeMillis();
+
+ List<IgfsBlockLocation> affinity = new ArrayList<>(rmtClient.affinity(path, start, len));
+
+ BlockLocation[] arr = new BlockLocation[affinity.size()];
+
+ for (int i = 0; i < arr.length; i++)
+ arr[i] = convert(affinity.get(i));
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Fetched file locations [path=" + path + ", fetchTime=" +
+ (System.currentTimeMillis() - now) + ", locations=" + Arrays.asList(arr) + ']');
+
+ return arr;
+ }
+ }
+ catch (FileNotFoundException ignored) {
+ return EMPTY_BLOCK_LOCATIONS;
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("deprecation")
+ @Override public long getDefaultBlockSize() {
+ return igfsGrpBlockSize;
+ }
+
+ /**
+ * Resolve path mode.
+ *
+ * @param path HDFS path.
+ * @return Path mode.
+ */
+ public IgfsMode mode(Path path) {
+ return mode(convert(path));
+ }
+
+ /**
+ * Resolve path mode.
+ *
+ * @param path IGFS path.
+ * @return Path mode.
+ */
+ public IgfsMode mode(IgfsPath path) {
+ return modeRslvr.resolveMode(path);
+ }
+
+ /**
+ * Convert the given path to path acceptable by the primary file system.
+ *
+ * @param path Path.
+ * @return Primary file system path.
+ */
+ private Path toPrimary(Path path) {
+ return convertPath(path, uri);
+ }
+
+ /**
+ * Convert the given path to path acceptable by the secondary file system.
+ *
+ * @param path Path.
+ * @return Secondary file system path.
+ */
+ private Path toSecondary(Path path) {
+ assert secondaryFs != null;
+ assert secondaryUri != null;
+
+ return convertPath(path, secondaryUri);
+ }
+
+ /**
+ * Convert path using the given new URI.
+ *
+ * @param path Old path.
+ * @param newUri New URI.
+ * @return New path.
+ */
+ private Path convertPath(Path path, URI newUri) {
+ assert newUri != null;
+
+ if (path != null) {
+ URI pathUri = path.toUri();
+
+ try {
+ return new Path(new URI(pathUri.getScheme() != null ? newUri.getScheme() : null,
+ pathUri.getAuthority() != null ? newUri.getAuthority() : null, pathUri.getPath(), null, null));
+ }
+ catch (URISyntaxException e) {
+ throw new IgniteException("Failed to construct secondary file system path from the primary file " +
+ "system path: " + path, e);
+ }
+ }
+ else
+ return null;
+ }
+
+ /**
+ * Convert a file status obtained from the secondary file system to a status of the primary file system.
+ *
+ * @param status Secondary file system status.
+ * @return Primary file system status.
+ */
+ @SuppressWarnings("deprecation")
+ private FileStatus toPrimary(FileStatus status) {
+ return status != null ? new FileStatus(status.getLen(), status.isDir(), status.getReplication(),
+ status.getBlockSize(), status.getModificationTime(), status.getAccessTime(), status.getPermission(),
+ status.getOwner(), status.getGroup(), toPrimary(status.getPath())) : null;
+ }
+
+ /**
+ * Convert IGFS path into Hadoop path.
+ *
+ * @param path IGFS path.
+ * @return Hadoop path.
+ */
+ private Path convert(IgfsPath path) {
+ return new Path(IGFS_SCHEME, uriAuthority, path.toString());
+ }
+
+ /**
+ * Convert Hadoop path into IGFS path.
+ *
+ * @param path Hadoop path.
+ * @return IGFS path.
+ */
+ @Nullable private IgfsPath convert(@Nullable Path path) {
+ if (path == null)
+ return null;
+
+ return path.isAbsolute() ? new IgfsPath(path.toUri().getPath()) :
+ new IgfsPath(convert(workingDir.get()), path.toUri().getPath());
+ }
+
+ /**
+ * Convert IGFS affinity block location into Hadoop affinity block location.
+ *
+ * @param block IGFS affinity block location.
+ * @return Hadoop affinity block location.
+ */
+ private BlockLocation convert(IgfsBlockLocation block) {
+ Collection<String> names = block.names();
+ Collection<String> hosts = block.hosts();
+
+ return new BlockLocation(
+ names.toArray(new String[names.size()]) /* hostname:portNumber of data nodes */,
+ hosts.toArray(new String[hosts.size()]) /* hostnames of data nodes */,
+ block.start(), block.length()
+ ) {
+ @Override public String toString() {
+ try {
+ return "BlockLocation [offset=" + getOffset() + ", length=" + getLength() +
+ ", hosts=" + Arrays.asList(getHosts()) + ", names=" + Arrays.asList(getNames()) + ']';
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+
+ /**
+ * Convert IGFS file information into Hadoop file status.
+ *
+ * @param file IGFS file information.
+ * @return Hadoop file status.
+ */
+ @SuppressWarnings("deprecation")
+ private FileStatus convert(IgfsFile file) {
+ return new FileStatus(file.length(), file.isDirectory(), getDefaultReplication(),
+ file.groupBlockSize(), file.modificationTime(), file.accessTime(), permission(file),
+ file.property(PROP_USER_NAME, DFLT_USER_NAME), file.property(PROP_GROUP_NAME, "users"),
+ convert(file.path())) {
+ @Override public String toString() {
+ return "FileStatus [path=" + getPath() + ", isDir=" + isDir() + ", len=" + getLen() +
+ ", mtime=" + getModificationTime() + ", atime=" + getAccessTime() + ']';
+ }
+ };
+ }
+
+ /**
+ * Convert Hadoop permission into IGFS file attribute.
+ *
+ * @param perm Hadoop permission.
+ * @return IGFS attributes.
+ */
+ private Map<String, String> permission(FsPermission perm) {
+ if (perm == null)
+ perm = FsPermission.getDefault();
+
+ return F.asMap(PROP_PERMISSION, toString(perm));
+ }
+
+ /**
+ * @param perm Permission.
+ * @return String.
+ */
+ private static String toString(FsPermission perm) {
+ return String.format("%04o", perm.toShort());
+ }
+
+ /**
+ * Convert IGFS file attributes into Hadoop permission.
+ *
+ * @param file File info.
+ * @return Hadoop permission.
+ */
+ private FsPermission permission(IgfsFile file) {
+ String perm = file.property(PROP_PERMISSION, null);
+
+ if (perm == null)
+ return FsPermission.getDefault();
+
+ try {
+ return new FsPermission((short)Integer.parseInt(perm, 8));
+ }
+ catch (NumberFormatException ignore) {
+ return FsPermission.getDefault();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgniteHadoopFileSystem.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/package.html
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/package.html b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/package.html
new file mode 100644
index 0000000..69801af
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/package.html
@@ -0,0 +1,24 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<body>
+ <!-- Package description. -->
+ Contains Ignite Hadoop 1.x <code>FileSystem</code> implementation.
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
new file mode 100644
index 0000000..70ad99f
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
@@ -0,0 +1,982 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.fs.v2;
+
+import org.apache.commons.logging.*;
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.permission.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.util.*;
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.igfs.common.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.configuration.FileSystemConfiguration.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*;
+import static org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils.*;
+import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
+
+/**
+ * {@code IGFS} Hadoop 2.x file system driver over file system API. To use
+ * {@code IGFS} as Hadoop file system, you should configure this class
+ * in Hadoop's {@code core-site.xml} as follows:
+ * <pre name="code" class="xml">
+ * <property>
+ * <name>fs.default.name</name>
+ * <value>igfs://ipc</value>
+ * </property>
+ *
+ * <property>
+ * <name>fs.igfs.impl</name>
+ * <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value>
+ * </property>
+ * </pre>
+ * You should also add Ignite JAR and all libraries to Hadoop classpath. To
+ * do this, add following lines to {@code conf/hadoop-env.sh} script in Hadoop
+ * distribution:
+ * <pre name="code" class="bash">
+ * export IGNITE_HOME=/path/to/Ignite/distribution
+ * export HADOOP_CLASSPATH=$IGNITE_HOME/ignite*.jar
+ *
+ * for f in $IGNITE_HOME/libs/*.jar; do
+ * export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f;
+ * done
+ * </pre>
+ * <h1 class="header">Data vs Clients Nodes</h1>
+ * Hadoop needs to use its FileSystem remotely from client nodes as well as directly on
+ * data nodes. Client nodes are responsible for basic file system operations as well as
+ * accessing data nodes remotely. Usually, client nodes are started together
+ * with {@code job-submitter} or {@code job-scheduler} processes, while data nodes are usually
+ * started together with Hadoop {@code task-tracker} processes.
+ * <p>
+ * For sample client and data node configuration refer to {@code config/hadoop/default-config-client.xml}
+ * and {@code config/hadoop/default-config.xml} configuration files in Ignite installation.
+ */
+public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closeable {
+ /** Logger. */
+ private static final Log LOG = LogFactory.getLog(IgniteHadoopFileSystem.class);
+
+ /** Ensures that close routine is invoked at most once. */
+ private final AtomicBoolean closeGuard = new AtomicBoolean();
+
+ /** Grid remote client. */
+ private HadoopIgfsWrapper rmtClient;
+
+ /** Working directory. */
+ private IgfsPath workingDir;
+
+ /** URI. */
+ private URI uri;
+
+ /** Authority. */
+ private String uriAuthority;
+
+ /** Client logger. */
+ private IgfsLogger clientLog;
+
+ /** Server block size. */
+ private long grpBlockSize;
+
+ /** Default replication factor. */
+ private short dfltReplication;
+
+ /** Secondary URI string. */
+ private URI secondaryUri;
+
+ /** Mode resolver. */
+ private IgfsModeResolver modeRslvr;
+
+ /** Secondary file system instance. */
+ private AbstractFileSystem secondaryFs;
+
+ /** Whether custom sequential reads before prefetch value is provided. */
+ private boolean seqReadsBeforePrefetchOverride;
+
+ /** Custom-provided sequential reads before prefetch. */
+ private int seqReadsBeforePrefetch;
+
+ /** Flag that controls whether file writes should be colocated on data node. */
+ private boolean colocateFileWrites;
+
+ /** Prefer local writes. */
+ private boolean preferLocFileWrites;
+
+ /**
+ * @param name URI for file system.
+ * @param cfg Configuration.
+ * @throws URISyntaxException if name has invalid syntax.
+ * @throws IOException If initialization failed.
+ */
+ public IgniteHadoopFileSystem(URI name, Configuration cfg) throws URISyntaxException, IOException {
+ super(HadoopIgfsEndpoint.normalize(name), IGFS_SCHEME, false, -1);
+
+ uri = name;
+
+ try {
+ initialize(name, cfg);
+ }
+ catch (IOException e) {
+ // Close client if exception occurred.
+ if (rmtClient != null)
+ rmtClient.close(false);
+
+ throw e;
+ }
+
+ workingDir = new IgfsPath("/user/" + cfg.get(MRJobConfig.USER_NAME, DFLT_USER_NAME));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void checkPath(Path path) {
+ URI uri = path.toUri();
+
+ if (uri.isAbsolute()) {
+ if (!F.eq(uri.getScheme(), IGFS_SCHEME))
+ throw new InvalidPathException("Wrong path scheme [expected=" + IGFS_SCHEME + ", actual=" +
+ uri.getAuthority() + ']');
+
+ if (!F.eq(uri.getAuthority(), uriAuthority))
+ throw new InvalidPathException("Wrong path authority [expected=" + uriAuthority + ", actual=" +
+ uri.getAuthority() + ']');
+ }
+ }
+
+ /**
+ * Public setter that can be used by direct users of FS or Visor.
+ *
+ * @param colocateFileWrites Whether all ongoing file writes should be colocated.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void colocateFileWrites(boolean colocateFileWrites) {
+ this.colocateFileWrites = colocateFileWrites;
+ }
+
+ /**
+ * Enter busy state.
+ *
+ * @throws IOException If file system is stopped.
+ */
+ private void enterBusy() throws IOException {
+ if (closeGuard.get())
+ throw new IOException("File system is stopped.");
+ }
+
+ /**
+ * Leave busy state.
+ */
+ private void leaveBusy() {
+ // No-op.
+ }
+
+ /**
+ * @param name URI passed to constructor.
+ * @param cfg Configuration passed to constructor.
+ * @throws IOException If initialization failed.
+ */
+ @SuppressWarnings("ConstantConditions")
+ private void initialize(URI name, Configuration cfg) throws IOException {
+ enterBusy();
+
+ try {
+ if (rmtClient != null)
+ throw new IOException("File system is already initialized: " + rmtClient);
+
+ A.notNull(name, "name");
+ A.notNull(cfg, "cfg");
+
+ if (!IGFS_SCHEME.equals(name.getScheme()))
+ throw new IOException("Illegal file system URI [expected=" + IGFS_SCHEME +
+ "://[name]/[optional_path], actual=" + name + ']');
+
+ uriAuthority = name.getAuthority();
+
+ // Override sequential reads before prefetch if needed.
+ seqReadsBeforePrefetch = parameter(cfg, PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH, uriAuthority, 0);
+
+ if (seqReadsBeforePrefetch > 0)
+ seqReadsBeforePrefetchOverride = true;
+
+ // In Ignite replication factor is controlled by data cache affinity.
+ // We use replication factor to force the whole file to be stored on local node.
+ dfltReplication = (short)cfg.getInt("dfs.replication", 3);
+
+ // Get file colocation control flag.
+ colocateFileWrites = parameter(cfg, PARAM_IGFS_COLOCATED_WRITES, uriAuthority, false);
+ preferLocFileWrites = cfg.getBoolean(PARAM_IGFS_PREFER_LOCAL_WRITES, false);
+
+ // Get log directory.
+ String logDirCfg = parameter(cfg, PARAM_IGFS_LOG_DIR, uriAuthority, DFLT_IGFS_LOG_DIR);
+
+ File logDirFile = U.resolveIgnitePath(logDirCfg);
+
+ String logDir = logDirFile != null ? logDirFile.getAbsolutePath() : null;
+
+ rmtClient = new HadoopIgfsWrapper(uriAuthority, logDir, cfg, LOG);
+
+ // Handshake.
+ IgfsHandshakeResponse handshake = rmtClient.handshake(logDir);
+
+ grpBlockSize = handshake.blockSize();
+
+ IgfsPaths paths = handshake.secondaryPaths();
+
+ Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);
+
+ if (handshake.sampling() != null ? handshake.sampling() : logEnabled) {
+ // Initiate client logger.
+ if (logDir == null)
+ throw new IOException("Failed to resolve log directory: " + logDirCfg);
+
+ Integer batchSize = parameter(cfg, PARAM_IGFS_LOG_BATCH_SIZE, uriAuthority, DFLT_IGFS_LOG_BATCH_SIZE);
+
+ clientLog = IgfsLogger.logger(uriAuthority, handshake.igfsName(), logDir, batchSize);
+ }
+ else
+ clientLog = IgfsLogger.disabledLogger();
+
+ modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes());
+
+ boolean initSecondary = paths.defaultMode() == PROXY;
+
+ if (!initSecondary && paths.pathModes() != null) {
+ for (T2<IgfsPath, IgfsMode> pathMode : paths.pathModes()) {
+ IgfsMode mode = pathMode.getValue();
+
+ if (mode == PROXY) {
+ initSecondary = true;
+
+ break;
+ }
+ }
+ }
+
+ if (initSecondary) {
+ Map<String, String> props = paths.properties();
+
+ String secUri = props.get(SECONDARY_FS_URI);
+ String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
+
+ try {
+ SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
+
+ secondaryFs = secProvider.createAbstractFileSystem();
+ secondaryUri = secProvider.uri();
+ }
+ catch (IOException e) {
+ throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
+ }
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IOException {
+ if (closeGuard.compareAndSet(false, true)) {
+ if (rmtClient == null)
+ return;
+
+ rmtClient.close(false);
+
+ if (clientLog.isLogEnabled())
+ clientLog.close();
+
+ // Reset initialized resources.
+ rmtClient = null;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public URI getUri() {
+ return uri;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getUriDefaultPort() {
+ return -1;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FsServerDefaults getServerDefaults() throws IOException {
+ return new FsServerDefaults(grpBlockSize, (int)grpBlockSize, (int)grpBlockSize, dfltReplication, 64 * 1024,
+ false, 0, DataChecksum.Type.NULL);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean setReplication(Path f, short replication) throws IOException {
+ return mode(f) == PROXY && secondaryFs.setReplication(f, replication);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setTimes(Path f, long mtime, long atime) throws IOException {
+ if (mode(f) == PROXY)
+ secondaryFs.setTimes(f, mtime, atime);
+ else {
+ if (mtime == -1 && atime == -1)
+ return;
+
+ rmtClient.setTimes(convert(f), atime, mtime);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public FsStatus getFsStatus() throws IOException {
+ IgfsStatus status = rmtClient.fsStatus();
+
+ return new FsStatus(status.spaceTotal(), status.spaceUsed(), status.spaceTotal() - status.spaceUsed());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setPermission(Path p, FsPermission perm) throws IOException {
+ enterBusy();
+
+ try {
+ A.notNull(p, "p");
+
+ if (mode(p) == PROXY)
+ secondaryFs.setPermission(toSecondary(p), perm);
+ else {
+ if (rmtClient.update(convert(p), permission(perm)) == null)
+ throw new IOException("Failed to set file permission (file not found?)" +
+ " [path=" + p + ", perm=" + perm + ']');
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setOwner(Path p, String usr, String grp) throws IOException {
+ A.notNull(p, "p");
+ A.notNull(usr, "username");
+ A.notNull(grp, "grpName");
+
+ enterBusy();
+
+ try {
+ if (mode(p) == PROXY)
+ secondaryFs.setOwner(toSecondary(p), usr, grp);
+ else if (rmtClient.update(convert(p), F.asMap(PROP_USER_NAME, usr, PROP_GROUP_NAME, grp)) == null)
+ throw new IOException("Failed to set file permission (file not found?)" +
+ " [path=" + p + ", username=" + usr + ", grpName=" + grp + ']');
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public FSDataInputStream open(Path f, int bufSize) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ try {
+ IgfsPath path = convert(f);
+ IgfsMode mode = modeRslvr.resolveMode(path);
+
+ if (mode == PROXY) {
+ FSDataInputStream is = secondaryFs.open(toSecondary(f), bufSize);
+
+ if (clientLog.isLogEnabled()) {
+ // At this point we do not know file size, so we perform additional request to remote FS to get it.
+ FileStatus status = secondaryFs.getFileStatus(toSecondary(f));
+
+ long size = status != null ? status.getLen() : -1;
+
+ long logId = IgfsLogger.nextId();
+
+ clientLog.logOpen(logId, path, PROXY, bufSize, size);
+
+ return new FSDataInputStream(new HadoopIgfsProxyInputStream(is, clientLog, logId));
+ }
+ else
+ return is;
+ }
+ else {
+ HadoopIgfsStreamDelegate stream = seqReadsBeforePrefetchOverride ?
+ rmtClient.open(path, seqReadsBeforePrefetch) : rmtClient.open(path);
+
+ long logId = -1;
+
+ if (clientLog.isLogEnabled()) {
+ logId = IgfsLogger.nextId();
+
+ clientLog.logOpen(logId, path, mode, bufSize, stream.length());
+ }
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Opening input stream [thread=" + Thread.currentThread().getName() + ", path=" + path +
+ ", bufSize=" + bufSize + ']');
+
+ HadoopIgfsInputStream igfsIn = new HadoopIgfsInputStream(stream, stream.length(),
+ bufSize, LOG, clientLog, logId);
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Opened input stream [path=" + path + ", delegate=" + stream + ']');
+
+ return new FSDataInputStream(igfsIn);
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("deprecation")
+ @Override public FSDataOutputStream createInternal(
+ Path f,
+ EnumSet<CreateFlag> flag,
+ FsPermission perm,
+ int bufSize,
+ short replication,
+ long blockSize,
+ Progressable progress,
+ Options.ChecksumOpt checksumOpt,
+ boolean createParent
+ ) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
+ boolean append = flag.contains(CreateFlag.APPEND);
+ boolean create = flag.contains(CreateFlag.CREATE);
+
+ OutputStream out = null;
+
+ try {
+ IgfsPath path = convert(f);
+ IgfsMode mode = modeRslvr.resolveMode(path);
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Opening output stream in create [thread=" + Thread.currentThread().getName() + "path=" +
+ path + ", overwrite=" + overwrite + ", bufSize=" + bufSize + ']');
+
+ if (mode == PROXY) {
+ FSDataOutputStream os = secondaryFs.createInternal(toSecondary(f), flag, perm, bufSize,
+ replication, blockSize, progress, checksumOpt, createParent);
+
+ if (clientLog.isLogEnabled()) {
+ long logId = IgfsLogger.nextId();
+
+ if (append)
+ clientLog.logAppend(logId, path, PROXY, bufSize); // Don't have stream ID.
+ else
+ clientLog.logCreate(logId, path, PROXY, overwrite, bufSize, replication, blockSize);
+
+ return new FSDataOutputStream(new HadoopIgfsProxyOutputStream(os, clientLog, logId));
+ }
+ else
+ return os;
+ }
+ else {
+ Map<String, String> permMap = F.asMap(PROP_PERMISSION, toString(perm),
+ PROP_PREFER_LOCAL_WRITES, Boolean.toString(preferLocFileWrites));
+
+ // Create stream and close it in the 'finally' section if any sequential operation failed.
+ HadoopIgfsStreamDelegate stream;
+
+ long logId = -1;
+
+ if (append) {
+ stream = rmtClient.append(path, create, permMap);
+
+ if (clientLog.isLogEnabled()) {
+ logId = IgfsLogger.nextId();
+
+ clientLog.logAppend(logId, path, mode, bufSize);
+ }
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Opened output stream in append [path=" + path + ", delegate=" + stream + ']');
+ }
+ else {
+ stream = rmtClient.create(path, overwrite, colocateFileWrites, replication, blockSize,
+ permMap);
+
+ if (clientLog.isLogEnabled()) {
+ logId = IgfsLogger.nextId();
+
+ clientLog.logCreate(logId, path, mode, overwrite, bufSize, replication, blockSize);
+ }
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Opened output stream in create [path=" + path + ", delegate=" + stream + ']');
+ }
+
+ assert stream != null;
+
+ HadoopIgfsOutputStream igfsOut = new HadoopIgfsOutputStream(stream, LOG,
+ clientLog, logId);
+
+ bufSize = Math.max(64 * 1024, bufSize);
+
+ out = new BufferedOutputStream(igfsOut, bufSize);
+
+ FSDataOutputStream res = new FSDataOutputStream(out, null, 0);
+
+ // Mark stream created successfully.
+ out = null;
+
+ return res;
+ }
+ }
+ finally {
+ // Close if failed during stream creation.
+ if (out != null)
+ U.closeQuiet(out);
+
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean supportsSymlinks() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void renameInternal(Path src, Path dst) throws IOException {
+ A.notNull(src, "src");
+ A.notNull(dst, "dst");
+
+ enterBusy();
+
+ try {
+ IgfsPath srcPath = convert(src);
+ IgfsPath dstPath = convert(dst);
+ Set<IgfsMode> childrenModes = modeRslvr.resolveChildrenModes(srcPath);
+
+ if (childrenModes.contains(PROXY)) {
+ if (clientLog.isLogEnabled())
+ clientLog.logRename(srcPath, PROXY, dstPath);
+
+ secondaryFs.renameInternal(toSecondary(src), toSecondary(dst));
+ }
+
+ rmtClient.rename(srcPath, dstPath);
+
+ if (clientLog.isLogEnabled())
+ clientLog.logRename(srcPath, modeRslvr.resolveMode(srcPath), dstPath);
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean delete(Path f, boolean recursive) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ try {
+ IgfsPath path = convert(f);
+ IgfsMode mode = modeRslvr.resolveMode(path);
+ Set<IgfsMode> childrenModes = modeRslvr.resolveChildrenModes(path);
+
+ if (childrenModes.contains(PROXY)) {
+ if (clientLog.isLogEnabled())
+ clientLog.logDelete(path, PROXY, recursive);
+
+ return secondaryFs.delete(toSecondary(f), recursive);
+ }
+
+ boolean res = rmtClient.delete(path, recursive);
+
+ if (clientLog.isLogEnabled())
+ clientLog.logDelete(path, mode, recursive);
+
+ return res;
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setVerifyChecksum(boolean verifyChecksum) throws IOException {
+ // Checksum has effect for secondary FS only.
+ if (secondaryFs != null)
+ secondaryFs.setVerifyChecksum(verifyChecksum);
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileChecksum getFileChecksum(Path f) throws IOException {
+ if (mode(f) == PROXY)
+ return secondaryFs.getFileChecksum(f);
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileStatus[] listStatus(Path f) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ try {
+ IgfsPath path = convert(f);
+ IgfsMode mode = modeRslvr.resolveMode(path);
+
+ if (mode == PROXY) {
+ FileStatus[] arr = secondaryFs.listStatus(toSecondary(f));
+
+ if (arr == null)
+ throw new FileNotFoundException("File " + f + " does not exist.");
+
+ for (int i = 0; i < arr.length; i++)
+ arr[i] = toPrimary(arr[i]);
+
+ if (clientLog.isLogEnabled()) {
+ String[] fileArr = new String[arr.length];
+
+ for (int i = 0; i < arr.length; i++)
+ fileArr[i] = arr[i].getPath().toString();
+
+ clientLog.logListDirectory(path, PROXY, fileArr);
+ }
+
+ return arr;
+ }
+ else {
+ Collection<IgfsFile> list = rmtClient.listFiles(path);
+
+ if (list == null)
+ throw new FileNotFoundException("File " + f + " does not exist.");
+
+ List<IgfsFile> files = new ArrayList<>(list);
+
+ FileStatus[] arr = new FileStatus[files.size()];
+
+ for (int i = 0; i < arr.length; i++)
+ arr[i] = convert(files.get(i));
+
+ if (clientLog.isLogEnabled()) {
+ String[] fileArr = new String[arr.length];
+
+ for (int i = 0; i < arr.length; i++)
+ fileArr[i] = arr[i].getPath().toString();
+
+ clientLog.logListDirectory(path, mode, fileArr);
+ }
+
+ return arr;
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdir(Path f, FsPermission perm, boolean createParent) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ try {
+ IgfsPath path = convert(f);
+ IgfsMode mode = modeRslvr.resolveMode(path);
+
+ if (mode == PROXY) {
+ if (clientLog.isLogEnabled())
+ clientLog.logMakeDirectory(path, PROXY);
+
+ secondaryFs.mkdir(toSecondary(f), perm, createParent);
+ }
+ else {
+ rmtClient.mkdirs(path, permission(perm));
+
+ if (clientLog.isLogEnabled())
+ clientLog.logMakeDirectory(path, mode);
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileStatus getFileStatus(Path f) throws IOException {
+ A.notNull(f, "f");
+
+ enterBusy();
+
+ try {
+ if (mode(f) == PROXY)
+ return toPrimary(secondaryFs.getFileStatus(toSecondary(f)));
+ else {
+ IgfsFile info = rmtClient.info(convert(f));
+
+ if (info == null)
+ throw new FileNotFoundException("File not found: " + f);
+
+ return convert(info);
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public BlockLocation[] getFileBlockLocations(Path path, long start, long len) throws IOException {
+ A.notNull(path, "path");
+
+ IgfsPath igfsPath = convert(path);
+
+ enterBusy();
+
+ try {
+ if (modeRslvr.resolveMode(igfsPath) == PROXY)
+ return secondaryFs.getFileBlockLocations(path, start, len);
+ else {
+ long now = System.currentTimeMillis();
+
+ List<IgfsBlockLocation> affinity = new ArrayList<>(
+ rmtClient.affinity(igfsPath, start, len));
+
+ BlockLocation[] arr = new BlockLocation[affinity.size()];
+
+ for (int i = 0; i < arr.length; i++)
+ arr[i] = convert(affinity.get(i));
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Fetched file locations [path=" + path + ", fetchTime=" +
+ (System.currentTimeMillis() - now) + ", locations=" + Arrays.asList(arr) + ']');
+
+ return arr;
+ }
+ }
+ finally {
+ leaveBusy();
+ }
+ }
+
+ /**
+ * Resolve path mode.
+ *
+ * @param path HDFS path.
+ * @return Path mode.
+ */
+ public IgfsMode mode(Path path) {
+ return modeRslvr.resolveMode(convert(path));
+ }
+
+ /**
+ * Convert the given path to path acceptable by the primary file system.
+ *
+ * @param path Path.
+ * @return Primary file system path.
+ */
+ private Path toPrimary(Path path) {
+ return convertPath(path, getUri());
+ }
+
+ /**
+ * Convert the given path to path acceptable by the secondary file system.
+ *
+ * @param path Path.
+ * @return Secondary file system path.
+ */
+ private Path toSecondary(Path path) {
+ assert secondaryFs != null;
+ assert secondaryUri != null;
+
+ return convertPath(path, secondaryUri);
+ }
+
+ /**
+ * Convert path using the given new URI.
+ *
+ * @param path Old path.
+ * @param newUri New URI.
+ * @return New path.
+ */
+ private Path convertPath(Path path, URI newUri) {
+ assert newUri != null;
+
+ if (path != null) {
+ URI pathUri = path.toUri();
+
+ try {
+ return new Path(new URI(pathUri.getScheme() != null ? newUri.getScheme() : null,
+ pathUri.getAuthority() != null ? newUri.getAuthority() : null, pathUri.getPath(), null, null));
+ }
+ catch (URISyntaxException e) {
+ throw new IgniteException("Failed to construct secondary file system path from the primary file " +
+ "system path: " + path, e);
+ }
+ }
+ else
+ return null;
+ }
+
+ /**
+ * Convert a file status obtained from the secondary file system to a status of the primary file system.
+ *
+ * @param status Secondary file system status.
+ * @return Primary file system status.
+ */
+ private FileStatus toPrimary(FileStatus status) {
+ return status != null ? new FileStatus(status.getLen(), status.isDirectory(), status.getReplication(),
+ status.getBlockSize(), status.getModificationTime(), status.getAccessTime(), status.getPermission(),
+ status.getOwner(), status.getGroup(), toPrimary(status.getPath())) : null;
+ }
+
+ /**
+ * Convert IGFS path into Hadoop path.
+ *
+ * @param path IGFS path.
+ * @return Hadoop path.
+ */
+ private Path convert(IgfsPath path) {
+ return new Path(IGFS_SCHEME, uriAuthority, path.toString());
+ }
+
+ /**
+ * Convert Hadoop path into IGFS path.
+ *
+ * @param path Hadoop path.
+ * @return IGFS path.
+ */
+ @Nullable private IgfsPath convert(Path path) {
+ if (path == null)
+ return null;
+
+ return path.isAbsolute() ? new IgfsPath(path.toUri().getPath()) :
+ new IgfsPath(workingDir, path.toUri().getPath());
+ }
+
+ /**
+ * Convert IGFS affinity block location into Hadoop affinity block location.
+ *
+ * @param block IGFS affinity block location.
+ * @return Hadoop affinity block location.
+ */
+ private BlockLocation convert(IgfsBlockLocation block) {
+ Collection<String> names = block.names();
+ Collection<String> hosts = block.hosts();
+
+ return new BlockLocation(
+ names.toArray(new String[names.size()]) /* hostname:portNumber of data nodes */,
+ hosts.toArray(new String[hosts.size()]) /* hostnames of data nodes */,
+ block.start(), block.length()
+ ) {
+ @Override public String toString() {
+ try {
+ return "BlockLocation [offset=" + getOffset() + ", length=" + getLength() +
+ ", hosts=" + Arrays.asList(getHosts()) + ", names=" + Arrays.asList(getNames()) + ']';
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+
+ /**
+ * Convert IGFS file information into Hadoop file status.
+ *
+ * @param file IGFS file information.
+ * @return Hadoop file status.
+ */
+ private FileStatus convert(IgfsFile file) {
+ return new FileStatus(
+ file.length(),
+ file.isDirectory(),
+ dfltReplication,
+ file.groupBlockSize(),
+ file.modificationTime(),
+ file.accessTime(),
+ permission(file),
+ file.property(PROP_USER_NAME, DFLT_USER_NAME),
+ file.property(PROP_GROUP_NAME, "users"),
+ convert(file.path())) {
+ @Override public String toString() {
+ return "FileStatus [path=" + getPath() + ", isDir=" + isDirectory() + ", len=" + getLen() + "]";
+ }
+ };
+ }
+
+ /**
+ * Convert Hadoop permission into IGFS file attribute.
+ *
+ * @param perm Hadoop permission.
+ * @return IGFS attributes.
+ */
+ private Map<String, String> permission(FsPermission perm) {
+ if (perm == null)
+ perm = FsPermission.getDefault();
+
+ return F.asMap(PROP_PERMISSION, toString(perm));
+ }
+
+ /**
+ * @param perm Permission.
+ * @return String.
+ */
+ private static String toString(FsPermission perm) {
+ return String.format("%04o", perm.toShort());
+ }
+
+ /**
+ * Convert IGFS file attributes into Hadoop permission.
+ *
+ * @param file File info.
+ * @return Hadoop permission.
+ */
+ private FsPermission permission(IgfsFile file) {
+ String perm = file.property(PROP_PERMISSION, null);
+
+ if (perm == null)
+ return FsPermission.getDefault();
+
+ try {
+ return new FsPermission((short)Integer.parseInt(perm, 8));
+ }
+ catch (NumberFormatException ignore) {
+ return FsPermission.getDefault();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgniteHadoopFileSystem.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/package.html
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/package.html b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/package.html
new file mode 100644
index 0000000..8adac96
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/package.html
@@ -0,0 +1,24 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<body>
+ <!-- Package description. -->
+ Contains Ignite Hadoop 2.x <code>FileSystem</code> implementation.
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java
new file mode 100644
index 0000000..83053ce
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.mapreduce;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.protocol.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.client.*;
+import org.apache.ignite.internal.client.marshaller.optimized.*;
+import org.apache.ignite.internal.processors.hadoop.proto.*;
+import org.apache.ignite.internal.util.future.*;
+import org.apache.ignite.internal.util.typedef.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.internal.client.GridClientProtocol.*;
+import static org.apache.ignite.internal.processors.hadoop.proto.HadoopClientProtocol.*;
+
+
+/**
+ * Ignite Hadoop client protocol provider.
+ */
+public class IgniteHadoopClientProtocolProvider extends ClientProtocolProvider {
+ /** Clients. */
+ private static final ConcurrentHashMap<String, IgniteInternalFuture<GridClient>> cliMap = new ConcurrentHashMap<>();
+
+ /** {@inheritDoc} */
+ @Override public ClientProtocol create(Configuration conf) throws IOException {
+ if (FRAMEWORK_NAME.equals(conf.get(MRConfig.FRAMEWORK_NAME))) {
+ String addr = conf.get(MRConfig.MASTER_ADDRESS);
+
+ if (F.isEmpty(addr))
+ throw new IOException("Failed to create client protocol because server address is not specified (is " +
+ MRConfig.MASTER_ADDRESS + " property set?).");
+
+ if (F.eq(addr, "local"))
+ throw new IOException("Local execution mode is not supported, please point " +
+ MRConfig.MASTER_ADDRESS + " to real Ignite node.");
+
+ return createProtocol(addr, conf);
+ }
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public ClientProtocol create(InetSocketAddress addr, Configuration conf) throws IOException {
+ if (FRAMEWORK_NAME.equals(conf.get(MRConfig.FRAMEWORK_NAME)))
+ return createProtocol(addr.getHostString() + ":" + addr.getPort(), conf);
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close(ClientProtocol cliProto) throws IOException {
+ // No-op.
+ }
+
+ /**
+ * Internal protocol creation routine.
+ *
+ * @param addr Address.
+ * @param conf Configuration.
+ * @return Client protocol.
+ * @throws IOException If failed.
+ */
+ private static ClientProtocol createProtocol(String addr, Configuration conf) throws IOException {
+ return new HadoopClientProtocol(conf, client(addr));
+ }
+
+ /**
+ * Create client.
+ *
+ * @param addr Endpoint address.
+ * @return Client.
+ * @throws IOException If failed.
+ */
+ private static GridClient client(String addr) throws IOException {
+ try {
+ IgniteInternalFuture<GridClient> fut = cliMap.get(addr);
+
+ if (fut == null) {
+ GridFutureAdapter<GridClient> fut0 = new GridFutureAdapter<>();
+
+ IgniteInternalFuture<GridClient> oldFut = cliMap.putIfAbsent(addr, fut0);
+
+ if (oldFut != null)
+ return oldFut.get();
+ else {
+ GridClientConfiguration cliCfg = new GridClientConfiguration();
+
+ cliCfg.setProtocol(TCP);
+ cliCfg.setServers(Collections.singletonList(addr));
+ cliCfg.setMarshaller(new GridClientOptimizedMarshaller());
+ cliCfg.setDaemon(true);
+
+ try {
+ GridClient cli = GridClientFactory.start(cliCfg);
+
+ fut0.onDone(cli);
+
+ return cli;
+ }
+ catch (GridClientException e) {
+ fut0.onDone(e);
+
+ throw new IOException("Failed to establish connection with Ignite node: " + addr, e);
+ }
+ }
+ }
+ else
+ return fut.get();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException("Failed to establish connection with Ignite node: " + addr, e);
+ }
+ }
+}
[25/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
new file mode 100644
index 0000000..422d941
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.*;
+import org.apache.ignite.cluster.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.util.future.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+/**
+ * Shuffle.
+ */
+public class HadoopShuffle extends HadoopComponent {
+ /** */
+ private final ConcurrentMap<HadoopJobId, HadoopShuffleJob<UUID>> jobs = new ConcurrentHashMap<>();
+
+ /** */
+ protected final GridUnsafeMemory mem = new GridUnsafeMemory(0);
+
+ /** {@inheritDoc} */
+ @Override public void start(HadoopContext ctx) throws IgniteCheckedException {
+ super.start(ctx);
+
+ ctx.kernalContext().io().addUserMessageListener(GridTopic.TOPIC_HADOOP,
+ new IgniteBiPredicate<UUID, Object>() {
+ @Override public boolean apply(UUID nodeId, Object msg) {
+ return onMessageReceived(nodeId, (HadoopMessage)msg);
+ }
+ });
+ }
+
+ /**
+ * Stops shuffle.
+ *
+ * @param cancel If should cancel all ongoing activities.
+ */
+ @Override public void stop(boolean cancel) {
+ for (HadoopShuffleJob job : jobs.values()) {
+ try {
+ job.close();
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to close job.", e);
+ }
+ }
+
+ jobs.clear();
+ }
+
+ /**
+ * Creates new shuffle job.
+ *
+ * @param jobId Job ID.
+ * @return Created shuffle job.
+ * @throws IgniteCheckedException If job creation failed.
+ */
+ private HadoopShuffleJob<UUID> newJob(HadoopJobId jobId) throws IgniteCheckedException {
+ HadoopMapReducePlan plan = ctx.jobTracker().plan(jobId);
+
+ HadoopShuffleJob<UUID> job = new HadoopShuffleJob<>(ctx.localNodeId(), log,
+ ctx.jobTracker().job(jobId, null), mem, plan.reducers(), plan.reducers(ctx.localNodeId()));
+
+ UUID[] rdcAddrs = new UUID[plan.reducers()];
+
+ for (int i = 0; i < rdcAddrs.length; i++) {
+ UUID nodeId = plan.nodeForReducer(i);
+
+ assert nodeId != null : "Plan is missing node for reducer [plan=" + plan + ", rdc=" + i + ']';
+
+ rdcAddrs[i] = nodeId;
+ }
+
+ boolean init = job.initializeReduceAddresses(rdcAddrs);
+
+ assert init;
+
+ return job;
+ }
+
+ /**
+ * @param nodeId Node ID to send message to.
+ * @param msg Message to send.
+ * @throws IgniteCheckedException If send failed.
+ */
+ private void send0(UUID nodeId, Object msg) throws IgniteCheckedException {
+ ClusterNode node = ctx.kernalContext().discovery().node(nodeId);
+
+ ctx.kernalContext().io().sendUserMessage(F.asList(node), msg, GridTopic.TOPIC_HADOOP, false, 0);
+ }
+
+ /**
+ * @param jobId Task info.
+ * @return Shuffle job.
+ */
+ private HadoopShuffleJob<UUID> job(HadoopJobId jobId) throws IgniteCheckedException {
+ HadoopShuffleJob<UUID> res = jobs.get(jobId);
+
+ if (res == null) {
+ res = newJob(jobId);
+
+ HadoopShuffleJob<UUID> old = jobs.putIfAbsent(jobId, res);
+
+ if (old != null) {
+ res.close();
+
+ res = old;
+ }
+ else if (res.reducersInitialized())
+ startSending(res);
+ }
+
+ return res;
+ }
+
+ /**
+ * Starts message sending thread.
+ *
+ * @param shuffleJob Job to start sending for.
+ */
+ private void startSending(HadoopShuffleJob<UUID> shuffleJob) {
+ shuffleJob.startSending(ctx.kernalContext().gridName(),
+ new IgniteInClosure2X<UUID, HadoopShuffleMessage>() {
+ @Override public void applyx(UUID dest, HadoopShuffleMessage msg) throws IgniteCheckedException {
+ send0(dest, msg);
+ }
+ }
+ );
+ }
+
+ /**
+ * Message received callback.
+ *
+ * @param src Sender node ID.
+ * @param msg Received message.
+ * @return {@code True}.
+ */
+ public boolean onMessageReceived(UUID src, HadoopMessage msg) {
+ if (msg instanceof HadoopShuffleMessage) {
+ HadoopShuffleMessage m = (HadoopShuffleMessage)msg;
+
+ try {
+ job(m.jobId()).onShuffleMessage(m);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Message handling failed.", e);
+ }
+
+ try {
+ // Reply with ack.
+ send0(src, new HadoopShuffleAck(m.id(), m.jobId()));
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to reply back to shuffle message sender [snd=" + src + ", msg=" + msg + ']', e);
+ }
+ }
+ else if (msg instanceof HadoopShuffleAck) {
+ HadoopShuffleAck m = (HadoopShuffleAck)msg;
+
+ try {
+ job(m.jobId()).onShuffleAck(m);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Message handling failed.", e);
+ }
+ }
+ else
+ throw new IllegalStateException("Unknown message type received to Hadoop shuffle [src=" + src +
+ ", msg=" + msg + ']');
+
+ return true;
+ }
+
+ /**
+ * @param taskCtx Task info.
+ * @return Output.
+ */
+ public HadoopTaskOutput output(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ return job(taskCtx.taskInfo().jobId()).output(taskCtx);
+ }
+
+ /**
+ * @param taskCtx Task info.
+ * @return Input.
+ */
+ public HadoopTaskInput input(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ return job(taskCtx.taskInfo().jobId()).input(taskCtx);
+ }
+
+ /**
+ * @param jobId Job id.
+ */
+ public void jobFinished(HadoopJobId jobId) {
+ HadoopShuffleJob job = jobs.remove(jobId);
+
+ if (job != null) {
+ try {
+ job.close();
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to close job: " + jobId, e);
+ }
+ }
+ }
+
+ /**
+ * Flushes all the outputs for the given job to remote nodes.
+ *
+ * @param jobId Job ID.
+ * @return Future.
+ */
+ public IgniteInternalFuture<?> flush(HadoopJobId jobId) {
+ HadoopShuffleJob job = jobs.get(jobId);
+
+ if (job == null)
+ return new GridFinishedFutureEx<>();
+
+ try {
+ return job.flush();
+ }
+ catch (IgniteCheckedException e) {
+ return new GridFinishedFutureEx<>(e);
+ }
+ }
+
+ /**
+ * @return Memory.
+ */
+ public GridUnsafeMemory memory() {
+ return mem;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
new file mode 100644
index 0000000..49cbd65
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.util.tostring.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+
+/**
+ * Acknowledgement message.
+ */
+public class HadoopShuffleAck implements HadoopMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ @GridToStringInclude
+ private long msgId;
+
+ /** */
+ @GridToStringInclude
+ private HadoopJobId jobId;
+
+ /**
+ *
+ */
+ public HadoopShuffleAck() {
+ // No-op.
+ }
+
+ /**
+ * @param msgId Message ID.
+ */
+ public HadoopShuffleAck(long msgId, HadoopJobId jobId) {
+ assert jobId != null;
+
+ this.msgId = msgId;
+ this.jobId = jobId;
+ }
+
+ /**
+ * @return Message ID.
+ */
+ public long id() {
+ return msgId;
+ }
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ jobId.writeExternal(out);
+ out.writeLong(msgId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobId = new HadoopJobId();
+
+ jobId.readExternal(in);
+ msgId = in.readLong();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopShuffleAck.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
new file mode 100644
index 0000000..7ae52df
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -0,0 +1,593 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.apache.ignite.internal.processors.hadoop.shuffle.collections.*;
+import org.apache.ignite.internal.util.future.*;
+import org.apache.ignite.internal.util.io.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.internal.util.worker.*;
+import org.apache.ignite.lang.*;
+import org.apache.ignite.thread.*;
+
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.*;
+import static org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory.*;
+
+/**
+ * Shuffle job.
+ */
+public class HadoopShuffleJob<T> implements AutoCloseable {
+ /** */
+ private static final int MSG_BUF_SIZE = 128 * 1024;
+
+ /** */
+ private final HadoopJob job;
+
+ /** */
+ private final GridUnsafeMemory mem;
+
+ /** */
+ private final boolean needPartitioner;
+
+ /** Collection of task contexts for each reduce task. */
+ private final Map<Integer, HadoopTaskContext> reducersCtx = new HashMap<>();
+
+ /** Reducers addresses. */
+ private T[] reduceAddrs;
+
+ /** Local reducers address. */
+ private final T locReduceAddr;
+
+ /** */
+ private final HadoopShuffleMessage[] msgs;
+
+ /** */
+ private final AtomicReferenceArray<HadoopMultimap> maps;
+
+ /** */
+ private volatile IgniteInClosure2X<T, HadoopShuffleMessage> io;
+
+ /** */
+ protected ConcurrentMap<Long, IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapterEx<?>>> sentMsgs =
+ new ConcurrentHashMap<>();
+
+ /** */
+ private volatile GridWorker snd;
+
+ /** Latch for remote addresses waiting. */
+ private final CountDownLatch ioInitLatch = new CountDownLatch(1);
+
+ /** Finished flag. Set on flush or close. */
+ private volatile boolean flushed;
+
+ /** */
+ private final IgniteLogger log;
+
+ /**
+ * @param locReduceAddr Local reducer address.
+ * @param log Logger.
+ * @param job Job.
+ * @param mem Memory.
+ * @param totalReducerCnt Amount of reducers in the Job.
+ * @param locReducers Reducers will work on current node.
+ * @throws IgniteCheckedException If error.
+ */
+ public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUnsafeMemory mem,
+ int totalReducerCnt, int[] locReducers) throws IgniteCheckedException {
+ this.locReduceAddr = locReduceAddr;
+ this.job = job;
+ this.mem = mem;
+ this.log = log.getLogger(HadoopShuffleJob.class);
+
+ if (!F.isEmpty(locReducers)) {
+ for (int rdc : locReducers) {
+ HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.REDUCE, job.id(), rdc, 0, null);
+
+ reducersCtx.put(rdc, job.getTaskContext(taskInfo));
+ }
+ }
+
+ needPartitioner = totalReducerCnt > 1;
+
+ maps = new AtomicReferenceArray<>(totalReducerCnt);
+ msgs = new HadoopShuffleMessage[totalReducerCnt];
+ }
+
+ /**
+ * @param reduceAddrs Addresses of reducers.
+ * @return {@code True} if addresses were initialized by this call.
+ */
+ public boolean initializeReduceAddresses(T[] reduceAddrs) {
+ if (this.reduceAddrs == null) {
+ this.reduceAddrs = reduceAddrs;
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @return {@code True} if reducers addresses were initialized.
+ */
+ public boolean reducersInitialized() {
+ return reduceAddrs != null;
+ }
+
+ /**
+ * @param gridName Grid name.
+ * @param io IO Closure for sending messages.
+ */
+ @SuppressWarnings("BusyWait")
+ public void startSending(String gridName, IgniteInClosure2X<T, HadoopShuffleMessage> io) {
+ assert snd == null;
+ assert io != null;
+
+ this.io = io;
+
+ if (!flushed) {
+ snd = new GridWorker(gridName, "hadoop-shuffle-" + job.id(), log) {
+ @Override protected void body() throws InterruptedException {
+ try {
+ while (!isCancelled()) {
+ Thread.sleep(5);
+
+ collectUpdatesAndSend(false);
+ }
+ }
+ catch (IgniteCheckedException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ };
+
+ new IgniteThread(snd).start();
+ }
+
+ ioInitLatch.countDown();
+ }
+
+ /**
+ * @param maps Maps.
+ * @param idx Index.
+ * @return Map.
+ */
+ private HadoopMultimap getOrCreateMap(AtomicReferenceArray<HadoopMultimap> maps, int idx) {
+ HadoopMultimap map = maps.get(idx);
+
+ if (map == null) { // Create new map.
+ map = get(job.info(), SHUFFLE_REDUCER_NO_SORTING, false) ?
+ new HadoopConcurrentHashMultimap(job.info(), mem, get(job.info(), PARTITION_HASHMAP_SIZE, 8 * 1024)):
+ new HadoopSkipList(job.info(), mem);
+
+ if (!maps.compareAndSet(idx, null, map)) {
+ map.close();
+
+ return maps.get(idx);
+ }
+ }
+
+ return map;
+ }
+
+ /**
+ * @param msg Message.
+ * @throws IgniteCheckedException Exception.
+ */
+ public void onShuffleMessage(HadoopShuffleMessage msg) throws IgniteCheckedException {
+ assert msg.buffer() != null;
+ assert msg.offset() > 0;
+
+ HadoopTaskContext taskCtx = reducersCtx.get(msg.reducer());
+
+ HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
+
+ perfCntr.onShuffleMessage(msg.reducer(), U.currentTimeMillis());
+
+ HadoopMultimap map = getOrCreateMap(maps, msg.reducer());
+
+ // Add data from message to the map.
+ try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
+ final GridUnsafeDataInput dataInput = new GridUnsafeDataInput();
+ final UnsafeValue val = new UnsafeValue(msg.buffer());
+
+ msg.visit(new HadoopShuffleMessage.Visitor() {
+ /** */
+ private HadoopMultimap.Key key;
+
+ @Override public void onKey(byte[] buf, int off, int len) throws IgniteCheckedException {
+ dataInput.bytes(buf, off, off + len);
+
+ key = adder.addKey(dataInput, key);
+ }
+
+ @Override public void onValue(byte[] buf, int off, int len) {
+ val.off = off;
+ val.size = len;
+
+ key.add(val);
+ }
+ });
+ }
+ }
+
+ /**
+ * @param ack Shuffle ack.
+ */
+ @SuppressWarnings("ConstantConditions")
+ public void onShuffleAck(HadoopShuffleAck ack) {
+ IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapterEx<?>> tup = sentMsgs.get(ack.id());
+
+ if (tup != null)
+ tup.get2().onDone();
+ else
+ log.warning("Received shuffle ack for not registered shuffle id: " + ack);
+ }
+
+ /**
+ * Unsafe value.
+ */
+ private static class UnsafeValue implements HadoopMultimap.Value {
+ /** */
+ private final byte[] buf;
+
+ /** */
+ private int off;
+
+ /** */
+ private int size;
+
+ /**
+ * @param buf Buffer.
+ */
+ private UnsafeValue(byte[] buf) {
+ assert buf != null;
+
+ this.buf = buf;
+ }
+
+ /** */
+ @Override public int size() {
+ return size;
+ }
+
+ /** */
+ @Override public void copyTo(long ptr) {
+ UNSAFE.copyMemory(buf, BYTE_ARR_OFF + off, null, ptr, size);
+ }
+ }
+
+ /**
+ * Sends map updates to remote reducers.
+ */
+ private void collectUpdatesAndSend(boolean flush) throws IgniteCheckedException {
+ for (int i = 0; i < maps.length(); i++) {
+ HadoopMultimap map = maps.get(i);
+
+ if (map == null || locReduceAddr.equals(reduceAddrs[i]))
+ continue; // Skip empty map and local node.
+
+ if (msgs[i] == null)
+ msgs[i] = new HadoopShuffleMessage(job.id(), i, MSG_BUF_SIZE);
+
+ final int idx = i;
+
+ map.visit(false, new HadoopMultimap.Visitor() {
+ /** */
+ private long keyPtr;
+
+ /** */
+ private int keySize;
+
+ /** */
+ private boolean keyAdded;
+
+ /** {@inheritDoc} */
+ @Override public void onKey(long keyPtr, int keySize) {
+ this.keyPtr = keyPtr;
+ this.keySize = keySize;
+
+ keyAdded = false;
+ }
+
+ private boolean tryAdd(long valPtr, int valSize) {
+ HadoopShuffleMessage msg = msgs[idx];
+
+ if (!keyAdded) { // Add key and value.
+ int size = keySize + valSize;
+
+ if (!msg.available(size, false))
+ return false;
+
+ msg.addKey(keyPtr, keySize);
+ msg.addValue(valPtr, valSize);
+
+ keyAdded = true;
+
+ return true;
+ }
+
+ if (!msg.available(valSize, true))
+ return false;
+
+ msg.addValue(valPtr, valSize);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onValue(long valPtr, int valSize) {
+ if (tryAdd(valPtr, valSize))
+ return;
+
+ send(idx, keySize + valSize);
+
+ keyAdded = false;
+
+ if (!tryAdd(valPtr, valSize))
+ throw new IllegalStateException();
+ }
+ });
+
+ if (flush && msgs[i].offset() != 0)
+ send(i, 0);
+ }
+ }
+
+ /**
+ * @param idx Index of message.
+ * @param newBufMinSize Min new buffer size.
+ */
+ private void send(final int idx, int newBufMinSize) {
+ final GridFutureAdapterEx<?> fut = new GridFutureAdapterEx<>();
+
+ HadoopShuffleMessage msg = msgs[idx];
+
+ final long msgId = msg.id();
+
+ IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapterEx<?>> old = sentMsgs.putIfAbsent(msgId,
+ new IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapterEx<?>>(msg, fut));
+
+ assert old == null;
+
+ try {
+ io.apply(reduceAddrs[idx], msg);
+ }
+ catch (GridClosureException e) {
+ fut.onDone(U.unwrap(e));
+ }
+
+ fut.listenAsync(new IgniteInClosure<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> f) {
+ try {
+ f.get();
+
+ // Clean up the future from map only if there was no exception.
+ // Otherwise flush() should fail.
+ sentMsgs.remove(msgId);
+ }
+ catch (IgniteCheckedException e) {
+ log.error("Failed to send message.", e);
+ }
+ }
+ });
+
+ msgs[idx] = newBufMinSize == 0 ? null : new HadoopShuffleMessage(job.id(), idx,
+ Math.max(MSG_BUF_SIZE, newBufMinSize));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ if (snd != null) {
+ snd.cancel();
+
+ try {
+ snd.join();
+ }
+ catch (InterruptedException e) {
+ throw new IgniteInterruptedCheckedException(e);
+ }
+ }
+
+ close(maps);
+ }
+
+ /**
+ * @param maps Maps.
+ */
+ private void close(AtomicReferenceArray<HadoopMultimap> maps) {
+ for (int i = 0; i < maps.length(); i++) {
+ HadoopMultimap map = maps.get(i);
+
+ if (map != null)
+ map.close();
+ }
+ }
+
+ /**
+ * @return Future.
+ */
+ @SuppressWarnings("unchecked")
+ public IgniteInternalFuture<?> flush() throws IgniteCheckedException {
+ if (log.isDebugEnabled())
+ log.debug("Flushing job " + job.id() + " on address " + locReduceAddr);
+
+ flushed = true;
+
+ if (maps.length() == 0)
+ return new GridFinishedFutureEx<>();
+
+ U.await(ioInitLatch);
+
+ GridWorker snd0 = snd;
+
+ if (snd0 != null) {
+ if (log.isDebugEnabled())
+ log.debug("Cancelling sender thread.");
+
+ snd0.cancel();
+
+ try {
+ snd0.join();
+
+ if (log.isDebugEnabled())
+ log.debug("Finished waiting for sending thread to complete on shuffle job flush: " + job.id());
+ }
+ catch (InterruptedException e) {
+ throw new IgniteInterruptedCheckedException(e);
+ }
+ }
+
+ collectUpdatesAndSend(true); // With flush.
+
+ if (log.isDebugEnabled())
+ log.debug("Finished sending collected updates to remote reducers: " + job.id());
+
+ GridCompoundFuture fut = new GridCompoundFuture<>();
+
+ for (IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapterEx<?>> tup : sentMsgs.values())
+ fut.add(tup.get2());
+
+ fut.markInitialized();
+
+ if (log.isDebugEnabled())
+ log.debug("Collected futures to compound futures for flush: " + sentMsgs.size());
+
+ return fut;
+ }
+
+ /**
+ * @param taskCtx Task context.
+ * @return Output.
+ * @throws IgniteCheckedException If failed.
+ */
+ public HadoopTaskOutput output(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ switch (taskCtx.taskInfo().type()) {
+ case MAP:
+ assert !job.info().hasCombiner() : "The output creation is allowed if combiner has not been defined.";
+
+ case COMBINE:
+ return new PartitionedOutput(taskCtx);
+
+ default:
+ throw new IllegalStateException("Illegal type: " + taskCtx.taskInfo().type());
+ }
+ }
+
+ /**
+ * @param taskCtx Task context.
+ * @return Input.
+ * @throws IgniteCheckedException If failed.
+ */
+ @SuppressWarnings("unchecked")
+ public HadoopTaskInput input(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ switch (taskCtx.taskInfo().type()) {
+ case REDUCE:
+ int reducer = taskCtx.taskInfo().taskNumber();
+
+ HadoopMultimap m = maps.get(reducer);
+
+ if (m != null)
+ return m.input(taskCtx);
+
+ return new HadoopTaskInput() { // Empty input.
+ @Override public boolean next() {
+ return false;
+ }
+
+ @Override public Object key() {
+ throw new IllegalStateException();
+ }
+
+ @Override public Iterator<?> values() {
+ throw new IllegalStateException();
+ }
+
+ @Override public void close() {
+ // No-op.
+ }
+ };
+
+ default:
+ throw new IllegalStateException("Illegal type: " + taskCtx.taskInfo().type());
+ }
+ }
+
+ /**
+ * Partitioned output.
+ */
+ private class PartitionedOutput implements HadoopTaskOutput {
+ /** */
+ private final HadoopTaskOutput[] adders = new HadoopTaskOutput[maps.length()];
+
+ /** */
+ private HadoopPartitioner partitioner;
+
+ /** */
+ private final HadoopTaskContext taskCtx;
+
+ /**
+ * Constructor.
+ * @param taskCtx Task context.
+ */
+ private PartitionedOutput(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ this.taskCtx = taskCtx;
+
+ if (needPartitioner)
+ partitioner = taskCtx.partitioner();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(Object key, Object val) throws IgniteCheckedException {
+ int part = 0;
+
+ if (partitioner != null) {
+ part = partitioner.partition(key, val, adders.length);
+
+ if (part < 0 || part >= adders.length)
+ throw new IgniteCheckedException("Invalid partition: " + part);
+ }
+
+ HadoopTaskOutput out = adders[part];
+
+ if (out == null)
+ adders[part] = out = getOrCreateMap(maps, part).startAdding(taskCtx);
+
+ out.write(key, val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ for (HadoopTaskOutput adder : adders) {
+ if (adder != null)
+ adder.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
new file mode 100644
index 0000000..c350552a
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.message.*;
+import org.apache.ignite.internal.util.tostring.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory.*;
+
+/**
+ * Shuffle message.
+ */
+public class HadoopShuffleMessage implements HadoopMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private static final AtomicLong ids = new AtomicLong();
+
+ /** */
+ private static final byte MARKER_KEY = (byte)17;
+
+ /** */
+ private static final byte MARKER_VALUE = (byte)31;
+
+ /** */
+ @GridToStringInclude
+ private long msgId;
+
+ /** */
+ @GridToStringInclude
+ private HadoopJobId jobId;
+
+ /** */
+ @GridToStringInclude
+ private int reducer;
+
+ /** */
+ private byte[] buf;
+
+ /** */
+ @GridToStringInclude
+ private int off;
+
+ /**
+ *
+ */
+ public HadoopShuffleMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param size Size.
+ */
+ public HadoopShuffleMessage(HadoopJobId jobId, int reducer, int size) {
+ assert jobId != null;
+
+ buf = new byte[size];
+
+ this.jobId = jobId;
+ this.reducer = reducer;
+
+ msgId = ids.incrementAndGet();
+ }
+
+ /**
+ * @return Message ID.
+ */
+ public long id() {
+ return msgId;
+ }
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /**
+ * @return Reducer.
+ */
+ public int reducer() {
+ return reducer;
+ }
+
+ /**
+ * @return Buffer.
+ */
+ public byte[] buffer() {
+ return buf;
+ }
+
+ /**
+ * @return Offset.
+ */
+ public int offset() {
+ return off;
+ }
+
+ /**
+ * @param size Size.
+ * @param valOnly Only value wll be added.
+ * @return {@code true} If this message can fit additional data of this size
+ */
+ public boolean available(int size, boolean valOnly) {
+ size += valOnly ? 5 : 10;
+
+ if (off + size > buf.length) {
+ if (off == 0) { // Resize if requested size is too big.
+ buf = new byte[size];
+
+ return true;
+ }
+
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * @param keyPtr Key pointer.
+ * @param keySize Key size.
+ */
+ public void addKey(long keyPtr, int keySize) {
+ add(MARKER_KEY, keyPtr, keySize);
+ }
+
+ /**
+ * @param valPtr Value pointer.
+ * @param valSize Value size.
+ */
+ public void addValue(long valPtr, int valSize) {
+ add(MARKER_VALUE, valPtr, valSize);
+ }
+
+ /**
+ * @param marker Marker.
+ * @param ptr Pointer.
+ * @param size Size.
+ */
+ private void add(byte marker, long ptr, int size) {
+ buf[off++] = marker;
+
+ UNSAFE.putInt(buf, BYTE_ARR_OFF + off, size);
+
+ off += 4;
+
+ UNSAFE.copyMemory(null, ptr, buf, BYTE_ARR_OFF + off, size);
+
+ off += size;
+ }
+
+ /**
+ * @param v Visitor.
+ */
+ public void visit(Visitor v) throws IgniteCheckedException {
+ for (int i = 0; i < off;) {
+ byte marker = buf[i++];
+
+ int size = UNSAFE.getInt(buf, BYTE_ARR_OFF + i);
+
+ i += 4;
+
+ if (marker == MARKER_VALUE)
+ v.onValue(buf, i, size);
+ else if (marker == MARKER_KEY)
+ v.onKey(buf, i, size);
+ else
+ throw new IllegalStateException();
+
+ i += size;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ jobId.writeExternal(out);
+ out.writeLong(msgId);
+ out.writeInt(reducer);
+ out.writeInt(off);
+ U.writeByteArray(out, buf);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobId = new HadoopJobId();
+
+ jobId.readExternal(in);
+ msgId = in.readLong();
+ reducer = in.readInt();
+ off = in.readInt();
+ buf = U.readByteArray(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopShuffleMessage.class, this);
+ }
+
+ /**
+ * Visitor.
+ */
+ public static interface Visitor {
+ /**
+ * @param buf Buffer.
+ * @param off Offset.
+ * @param len Length.
+ */
+ public void onKey(byte[] buf, int off, int len) throws IgniteCheckedException;
+
+ /**
+ * @param buf Buffer.
+ * @param off Offset.
+ * @param len Length.
+ */
+ public void onValue(byte[] buf, int off, int len) throws IgniteCheckedException;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimap.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimap.java
deleted file mode 100644
index 32db722..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimap.java
+++ /dev/null
@@ -1,611 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-
-/**
- * Multimap for map reduce intermediate results.
- */
-public class GridHadoopConcurrentHashMultimap extends GridHadoopHashMultimapBase {
- /** */
- private final AtomicReference<State> state = new AtomicReference<>(State.READING_WRITING);
-
- /** */
- private volatile AtomicLongArray oldTbl;
-
- /** */
- private volatile AtomicLongArray newTbl;
-
- /** */
- private final AtomicInteger keys = new AtomicInteger();
-
- /** */
- private final CopyOnWriteArrayList<AdderImpl> adders = new CopyOnWriteArrayList<>();
-
- /** */
- private final AtomicInteger inputs = new AtomicInteger();
-
- /**
- * @param jobInfo Job info.
- * @param mem Memory.
- * @param cap Initial capacity.
- */
- public GridHadoopConcurrentHashMultimap(GridHadoopJobInfo jobInfo, GridUnsafeMemory mem, int cap) {
- super(jobInfo, mem);
-
- assert U.isPow2(cap);
-
- newTbl = oldTbl = new AtomicLongArray(cap);
- }
-
- /**
- * @return Number of keys.
- */
- public long keys() {
- int res = keys.get();
-
- for (AdderImpl adder : adders)
- res += adder.locKeys.get();
-
- return res;
- }
-
- /**
- * @return Current table capacity.
- */
- @Override public int capacity() {
- return oldTbl.length();
- }
-
- /**
- * @return Adder object.
- * @param ctx Task context.
- */
- @Override public Adder startAdding(GridHadoopTaskContext ctx) throws IgniteCheckedException {
- if (inputs.get() != 0)
- throw new IllegalStateException("Active inputs.");
-
- if (state.get() == State.CLOSING)
- throw new IllegalStateException("Closed.");
-
- return new AdderImpl(ctx);
- }
-
- /** {@inheritDoc} */
- @Override public void close() {
- assert inputs.get() == 0 : inputs.get();
- assert adders.isEmpty() : adders.size();
-
- state(State.READING_WRITING, State.CLOSING);
-
- if (keys() == 0)
- return;
-
- super.close();
- }
-
- /** {@inheritDoc} */
- @Override protected long meta(int idx) {
- return oldTbl.get(idx);
- }
-
- /**
- * Incrementally visits all the keys and values in the map.
- *
- * @param ignoreLastVisited Flag indicating that visiting must be started from the beginning.
- * @param v Visitor.
- * @return {@code false} If visiting was impossible due to rehashing.
- */
- @Override public boolean visit(boolean ignoreLastVisited, Visitor v) throws IgniteCheckedException {
- if (!state.compareAndSet(State.READING_WRITING, State.VISITING)) {
- assert state.get() != State.CLOSING;
-
- return false; // Can not visit while rehashing happens.
- }
-
- AtomicLongArray tbl0 = oldTbl;
-
- for (int i = 0; i < tbl0.length(); i++) {
- long meta = tbl0.get(i);
-
- while (meta != 0) {
- long valPtr = value(meta);
-
- long lastVisited = ignoreLastVisited ? 0 : lastVisitedValue(meta);
-
- if (valPtr != lastVisited) {
- v.onKey(key(meta), keySize(meta));
-
- lastVisitedValue(meta, valPtr); // Set it to the first value in chain.
-
- do {
- v.onValue(valPtr + 12, valueSize(valPtr));
-
- valPtr = nextValue(valPtr);
- }
- while (valPtr != lastVisited);
- }
-
- meta = collision(meta);
- }
- }
-
- state(State.VISITING, State.READING_WRITING);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopTaskInput input(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- inputs.incrementAndGet();
-
- if (!adders.isEmpty())
- throw new IllegalStateException("Active adders.");
-
- State s = state.get();
-
- if (s == State.CLOSING)
- throw new IllegalStateException("Closed.");
-
- assert s != State.REHASHING;
-
- return new Input(taskCtx) {
- @Override public void close() throws IgniteCheckedException {
- if (inputs.decrementAndGet() < 0)
- throw new IllegalStateException();
-
- super.close();
- }
- };
- }
-
- /**
- * @param fromTbl Table.
- */
- private void rehashIfNeeded(AtomicLongArray fromTbl) {
- if (fromTbl.length() == Integer.MAX_VALUE)
- return;
-
- long keys0 = keys();
-
- if (keys0 < 3 * (fromTbl.length() >>> 2)) // New size has to be >= than 3/4 of capacity to rehash.
- return;
-
- if (fromTbl != newTbl) // Check if someone else have done the job.
- return;
-
- if (!state.compareAndSet(State.READING_WRITING, State.REHASHING)) {
- assert state.get() != State.CLOSING; // Visiting is allowed, but we will not rehash.
-
- return;
- }
-
- if (fromTbl != newTbl) { // Double check.
- state(State.REHASHING, State.READING_WRITING); // Switch back.
-
- return;
- }
-
- // Calculate new table capacity.
- int newLen = fromTbl.length();
-
- do {
- newLen <<= 1;
- }
- while (newLen < keys0);
-
- if (keys0 >= 3 * (newLen >>> 2)) // Still more than 3/4.
- newLen <<= 1;
-
- // This is our target table for rehashing.
- AtomicLongArray toTbl = new AtomicLongArray(newLen);
-
- // Make the new table visible before rehashing.
- newTbl = toTbl;
-
- // Rehash.
- int newMask = newLen - 1;
-
- long failedMeta = 0;
-
- GridLongList collisions = new GridLongList(16);
-
- for (int i = 0; i < fromTbl.length(); i++) { // Scan source table.
- long meta = fromTbl.get(i);
-
- assert meta != -1;
-
- if (meta == 0) { // No entry.
- failedMeta = 0;
-
- if (!fromTbl.compareAndSet(i, 0, -1)) // Mark as moved.
- i--; // Retry.
-
- continue;
- }
-
- do { // Collect all the collisions before the last one failed to nullify or 0.
- collisions.add(meta);
-
- meta = collision(meta);
- }
- while (meta != failedMeta);
-
- do { // Go from the last to the first to avoid 'in-flight' state for meta entries.
- meta = collisions.remove();
-
- int addr = keyHash(meta) & newMask;
-
- for (;;) { // Move meta entry to the new table.
- long toCollision = toTbl.get(addr);
-
- collision(meta, toCollision);
-
- if (toTbl.compareAndSet(addr, toCollision, meta))
- break;
- }
- }
- while (!collisions.isEmpty());
-
- // Here 'meta' will be a root pointer in old table.
- if (!fromTbl.compareAndSet(i, meta, -1)) { // Try to mark as moved.
- failedMeta = meta;
-
- i--; // Retry the same address in table because new keys were added.
- }
- else
- failedMeta = 0;
- }
-
- // Now old and new tables will be the same again.
- oldTbl = toTbl;
-
- state(State.REHASHING, State.READING_WRITING);
- }
-
- /**
- * Switch state.
- *
- * @param oldState Expected state.
- * @param newState New state.
- */
- private void state(State oldState, State newState) {
- if (!state.compareAndSet(oldState, newState))
- throw new IllegalStateException();
- }
-
- /**
- * @param meta Meta pointer.
- * @return Value pointer.
- */
- @Override protected long value(long meta) {
- return mem.readLongVolatile(meta + 16);
- }
-
- /**
- * @param meta Meta pointer.
- * @param oldValPtr Old value.
- * @param newValPtr New value.
- * @return {@code true} If succeeded.
- */
- private boolean casValue(long meta, long oldValPtr, long newValPtr) {
- return mem.casLong(meta + 16, oldValPtr, newValPtr);
- }
-
- /**
- * @param meta Meta pointer.
- * @return Collision pointer.
- */
- @Override protected long collision(long meta) {
- return mem.readLongVolatile(meta + 24);
- }
-
- /**
- * @param meta Meta pointer.
- * @param collision Collision pointer.
- */
- @Override protected void collision(long meta, long collision) {
- assert meta != collision : meta;
-
- mem.writeLongVolatile(meta + 24, collision);
- }
-
- /**
- * @param meta Meta pointer.
- * @return Last visited value pointer.
- */
- private long lastVisitedValue(long meta) {
- return mem.readLong(meta + 32);
- }
-
- /**
- * @param meta Meta pointer.
- * @param valPtr Last visited value pointer.
- */
- private void lastVisitedValue(long meta, long valPtr) {
- mem.writeLong(meta + 32, valPtr);
- }
-
- /**
- * Adder. Must not be shared between threads.
- */
- private class AdderImpl extends AdderBase {
- /** */
- private final Reader keyReader;
-
- /** */
- private final AtomicInteger locKeys = new AtomicInteger();
-
- /** */
- private final Random rnd = new GridRandom();
-
- /**
- * @param ctx Task context.
- * @throws IgniteCheckedException If failed.
- */
- private AdderImpl(GridHadoopTaskContext ctx) throws IgniteCheckedException {
- super(ctx);
-
- keyReader = new Reader(keySer);
-
- rehashIfNeeded(oldTbl);
-
- adders.add(this);
- }
-
- /**
- * @param in Data input.
- * @param reuse Reusable key.
- * @return Key.
- * @throws IgniteCheckedException If failed.
- */
- @Override public Key addKey(DataInput in, @Nullable Key reuse) throws IgniteCheckedException {
- KeyImpl k = reuse == null ? new KeyImpl() : (KeyImpl)reuse;
-
- k.tmpKey = keySer.read(in, k.tmpKey);
-
- k.meta = add(k.tmpKey, null);
-
- return k;
- }
-
- /** {@inheritDoc} */
- @Override public void write(Object key, Object val) throws IgniteCheckedException {
- A.notNull(val, "val");
-
- add(key, val);
- }
-
- /**
- * @param tbl Table.
- */
- private void incrementKeys(AtomicLongArray tbl) {
- locKeys.lazySet(locKeys.get() + 1);
-
- if (rnd.nextInt(tbl.length()) < 512)
- rehashIfNeeded(tbl);
- }
-
- /**
- * @param keyHash Key hash.
- * @param keySize Key size.
- * @param keyPtr Key pointer.
- * @param valPtr Value page pointer.
- * @param collisionPtr Pointer to meta with hash collision.
- * @param lastVisitedVal Last visited value pointer.
- * @return Created meta page pointer.
- */
- private long createMeta(int keyHash, int keySize, long keyPtr, long valPtr, long collisionPtr, long lastVisitedVal) {
- long meta = allocate(40);
-
- mem.writeInt(meta, keyHash);
- mem.writeInt(meta + 4, keySize);
- mem.writeLong(meta + 8, keyPtr);
- mem.writeLong(meta + 16, valPtr);
- mem.writeLong(meta + 24, collisionPtr);
- mem.writeLong(meta + 32, lastVisitedVal);
-
- return meta;
- }
-
- /**
- * @param key Key.
- * @param val Value.
- * @return Updated or created meta page pointer.
- * @throws IgniteCheckedException If failed.
- */
- private long add(Object key, @Nullable Object val) throws IgniteCheckedException {
- AtomicLongArray tbl = oldTbl;
-
- int keyHash = U.hash(key.hashCode());
-
- long newMetaPtr = 0;
-
- long valPtr = 0;
-
- if (val != null) {
- valPtr = write(12, val, valSer);
- int valSize = writtenSize() - 12;
-
- valueSize(valPtr, valSize);
- }
-
- for (AtomicLongArray old = null;;) {
- int addr = keyHash & (tbl.length() - 1);
-
- long metaPtrRoot = tbl.get(addr); // Read root meta pointer at this address.
-
- if (metaPtrRoot == -1) { // The cell was already moved by rehashing.
- AtomicLongArray n = newTbl; // Need to read newTbl first here.
- AtomicLongArray o = oldTbl;
-
- tbl = tbl == o ? n : o; // Trying to get the oldest table but newer than ours.
-
- old = null;
-
- continue;
- }
-
- if (metaPtrRoot != 0) { // Not empty slot.
- long metaPtr = metaPtrRoot;
-
- do { // Scan all the collisions.
- if (keyHash(metaPtr) == keyHash && key.equals(keyReader.readKey(metaPtr))) { // Found key.
- if (newMetaPtr != 0) // Deallocate new meta if one was allocated.
- localDeallocate(key(newMetaPtr)); // Key was allocated first, so rewind to it's pointer.
-
- if (valPtr != 0) { // Add value if it exists.
- long nextValPtr;
-
- // Values are linked to each other to a stack like structure.
- // Replace the last value in meta with ours and link it as next.
- do {
- nextValPtr = value(metaPtr);
-
- nextValue(valPtr, nextValPtr);
- }
- while (!casValue(metaPtr, nextValPtr, valPtr));
- }
-
- return metaPtr;
- }
-
- metaPtr = collision(metaPtr);
- }
- while (metaPtr != 0);
-
- // Here we did not find our key, need to check if it was moved by rehashing to the new table.
- if (old == null) { // If the old table already set, then we will just try to update it.
- AtomicLongArray n = newTbl;
-
- if (n != tbl) { // Rehashing happens, try to find the key in new table but preserve the old one.
- old = tbl;
- tbl = n;
-
- continue;
- }
- }
- }
-
- if (old != null) { // We just checked new table but did not find our key as well as in the old one.
- tbl = old; // Try to add new key to the old table.
-
- addr = keyHash & (tbl.length() - 1);
-
- old = null;
- }
-
- if (newMetaPtr == 0) { // Allocate new meta page.
- long keyPtr = write(0, key, keySer);
- int keySize = writtenSize();
-
- if (valPtr != 0)
- nextValue(valPtr, 0);
-
- newMetaPtr = createMeta(keyHash, keySize, keyPtr, valPtr, metaPtrRoot, 0);
- }
- else // Update new meta with root pointer collision.
- collision(newMetaPtr, metaPtrRoot);
-
- if (tbl.compareAndSet(addr, metaPtrRoot, newMetaPtr)) { // Try to replace root pointer with new one.
- incrementKeys(tbl);
-
- return newMetaPtr;
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteCheckedException {
- if (!adders.remove(this))
- throw new IllegalStateException();
-
- keys.addAndGet(locKeys.get()); // Here we have race and #keys() method can return wrong result but it is ok.
-
- super.close();
- }
-
- /**
- * Key.
- */
- private class KeyImpl implements Key {
- /** */
- private long meta;
-
- /** */
- private Object tmpKey;
-
- /**
- * @return Meta pointer for the key.
- */
- public long address() {
- return meta;
- }
-
- /**
- * @param val Value.
- */
- @Override public void add(Value val) {
- int size = val.size();
-
- long valPtr = allocate(size + 12);
-
- val.copyTo(valPtr + 12);
-
- valueSize(valPtr, size);
-
- long nextVal;
-
- do {
- nextVal = value(meta);
-
- nextValue(valPtr, nextVal);
- }
- while(!casValue(meta, nextVal, valPtr));
- }
- }
- }
-
- /**
- * Current map state.
- */
- private enum State {
- /** */
- REHASHING,
-
- /** */
- VISITING,
-
- /** */
- READING_WRITING,
-
- /** */
- CLOSING
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimap.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimap.java
deleted file mode 100644
index 2795b77..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimap.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-/**
- * Hash multimap.
- */
-public class GridHadoopHashMultimap extends GridHadoopHashMultimapBase {
- /** */
- private long[] tbl;
-
- /** */
- private int keys;
-
- /**
- * @param jobInfo Job info.
- * @param mem Memory.
- * @param cap Initial capacity.
- */
- public GridHadoopHashMultimap(GridHadoopJobInfo jobInfo, GridUnsafeMemory mem, int cap) {
- super(jobInfo, mem);
-
- assert U.isPow2(cap) : cap;
-
- tbl = new long[cap];
- }
-
- /** {@inheritDoc} */
- @Override public Adder startAdding(GridHadoopTaskContext ctx) throws IgniteCheckedException {
- return new AdderImpl(ctx);
- }
-
- /**
- * Rehash.
- */
- private void rehash() {
- long[] newTbl = new long[tbl.length << 1];
-
- int newMask = newTbl.length - 1;
-
- for (long meta : tbl) {
- while (meta != 0) {
- long collision = collision(meta);
-
- int idx = keyHash(meta) & newMask;
-
- collision(meta, newTbl[idx]);
-
- newTbl[idx] = meta;
-
- meta = collision;
- }
- }
-
- tbl = newTbl;
- }
-
- /**
- * @return Keys count.
- */
- public int keys() {
- return keys;
- }
-
- /** {@inheritDoc} */
- @Override public int capacity() {
- return tbl.length;
- }
-
- /** {@inheritDoc} */
- @Override protected long meta(int idx) {
- return tbl[idx];
- }
-
- /**
- * Adder.
- */
- private class AdderImpl extends AdderBase {
- /** */
- private final Reader keyReader;
-
- /**
- * @param ctx Task context.
- * @throws IgniteCheckedException If failed.
- */
- protected AdderImpl(GridHadoopTaskContext ctx) throws IgniteCheckedException {
- super(ctx);
-
- keyReader = new Reader(keySer);
- }
-
- /**
- * @param keyHash Key hash.
- * @param keySize Key size.
- * @param keyPtr Key pointer.
- * @param valPtr Value page pointer.
- * @param collisionPtr Pointer to meta with hash collision.
- * @return Created meta page pointer.
- */
- private long createMeta(int keyHash, int keySize, long keyPtr, long valPtr, long collisionPtr) {
- long meta = allocate(32);
-
- mem.writeInt(meta, keyHash);
- mem.writeInt(meta + 4, keySize);
- mem.writeLong(meta + 8, keyPtr);
- mem.writeLong(meta + 16, valPtr);
- mem.writeLong(meta + 24, collisionPtr);
-
- return meta;
- }
-
- /** {@inheritDoc} */
- @Override public void write(Object key, Object val) throws IgniteCheckedException {
- A.notNull(val, "val");
-
- int keyHash = U.hash(key.hashCode());
-
- // Write value.
- long valPtr = write(12, val, valSer);
- int valSize = writtenSize() - 12;
-
- valueSize(valPtr, valSize);
-
- // Find position in table.
- int idx = keyHash & (tbl.length - 1);
-
- long meta = tbl[idx];
-
- // Search for our key in collisions.
- while (meta != 0) {
- if (keyHash(meta) == keyHash && key.equals(keyReader.readKey(meta))) { // Found key.
- nextValue(valPtr, value(meta));
-
- value(meta, valPtr);
-
- return;
- }
-
- meta = collision(meta);
- }
-
- // Write key.
- long keyPtr = write(0, key, keySer);
- int keySize = writtenSize();
-
- nextValue(valPtr, 0);
-
- tbl[idx] = createMeta(keyHash, keySize, keyPtr, valPtr, tbl[idx]);
-
- if (++keys > (tbl.length >>> 2) * 3)
- rehash();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimapBase.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimapBase.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimapBase.java
deleted file mode 100644
index 92854f1..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimapBase.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-
-import java.util.*;
-
-/**
- * Base class for hash multimaps.
- */
-public abstract class GridHadoopHashMultimapBase extends GridHadoopMultimapBase {
- /**
- * @param jobInfo Job info.
- * @param mem Memory.
- */
- protected GridHadoopHashMultimapBase(GridHadoopJobInfo jobInfo, GridUnsafeMemory mem) {
- super(jobInfo, mem);
- }
-
- /** {@inheritDoc} */
- @Override public boolean visit(boolean ignoreLastVisited, Visitor v) throws IgniteCheckedException {
- throw new UnsupportedOperationException("visit");
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopTaskInput input(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- return new Input(taskCtx);
- }
-
- /**
- * @return Hash table capacity.
- */
- public abstract int capacity();
-
- /**
- * @param idx Index in hash table.
- * @return Meta page pointer.
- */
- protected abstract long meta(int idx);
-
- /**
- * @param meta Meta pointer.
- * @return Key hash.
- */
- protected int keyHash(long meta) {
- return mem.readInt(meta);
- }
-
- /**
- * @param meta Meta pointer.
- * @return Key size.
- */
- protected int keySize(long meta) {
- return mem.readInt(meta + 4);
- }
-
- /**
- * @param meta Meta pointer.
- * @return Key pointer.
- */
- protected long key(long meta) {
- return mem.readLong(meta + 8);
- }
-
- /**
- * @param meta Meta pointer.
- * @return Value pointer.
- */
- protected long value(long meta) {
- return mem.readLong(meta + 16);
- }
- /**
- * @param meta Meta pointer.
- * @param val Value pointer.
- */
- protected void value(long meta, long val) {
- mem.writeLong(meta + 16, val);
- }
-
- /**
- * @param meta Meta pointer.
- * @return Collision pointer.
- */
- protected long collision(long meta) {
- return mem.readLong(meta + 24);
- }
-
- /**
- * @param meta Meta pointer.
- * @param collision Collision pointer.
- */
- protected void collision(long meta, long collision) {
- assert meta != collision : meta;
-
- mem.writeLong(meta + 24, collision);
- }
-
- /**
- * Reader for key and value.
- */
- protected class Reader extends ReaderBase {
- /**
- * @param ser Serialization.
- */
- protected Reader(GridHadoopSerialization ser) {
- super(ser);
- }
-
- /**
- * @param meta Meta pointer.
- * @return Key.
- */
- public Object readKey(long meta) {
- assert meta > 0 : meta;
-
- try {
- return read(key(meta), keySize(meta));
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
- }
-
- /**
- * Task input.
- */
- protected class Input implements GridHadoopTaskInput {
- /** */
- private int idx = -1;
-
- /** */
- private long metaPtr;
-
- /** */
- private final int cap;
-
- /** */
- private final Reader keyReader;
-
- /** */
- private final Reader valReader;
-
- /**
- * @param taskCtx Task context.
- * @throws IgniteCheckedException If failed.
- */
- public Input(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- cap = capacity();
-
- keyReader = new Reader(taskCtx.keySerialization());
- valReader = new Reader(taskCtx.valueSerialization());
- }
-
- /** {@inheritDoc} */
- @Override public boolean next() {
- if (metaPtr != 0) {
- metaPtr = collision(metaPtr);
-
- if (metaPtr != 0)
- return true;
- }
-
- while (++idx < cap) { // Scan table.
- metaPtr = meta(idx);
-
- if (metaPtr != 0)
- return true;
- }
-
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public Object key() {
- return keyReader.readKey(metaPtr);
- }
-
- /** {@inheritDoc} */
- @Override public Iterator<?> values() {
- return new ValueIterator(value(metaPtr), valReader);
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteCheckedException {
- keyReader.close();
- valReader.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimap.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimap.java
deleted file mode 100644
index b8eb12c..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimap.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * Multimap for hadoop intermediate results.
- */
-@SuppressWarnings("PublicInnerClass")
-public interface GridHadoopMultimap extends AutoCloseable {
- /**
- * Incrementally visits all the keys and values in the map.
- *
- * @param ignoreLastVisited Flag indicating that visiting must be started from the beginning.
- * @param v Visitor.
- * @return {@code false} If visiting was impossible.
- */
- public boolean visit(boolean ignoreLastVisited, Visitor v) throws IgniteCheckedException;
-
- /**
- * @param ctx Task context.
- * @return Adder.
- * @throws IgniteCheckedException If failed.
- */
- public Adder startAdding(GridHadoopTaskContext ctx) throws IgniteCheckedException;
-
- /**
- * @param taskCtx Task context.
- * @return Task input.
- * @throws IgniteCheckedException If failed.
- */
- public GridHadoopTaskInput input(GridHadoopTaskContext taskCtx)
- throws IgniteCheckedException;
-
- /** {@inheritDoc} */
- @Override public void close();
-
- /**
- * Adder.
- */
- public interface Adder extends GridHadoopTaskOutput {
- /**
- * @param in Data input.
- * @param reuse Reusable key.
- * @return Key.
- * @throws IgniteCheckedException If failed.
- */
- public Key addKey(DataInput in, @Nullable Key reuse) throws IgniteCheckedException;
- }
-
- /**
- * Key add values to.
- */
- public interface Key {
- /**
- * @param val Value.
- */
- public void add(Value val);
- }
-
- /**
- * Value.
- */
- public interface Value {
- /**
- * @return Size in bytes.
- */
- public int size();
-
- /**
- * @param ptr Pointer.
- */
- public void copyTo(long ptr);
- }
-
- /**
- * Key and values visitor.
- */
- public interface Visitor {
- /**
- * @param keyPtr Key pointer.
- * @param keySize Key size.
- */
- public void onKey(long keyPtr, int keySize) throws IgniteCheckedException;
-
- /**
- * @param valPtr Value pointer.
- * @param valSize Value size.
- */
- public void onValue(long valPtr, int valSize) throws IgniteCheckedException;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimapBase.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimapBase.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimapBase.java
deleted file mode 100644
index 2d8660f..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimapBase.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.shuffle.streams.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopJobProperty.*;
-
-/**
- * Base class for all multimaps.
- */
-public abstract class GridHadoopMultimapBase implements GridHadoopMultimap {
- /** */
- protected final GridUnsafeMemory mem;
-
- /** */
- protected final int pageSize;
-
- /** */
- private final Collection<GridLongList> allPages = new ConcurrentLinkedQueue<>();
-
- /**
- * @param jobInfo Job info.
- * @param mem Memory.
- */
- protected GridHadoopMultimapBase(GridHadoopJobInfo jobInfo, GridUnsafeMemory mem) {
- assert jobInfo != null;
- assert mem != null;
-
- this.mem = mem;
-
- pageSize = get(jobInfo, SHUFFLE_OFFHEAP_PAGE_SIZE, 32 * 1024);
- }
-
- /**
- * @param ptrs Page pointers.
- */
- private void deallocate(GridLongList ptrs) {
- while (!ptrs.isEmpty())
- mem.release(ptrs.remove(), ptrs.remove());
- }
-
- /**
- * @param valPtr Value page pointer.
- * @param nextValPtr Next value page pointer.
- */
- protected void nextValue(long valPtr, long nextValPtr) {
- mem.writeLong(valPtr, nextValPtr);
- }
-
- /**
- * @param valPtr Value page pointer.
- * @return Next value page pointer.
- */
- protected long nextValue(long valPtr) {
- return mem.readLong(valPtr);
- }
-
- /**
- * @param valPtr Value page pointer.
- * @param size Size.
- */
- protected void valueSize(long valPtr, int size) {
- mem.writeInt(valPtr + 8, size);
- }
-
- /**
- * @param valPtr Value page pointer.
- * @return Value size.
- */
- protected int valueSize(long valPtr) {
- return mem.readInt(valPtr + 8);
- }
-
- /** {@inheritDoc} */
- @Override public void close() {
- for (GridLongList list : allPages)
- deallocate(list);
- }
-
- /**
- * Reader for key and value.
- */
- protected class ReaderBase implements AutoCloseable {
- /** */
- private Object tmp;
-
- /** */
- private final GridHadoopSerialization ser;
-
- /** */
- private final GridHadoopDataInStream in = new GridHadoopDataInStream(mem);
-
- /**
- * @param ser Serialization.
- */
- protected ReaderBase(GridHadoopSerialization ser) {
- assert ser != null;
-
- this.ser = ser;
- }
-
- /**
- * @param valPtr Value page pointer.
- * @return Value.
- */
- public Object readValue(long valPtr) {
- assert valPtr > 0 : valPtr;
-
- try {
- return read(valPtr + 12, valueSize(valPtr));
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- /**
- * Resets temporary object to the given one.
- *
- * @param tmp Temporary object for reuse.
- */
- public void resetReusedObject(Object tmp) {
- this.tmp = tmp;
- }
-
- /**
- * @param ptr Pointer.
- * @param size Object size.
- * @return Object.
- */
- protected Object read(long ptr, long size) throws IgniteCheckedException {
- in.buffer().set(ptr, size);
-
- tmp = ser.read(in, tmp);
-
- return tmp;
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteCheckedException {
- ser.close();
- }
- }
-
- /**
- * Base class for adders.
- */
- protected abstract class AdderBase implements Adder {
- /** */
- protected final GridHadoopSerialization keySer;
-
- /** */
- protected final GridHadoopSerialization valSer;
-
- /** */
- private final GridHadoopDataOutStream out;
-
- /** */
- private long writeStart;
-
- /** Size and pointer pairs list. */
- private final GridLongList pages = new GridLongList(16);
-
- /**
- * @param ctx Task context.
- * @throws IgniteCheckedException If failed.
- */
- protected AdderBase(GridHadoopTaskContext ctx) throws IgniteCheckedException {
- valSer = ctx.valueSerialization();
- keySer = ctx.keySerialization();
-
- out = new GridHadoopDataOutStream(mem) {
- @Override public long move(long size) {
- long ptr = super.move(size);
-
- if (ptr == 0) // Was not able to move - not enough free space.
- ptr = allocateNextPage(size);
-
- assert ptr != 0;
-
- return ptr;
- }
- };
- }
-
- /**
- * @param requestedSize Requested size.
- * @return Next write pointer.
- */
- private long allocateNextPage(long requestedSize) {
- int writtenSize = writtenSize();
-
- long newPageSize = Math.max(writtenSize + requestedSize, pageSize);
- long newPagePtr = mem.allocate(newPageSize);
-
- pages.add(newPageSize);
- pages.add(newPagePtr);
-
- GridHadoopOffheapBuffer b = out.buffer();
-
- b.set(newPagePtr, newPageSize);
-
- if (writtenSize != 0) {
- mem.copyMemory(writeStart, newPagePtr, writtenSize);
-
- b.move(writtenSize);
- }
-
- writeStart = newPagePtr;
-
- return b.move(requestedSize);
- }
-
- /**
- * @return Fixed pointer.
- */
- private long fixAlignment() {
- GridHadoopOffheapBuffer b = out.buffer();
-
- long ptr = b.pointer();
-
- if ((ptr & 7L) != 0) { // Address is not aligned by octet.
- ptr = (ptr + 8L) & ~7L;
-
- b.pointer(ptr);
- }
-
- return ptr;
- }
-
- /**
- * @param off Offset.
- * @param o Object.
- * @return Page pointer.
- * @throws IgniteCheckedException If failed.
- */
- protected long write(int off, Object o, GridHadoopSerialization ser) throws IgniteCheckedException {
- writeStart = fixAlignment();
-
- if (off != 0)
- out.move(off);
-
- ser.write(out, o);
-
- return writeStart;
- }
-
- /**
- * @param size Size.
- * @return Pointer.
- */
- protected long allocate(int size) {
- writeStart = fixAlignment();
-
- out.move(size);
-
- return writeStart;
- }
-
- /**
- * Rewinds local allocation pointer to the given pointer if possible.
- *
- * @param ptr Pointer.
- */
- protected void localDeallocate(long ptr) {
- GridHadoopOffheapBuffer b = out.buffer();
-
- if (b.isInside(ptr))
- b.pointer(ptr);
- else
- b.reset();
- }
-
- /**
- * @return Written size.
- */
- protected int writtenSize() {
- return (int)(out.buffer().pointer() - writeStart);
- }
-
- /** {@inheritDoc} */
- @Override public Key addKey(DataInput in, @Nullable Key reuse) throws IgniteCheckedException {
- throw new UnsupportedOperationException();
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteCheckedException {
- allPages.add(pages);
-
- keySer.close();
- valSer.close();
- }
- }
-
- /**
- * Iterator over values.
- */
- protected class ValueIterator implements Iterator<Object> {
- /** */
- private long valPtr;
-
- /** */
- private final ReaderBase valReader;
-
- /**
- * @param valPtr Value page pointer.
- * @param valReader Value reader.
- */
- protected ValueIterator(long valPtr, ReaderBase valReader) {
- this.valPtr = valPtr;
- this.valReader = valReader;
- }
-
- /**
- * @param valPtr Head value pointer.
- */
- public void head(long valPtr) {
- this.valPtr = valPtr;
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasNext() {
- return valPtr != 0;
- }
-
- /** {@inheritDoc} */
- @Override public Object next() {
- if (!hasNext())
- throw new NoSuchElementException();
-
- Object res = valReader.readValue(valPtr);
-
- valPtr = nextValue(valPtr);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public void remove() {
- throw new UnsupportedOperationException();
- }
- }
-}
[13/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
index 4ad74d0..2994107 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
@@ -2,14 +2,15 @@ package org.apache.ignite.igfs;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.permission.*;
import org.apache.ignite.*;
import org.apache.ignite.cache.*;
import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.hadoop.*;
-import org.apache.ignite.igfs.hadoop.v1.*;
-import org.apache.ignite.internal.igfs.hadoop.*;
+import org.apache.ignite.hadoop.fs.*;
+import org.apache.ignite.hadoop.fs.v1.*;
import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.igfs.*;
import org.apache.ignite.internal.processors.igfs.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
@@ -221,7 +222,7 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
* Starts secondary IGFS
*/
private void startSecondary() {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName("partitioned");
igfsCfg.setMetaCacheName("replicated");
@@ -259,7 +260,7 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
cfg.setDiscoverySpi(discoSpi);
cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
cfg.setCommunicationSpi(communicationSpi());
@@ -295,7 +296,7 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
cfg.setDiscoverySpi(discoSpi);
cfg.setCacheConfiguration(cacheConfiguration());
- cfg.setIgfsConfiguration(igfsConfiguration(gridName));
+ cfg.setFileSystemConfiguration(fsConfiguration(gridName));
cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
cfg.setCommunicationSpi(communicationSpi());
@@ -336,8 +337,8 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
* @param gridName Grid name.
* @return IGFS configuration.
*/
- protected IgfsConfiguration igfsConfiguration(String gridName) throws IgniteCheckedException {
- IgfsConfiguration cfg = new IgfsConfiguration();
+ protected FileSystemConfiguration fsConfiguration(String gridName) throws IgniteCheckedException {
+ FileSystemConfiguration cfg = new FileSystemConfiguration();
cfg.setDataCacheName("partitioned");
cfg.setMetaCacheName("replicated");
@@ -347,7 +348,7 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
if (mode != PRIMARY)
cfg.setSecondaryFileSystem(
- new IgfsHadoopFileSystemWrapper(secondaryFsUriStr, secondaryConfFullPath));
+ new IgniteHadoopIgfsSecondaryFileSystem(secondaryFsUriStr, secondaryConfFullPath));
cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(gridName));
@@ -462,10 +463,10 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
if (authority != null) {
if (skipEmbed)
- cfg.setBoolean(String.format(IgfsHadoopUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);
if (skipLocShmem)
- cfg.setBoolean(String.format(IgfsHadoopUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);
+ cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);
}
return cfg;
@@ -477,10 +478,10 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
* @param cfg the configuration to set parameters into.
*/
private static void setImplClasses(Configuration cfg) {
- cfg.set("fs.igfs.impl", IgfsHadoopFileSystem.class.getName());
+ cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
cfg.set("fs.AbstractFileSystem.igfs.impl",
- org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem.class.getName());
+ org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName());
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsEventsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsEventsTestSuite.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsEventsTestSuite.java
index 29696bf..f6f5bae 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsEventsTestSuite.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsEventsTestSuite.java
@@ -20,7 +20,7 @@ package org.apache.ignite.igfs;
import junit.framework.*;
import org.apache.ignite.*;
import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.hadoop.*;
+import org.apache.ignite.hadoop.fs.*;
import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.ipc.shmem.*;
import org.apache.ignite.internal.util.typedef.*;
@@ -40,7 +40,7 @@ public class IgfsEventsTestSuite extends TestSuite {
* @throws Exception Thrown in case of the failure.
*/
public static TestSuite suite() throws Exception {
- GridHadoopClassLoader ldr = new GridHadoopClassLoader(null);
+ HadoopClassLoader ldr = new HadoopClassLoader(null);
TestSuite suite = new TestSuite("Ignite FS Events Test Suite");
@@ -60,7 +60,7 @@ public class IgfsEventsTestSuite extends TestSuite {
* @throws Exception Thrown in case of the failure.
*/
public static TestSuite suiteNoarchOnly() throws Exception {
- GridHadoopClassLoader ldr = new GridHadoopClassLoader(null);
+ HadoopClassLoader ldr = new HadoopClassLoader(null);
TestSuite suite = new TestSuite("Ignite IGFS Events Test Suite Noarch Only");
@@ -76,8 +76,8 @@ public class IgfsEventsTestSuite extends TestSuite {
*/
public static class ShmemPrivate extends IgfsEventsAbstractSelfTest {
/** {@inheritDoc} */
- @Override protected IgfsConfiguration getIgfsConfiguration() throws IgniteCheckedException {
- IgfsConfiguration igfsCfg = super.getIgfsConfiguration();
+ @Override protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException {
+ FileSystemConfiguration igfsCfg = super.getIgfsConfiguration();
igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
put("type", "shmem");
@@ -93,8 +93,8 @@ public class IgfsEventsTestSuite extends TestSuite {
*/
public static class LoopbackPrivate extends IgfsEventsAbstractSelfTest {
/** {@inheritDoc} */
- @Override protected IgfsConfiguration getIgfsConfiguration() throws IgniteCheckedException {
- IgfsConfiguration igfsCfg = super.getIgfsConfiguration();
+ @Override protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException {
+ FileSystemConfiguration igfsCfg = super.getIgfsConfiguration();
igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{
put("type", "tcp");
@@ -110,13 +110,13 @@ public class IgfsEventsTestSuite extends TestSuite {
*/
public abstract static class PrimarySecondaryTest extends IgfsEventsAbstractSelfTest {
/** Secondary file system. */
- private static IgniteFs igfsSec;
+ private static IgniteFileSystem igfsSec;
/** {@inheritDoc} */
- @Override protected IgfsConfiguration getIgfsConfiguration() throws IgniteCheckedException {
- IgfsConfiguration igfsCfg = super.getIgfsConfiguration();
+ @Override protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException {
+ FileSystemConfiguration igfsCfg = super.getIgfsConfiguration();
- igfsCfg.setSecondaryFileSystem(new IgfsHadoopFileSystemWrapper(
+ igfsCfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(
"igfs://igfs-secondary:grid-secondary@127.0.0.1:11500/",
"modules/core/src/test/config/hadoop/core-site-secondary.xml"));
@@ -126,8 +126,8 @@ public class IgfsEventsTestSuite extends TestSuite {
/**
* @return IGFS configuration for secondary file system.
*/
- protected IgfsConfiguration getSecondaryIgfsConfiguration() throws IgniteCheckedException {
- IgfsConfiguration igfsCfg = super.getIgfsConfiguration();
+ protected FileSystemConfiguration getSecondaryIgfsConfiguration() throws IgniteCheckedException {
+ FileSystemConfiguration igfsCfg = super.getIgfsConfiguration();
igfsCfg.setName("igfs-secondary");
igfsCfg.setDefaultMode(PRIMARY);
@@ -167,7 +167,7 @@ public class IgfsEventsTestSuite extends TestSuite {
* @return Secondary file system handle.
* @throws Exception If failed.
*/
- @Nullable private IgniteFs startSecondary() throws Exception {
+ @Nullable private IgniteFileSystem startSecondary() throws Exception {
IgniteConfiguration cfg = getConfiguration("grid-secondary", getSecondaryIgfsConfiguration());
cfg.setLocalHost("127.0.0.1");
@@ -184,8 +184,8 @@ public class IgfsEventsTestSuite extends TestSuite {
*/
public static class ShmemDualSync extends PrimarySecondaryTest {
/** {@inheritDoc} */
- @Override protected IgfsConfiguration getIgfsConfiguration() throws IgniteCheckedException {
- IgfsConfiguration igfsCfg = super.getIgfsConfiguration();
+ @Override protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException {
+ FileSystemConfiguration igfsCfg = super.getIgfsConfiguration();
igfsCfg.setDefaultMode(DUAL_SYNC);
@@ -198,8 +198,8 @@ public class IgfsEventsTestSuite extends TestSuite {
*/
public static class ShmemDualAsync extends PrimarySecondaryTest {
/** {@inheritDoc} */
- @Override protected IgfsConfiguration getIgfsConfiguration() throws IgniteCheckedException {
- IgfsConfiguration igfsCfg = super.getIgfsConfiguration();
+ @Override protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException {
+ FileSystemConfiguration igfsCfg = super.getIgfsConfiguration();
igfsCfg.setDefaultMode(DUAL_ASYNC);
@@ -212,10 +212,10 @@ public class IgfsEventsTestSuite extends TestSuite {
*/
public abstract static class LoopbackPrimarySecondaryTest extends PrimarySecondaryTest {
/** {@inheritDoc} */
- @Override protected IgfsConfiguration getIgfsConfiguration() throws IgniteCheckedException {
- IgfsConfiguration igfsCfg = super.getIgfsConfiguration();
+ @Override protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException {
+ FileSystemConfiguration igfsCfg = super.getIgfsConfiguration();
- igfsCfg.setSecondaryFileSystem(new IgfsHadoopFileSystemWrapper(
+ igfsCfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(
"igfs://igfs-secondary:grid-secondary@127.0.0.1:11500/",
"modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml"));
@@ -223,8 +223,8 @@ public class IgfsEventsTestSuite extends TestSuite {
}
/** {@inheritDoc} */
- @Override protected IgfsConfiguration getSecondaryIgfsConfiguration() throws IgniteCheckedException {
- IgfsConfiguration igfsCfg = super.getSecondaryIgfsConfiguration();
+ @Override protected FileSystemConfiguration getSecondaryIgfsConfiguration() throws IgniteCheckedException {
+ FileSystemConfiguration igfsCfg = super.getSecondaryIgfsConfiguration();
igfsCfg.setName("igfs-secondary");
igfsCfg.setDefaultMode(PRIMARY);
@@ -242,8 +242,8 @@ public class IgfsEventsTestSuite extends TestSuite {
*/
public static class LoopbackDualSync extends LoopbackPrimarySecondaryTest {
/** {@inheritDoc} */
- @Override protected IgfsConfiguration getIgfsConfiguration() throws IgniteCheckedException {
- IgfsConfiguration igfsCfg = super.getIgfsConfiguration();
+ @Override protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException {
+ FileSystemConfiguration igfsCfg = super.getIgfsConfiguration();
igfsCfg.setDefaultMode(DUAL_SYNC);
@@ -256,8 +256,8 @@ public class IgfsEventsTestSuite extends TestSuite {
*/
public static class LoopbackDualAsync extends LoopbackPrimarySecondaryTest {
/** {@inheritDoc} */
- @Override protected IgfsConfiguration getIgfsConfiguration() throws IgniteCheckedException {
- IgfsConfiguration igfsCfg = super.getIgfsConfiguration();
+ @Override protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException {
+ FileSystemConfiguration igfsCfg = super.getIgfsConfiguration();
igfsCfg.setDefaultMode(DUAL_ASYNC);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoop20FileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoop20FileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoop20FileSystemAbstractSelfTest.java
deleted file mode 100644
index 9f9a6d8..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoop20FileSystemAbstractSelfTest.java
+++ /dev/null
@@ -1,1967 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.*;
-import org.apache.ignite.*;
-import org.apache.ignite.cache.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.hadoop.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.lang.*;
-import org.apache.ignite.spi.communication.*;
-import org.apache.ignite.spi.communication.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
-import org.apache.ignite.testframework.*;
-import org.jdk8.backport.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.cache.CacheMode.*;
-import static org.apache.ignite.events.EventType.*;
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * Hadoop 2.x compliant file system.
- */
-public abstract class IgfsHadoop20FileSystemAbstractSelfTest extends IgfsCommonAbstractTest {
- /** Group size. */
- public static final int GRP_SIZE = 128;
-
- /** Thread count for multithreaded tests. */
- private static final int THREAD_CNT = 8;
-
- /** IP finder. */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** Barrier for multithreaded tests. */
- private static CyclicBarrier barrier;
-
- /** File system. */
- private static AbstractFileSystem fs;
-
- /** Default IGFS mode. */
- protected IgfsMode mode;
-
- /** Primary file system URI. */
- protected URI primaryFsUri;
-
- /** Primary file system configuration. */
- protected Configuration primaryFsCfg;
-
- /**
- * Constructor.
- *
- * @param mode Default IGFS mode.
- */
- protected IgfsHadoop20FileSystemAbstractSelfTest(IgfsMode mode) {
- this.mode = mode;
- }
-
- /**
- * Gets primary file system URI path.
- *
- * @return Primary file system URI path.
- */
- protected abstract String primaryFileSystemUriPath();
-
- /**
- * Gets primary file system config path.
- *
- * @return Primary file system config path.
- */
- protected abstract String primaryFileSystemConfigPath();
-
- /**
- * Get primary IPC endpoint configuration.
- *
- * @param gridName Grid name.
- * @return IPC primary endpoint configuration.
- */
- protected abstract Map<String, String> primaryIpcEndpointConfiguration(String gridName);
-
- /**
- * Gets secondary file system URI path.
- *
- * @return Secondary file system URI path.
- */
- protected abstract String secondaryFileSystemUriPath();
-
- /**
- * Gets secondary file system config path.
- *
- * @return Secondary file system config path.
- */
- protected abstract String secondaryFileSystemConfigPath();
-
- /**
- * Get secondary IPC endpoint configuration.
- *
- * @return Secondary IPC endpoint configuration.
- */
- protected abstract Map<String, String> secondaryIpcEndpointConfiguration();
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- startNodes();
- }
-
- /**
- * Starts the nodes for this test.
- *
- * @throws Exception If failed.
- */
- private void startNodes() throws Exception {
- if (mode != PRIMARY) {
- // Start secondary IGFS.
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("partitioned");
- igfsCfg.setMetaCacheName("replicated");
- igfsCfg.setName("igfs_secondary");
- igfsCfg.setIpcEndpointConfiguration(secondaryIpcEndpointConfiguration());
- igfsCfg.setManagementPort(-1);
- igfsCfg.setBlockSize(512 * 1024);
- igfsCfg.setPrefetchBlocks(1);
-
- CacheConfiguration cacheCfg = defaultCacheConfiguration();
-
- cacheCfg.setName("partitioned");
- cacheCfg.setCacheMode(PARTITIONED);
- cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
- cacheCfg.setBackups(0);
- cacheCfg.setQueryIndexEnabled(false);
- cacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("replicated");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- IgniteConfiguration cfg = new IgniteConfiguration();
-
- cfg.setGridName("grid_secondary");
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
- cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
- cfg.setLocalHost(U.getLocalHost().getHostAddress());
- cfg.setCommunicationSpi(communicationSpi());
-
- G.start(cfg);
- }
-
- startGrids(4);
- }
-
- /** {@inheritDoc} */
- @Override public String getTestGridName() {
- return "grid";
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(IP_FINDER);
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(cacheConfiguration(gridName));
- cfg.setIgfsConfiguration(igfsConfiguration(gridName));
- cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
- cfg.setLocalHost("127.0.0.1");
- cfg.setCommunicationSpi(communicationSpi());
-
- return cfg;
- }
-
- /**
- * Gets cache configuration.
- *
- * @param gridName Grid name.
- * @return Cache configuration.
- */
- protected CacheConfiguration[] cacheConfiguration(String gridName) {
- CacheConfiguration cacheCfg = defaultCacheConfiguration();
-
- cacheCfg.setName("partitioned");
- cacheCfg.setCacheMode(PARTITIONED);
- cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
- cacheCfg.setBackups(0);
- cacheCfg.setQueryIndexEnabled(false);
- cacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("replicated");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- return new CacheConfiguration[] {metaCacheCfg, cacheCfg};
- }
-
- /**
- * Gets IGFS configuration.
- *
- * @param gridName Grid name.
- * @return IGFS configuration.
- */
- protected IgfsConfiguration igfsConfiguration(String gridName) throws IgniteCheckedException {
- IgfsConfiguration cfg = new IgfsConfiguration();
-
- cfg.setDataCacheName("partitioned");
- cfg.setMetaCacheName("replicated");
- cfg.setName("igfs");
- cfg.setPrefetchBlocks(1);
- cfg.setMaxSpaceSize(64 * 1024 * 1024);
- cfg.setDefaultMode(mode);
-
- if (mode != PRIMARY)
- cfg.setSecondaryFileSystem(new IgfsHadoopFileSystemWrapper(secondaryFileSystemUriPath(),
- secondaryFileSystemConfigPath()));
-
- cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(gridName));
- cfg.setManagementPort(-1);
-
- cfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups.
-
- return cfg;
- }
-
- /** @return Communication SPI. */
- private CommunicationSpi communicationSpi() {
- TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
-
- commSpi.setSharedMemoryPort(-1);
-
- return commSpi;
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- G.stopAll(true);
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- primaryFsUri = new URI(primaryFileSystemUriPath());
-
- primaryFsCfg = new Configuration();
-
- primaryFsCfg.addResource(U.resolveIgniteUrl(primaryFileSystemConfigPath()));
-
- fs = AbstractFileSystem.get(primaryFsUri, primaryFsCfg);
-
- barrier = new CyclicBarrier(THREAD_CNT);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- try {
- fs.delete(new Path("/"), true);
- }
- catch (Exception ignore) {
- // No-op.
- }
-
- U.closeQuiet((Closeable)fs);
- }
-
- /** @throws Exception If failed. */
- public void testStatus() throws Exception {
-
- try (FSDataOutputStream file = fs.create(new Path("/file1"), EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()))) {
- file.write(new byte[1024 * 1024]);
- }
-
- FsStatus status = fs.getFsStatus();
-
- assertEquals(4, grid(0).cluster().nodes().size());
-
- long used = 0, max = 0;
-
- for (int i = 0; i < 4; i++) {
- IgniteFs igfs = grid(i).fileSystem("igfs");
-
- IgfsMetrics metrics = igfs.metrics();
-
- used += metrics.localSpaceSize();
- max += metrics.maxSpaceSize();
- }
-
- assertEquals(used, status.getUsed());
- assertEquals(max, status.getCapacity());
- }
-
- /** @throws Exception If failed. */
- public void testTimes() throws Exception {
- Path file = new Path("/file1");
-
- long now = System.currentTimeMillis();
-
- try (FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()))) {
- os.write(new byte[1024 * 1024]);
- }
-
- FileStatus status = fs.getFileStatus(file);
-
- assertTrue(status.getAccessTime() >= now);
- assertTrue(status.getModificationTime() >= now);
-
- long accessTime = now - 10 * 60 * 1000;
- long modificationTime = now - 5 * 60 * 1000;
-
- fs.setTimes(file, modificationTime, accessTime);
-
- status = fs.getFileStatus(file);
- assertEquals(accessTime, status.getAccessTime());
- assertEquals(modificationTime, status.getModificationTime());
-
- // Check listing is updated as well.
- FileStatus[] files = fs.listStatus(new Path("/"));
-
- assertEquals(1, files.length);
-
- assertEquals(file.getName(), files[0].getPath().getName());
- assertEquals(accessTime, files[0].getAccessTime());
- assertEquals(modificationTime, files[0].getModificationTime());
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.setTimes(new Path("/unknownFile"), 0, 0);
-
- return null;
- }
- }, FileNotFoundException.class, null);
- }
-
- /** @throws Exception If failed. */
- public void testCreateCheckParameters() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.create(null, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
- }
- }, NullPointerException.class, null);
- }
-
- /** @throws Exception If failed. */
- public void testCreateBase() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
- Path file = new Path(dir, "someFile");
-
- assertPathDoesNotExist(fs, file);
-
- FsPermission fsPerm = new FsPermission((short)644);
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(fsPerm));
-
- // Try to write something in file.
- os.write("abc".getBytes());
-
- os.close();
-
- // Check file status.
- FileStatus fileStatus = fs.getFileStatus(file);
-
- assertFalse(fileStatus.isDirectory());
- assertEquals(file, fileStatus.getPath());
- assertEquals(fsPerm, fileStatus.getPermission());
- }
-
- /** @throws Exception If failed. */
- public void testCreateCheckOverwrite() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
- final Path file = new Path(dir, "someFile");
-
- FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- out.close();
-
- // Check intermediate directory permissions.
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir).getPermission());
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir.getParent()).getPermission());
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir.getParent().getParent()).getPermission());
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
- }
- }, PathExistsException.class, null);
-
- // Overwrite should be successful.
- FSDataOutputStream out1 = fs.create(file, EnumSet.of(CreateFlag.OVERWRITE),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- out1.close();
- }
-
- /** @throws Exception If failed. */
- public void testDeleteIfNoSuchPath() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
-
- assertPathDoesNotExist(fs, dir);
-
- assertFalse(fs.delete(dir, true));
- }
-
- /** @throws Exception If failed. */
- public void testDeleteSuccessfulIfPathIsOpenedToRead() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "myFile");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- final int cnt = 5 * IgfsConfiguration.DFLT_BLOCK_SIZE; // Write 5 blocks.
-
- for (int i = 0; i < cnt; i++)
- os.writeInt(i);
-
- os.close();
-
- final FSDataInputStream is = fs.open(file, -1);
-
- for (int i = 0; i < cnt / 2; i++)
- assertEquals(i, is.readInt());
-
- assert fs.delete(file, false);
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.getFileStatus(file);
-
- return null;
- }
- }, FileNotFoundException.class, null);
-
- is.close();
- }
-
- /** @throws Exception If failed. */
- public void testDeleteIfFilePathExists() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "myFile");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- assertTrue(fs.delete(file, false));
-
- assertPathDoesNotExist(fs, file);
- }
-
- /** @throws Exception If failed. */
- public void testDeleteIfDirectoryPathExists() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
-
- FSDataOutputStream os = fs.create(dir, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- assertTrue(fs.delete(dir, false));
-
- assertPathDoesNotExist(fs, dir);
- }
-
- /** @throws Exception If failed. */
- public void testDeleteFailsIfNonRecursive() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
-
- FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- final Path someDir2 = new Path(fsHome, "/someDir1/someDir2");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.delete(someDir2, false);
-
- return null;
- }
- }, PathIsNotEmptyDirectoryException.class, null);
-
- assertPathExists(fs, someDir2);
- assertPathExists(fs, someDir3);
- }
-
- /** @throws Exception If failed. */
- public void testDeleteRecursively() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
-
- FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- Path someDir2 = new Path(fsHome, "/someDir1/someDir2");
-
- assertTrue(fs.delete(someDir2, true));
-
- assertPathDoesNotExist(fs, someDir2);
- assertPathDoesNotExist(fs, someDir3);
- }
-
- /** @throws Exception If failed. */
- public void testDeleteRecursivelyFromRoot() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
-
- FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- Path root = new Path(fsHome, "/");
-
- assertTrue(fs.delete(root, true));
-
- assertPathDoesNotExist(fs, someDir3);
- assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));
- assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1"));
- assertPathExists(fs, root);
- }
-
- /** @throws Exception If failed. */
- public void testSetPermissionCheckDefaultPermission() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- fs.setPermission(file, null);
-
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(file.getParent()).getPermission());
- }
-
- /** @throws Exception If failed. */
- public void testSetPermissionCheckNonRecursiveness() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- Path tmpDir = new Path(fsHome, "/tmp");
-
- FsPermission perm = new FsPermission((short)123);
-
- fs.setPermission(tmpDir, perm);
-
- assertEquals(perm, fs.getFileStatus(tmpDir).getPermission());
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
- }
-
- /** @throws Exception If failed. */
- @SuppressWarnings("OctalInteger")
- public void testSetPermission() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- for (short i = 0; i <= 0777; i += 7) {
- FsPermission perm = new FsPermission(i);
-
- fs.setPermission(file, perm);
-
- assertEquals(perm, fs.getFileStatus(file).getPermission());
- }
- }
-
- /** @throws Exception If failed. */
- public void testSetPermissionIfOutputStreamIsNotClosed() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "myFile");
-
- FsPermission perm = new FsPermission((short)123);
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- fs.setPermission(file, perm);
-
- os.close();
-
- assertEquals(perm, fs.getFileStatus(file).getPermission());
- }
-
- /** @throws Exception If failed. */
- public void testSetOwnerCheckParametersPathIsNull() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.setOwner(null, "aUser", "aGroup");
-
- return null;
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: p");
- }
-
- /** @throws Exception If failed. */
- public void testSetOwnerCheckParametersUserIsNull() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.setOwner(file, null, "aGroup");
-
- return null;
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: username");
- }
-
- /** @throws Exception If failed. */
- public void testSetOwnerCheckParametersGroupIsNull() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- fs.setOwner(file, "aUser", null);
-
- return null;
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: grpName");
- }
-
- /** @throws Exception If failed. */
- public void testSetOwner() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- fs.setOwner(file, "aUser", "aGroup");
-
- assertEquals("aUser", fs.getFileStatus(file).getOwner());
- assertEquals("aGroup", fs.getFileStatus(file).getGroup());
- }
-
- /** @throws Exception If failed. */
- public void testSetOwnerIfOutputStreamIsNotClosed() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "myFile");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- fs.setOwner(file, "aUser", "aGroup");
-
- os.close();
-
- assertEquals("aUser", fs.getFileStatus(file).getOwner());
- assertEquals("aGroup", fs.getFileStatus(file).getGroup());
- }
-
- /** @throws Exception If failed. */
- public void testSetOwnerCheckNonRecursiveness() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "/tmp/my");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- Path tmpDir = new Path(fsHome, "/tmp");
-
- fs.setOwner(file, "fUser", "fGroup");
- fs.setOwner(tmpDir, "dUser", "dGroup");
-
- assertEquals("dUser", fs.getFileStatus(tmpDir).getOwner());
- assertEquals("dGroup", fs.getFileStatus(tmpDir).getGroup());
-
- assertEquals("fUser", fs.getFileStatus(file).getOwner());
- assertEquals("fGroup", fs.getFileStatus(file).getGroup());
- }
-
- /** @throws Exception If failed. */
- public void testOpenCheckParametersPathIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.open(null, 1024);
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: f");
- }
-
- /** @throws Exception If failed. */
- public void testOpenNoSuchPath() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "someFile");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.open(file, 1024);
- }
- }, FileNotFoundException.class, null);
- }
-
- /** @throws Exception If failed. */
- public void testOpenIfPathIsAlreadyOpened() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "someFile");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- FSDataInputStream is1 = fs.open(file);
- FSDataInputStream is2 = fs.open(file);
-
- is1.close();
- is2.close();
- }
-
- /** @throws Exception If failed. */
- public void testOpen() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "someFile");
-
- int cnt = 2 * 1024;
-
- FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- for (long i = 0; i < cnt; i++)
- out.writeLong(i);
-
- out.close();
-
- FSDataInputStream in = fs.open(file, 1024);
-
- for (long i = 0; i < cnt; i++)
- assertEquals(i, in.readLong());
-
- in.close();
- }
-
- /** @throws Exception If failed. */
- public void testAppendIfPathPointsToDirectory() throws Exception {
- final Path fsHome = new Path(primaryFsUri);
- final Path dir = new Path(fsHome, "/tmp");
- Path file = new Path(dir, "my");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.create(new Path(fsHome, dir), EnumSet.of(CreateFlag.APPEND),
- Options.CreateOpts.perms(FsPermission.getDefault()));
- }
- }, IOException.class, null);
- }
-
- /** @throws Exception If failed. */
- public void testAppendIfFileIsAlreadyBeingOpenedToWrite() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "someFile");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- FSDataOutputStream appendOs = fs.create(file, EnumSet.of(CreateFlag.APPEND),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- return fs.create(file, EnumSet.of(CreateFlag.APPEND),
- Options.CreateOpts.perms(FsPermission.getDefault()));
- }
- }, IOException.class, null);
-
- appendOs.close();
- }
-
- /** @throws Exception If failed. */
- public void testAppend() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path file = new Path(fsHome, "someFile");
-
- int cnt = 1024;
-
- FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- for (int i = 0; i < cnt; i++)
- out.writeLong(i);
-
- out.close();
-
- out = fs.create(file, EnumSet.of(CreateFlag.APPEND),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- for (int i = cnt; i < cnt * 2; i++)
- out.writeLong(i);
-
- out.close();
-
- FSDataInputStream in = fs.open(file, 1024);
-
- for (int i = 0; i < cnt * 2; i++)
- assertEquals(i, in.readLong());
-
- in.close();
- }
-
- /** @throws Exception If failed. */
- public void testRenameCheckParametersSrcPathIsNull() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "someFile");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.rename(null, file);
-
- return null;
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: f");
- }
-
- /** @throws Exception If failed. */
- public void testRenameCheckParametersDstPathIsNull() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path file = new Path(fsHome, "someFile");
-
- fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault())).close();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- fs.rename(file, null);
-
- return null;
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: f");
- }
-
- /** @throws Exception If failed. */
- public void testRenameIfSrcPathDoesNotExist() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path srcFile = new Path(fsHome, "srcFile");
- final Path dstFile = new Path(fsHome, "dstFile");
-
- assertPathDoesNotExist(fs, srcFile);
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.rename(srcFile, dstFile);
-
- return null;
- }
- }, FileNotFoundException.class, null);
-
- assertPathDoesNotExist(fs, dstFile);
- }
-
- /** @throws Exception If failed. */
- public void testRenameIfSrcPathIsAlreadyBeingOpenedToWrite() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path srcFile = new Path(fsHome, "srcFile");
- Path dstFile = new Path(fsHome, "dstFile");
-
- FSDataOutputStream os = fs.create(srcFile, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- os = fs.create(srcFile, EnumSet.of(CreateFlag.APPEND),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- fs.rename(srcFile, dstFile);
-
- assertPathExists(fs, dstFile);
-
- String testStr = "Test";
-
- try {
- os.writeBytes(testStr);
- }
- finally {
- os.close();
- }
-
- try (FSDataInputStream is = fs.open(dstFile)) {
- byte[] buf = new byte[testStr.getBytes().length];
-
- is.readFully(buf);
-
- assertEquals(testStr, new String(buf));
- }
- }
-
- /** @throws Exception If failed. */
- public void testRenameFileIfDstPathExists() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- final Path srcFile = new Path(fsHome, "srcFile");
- final Path dstFile = new Path(fsHome, "dstFile");
-
- FSDataOutputStream os = fs.create(srcFile, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- os = fs.create(dstFile, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.rename(srcFile, dstFile);
-
- return null;
- }
- }, FileAlreadyExistsException.class, null);
-
- assertPathExists(fs, srcFile);
- assertPathExists(fs, dstFile);
- }
-
- /** @throws Exception If failed. */
- public void testRenameFile() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path srcFile = new Path(fsHome, "/tmp/srcFile");
- Path dstFile = new Path(fsHome, "/tmp/dstFile");
-
- FSDataOutputStream os = fs.create(srcFile, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- fs.rename(srcFile, dstFile);
-
- assertPathDoesNotExist(fs, srcFile);
- assertPathExists(fs, dstFile);
- }
-
- /** @throws Exception If failed. */
- public void testRenameIfSrcPathIsAlreadyBeingOpenedToRead() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path srcFile = new Path(fsHome, "srcFile");
- Path dstFile = new Path(fsHome, "dstFile");
-
- FSDataOutputStream os = fs.create(srcFile, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- int cnt = 1024;
-
- for (int i = 0; i < cnt; i++)
- os.writeInt(i);
-
- os.close();
-
- FSDataInputStream is = fs.open(srcFile);
-
- for (int i = 0; i < cnt; i++) {
- if (i == 100)
- // Rename file during the read process.
- fs.rename(srcFile, dstFile);
-
- assertEquals(i, is.readInt());
- }
-
- assertPathDoesNotExist(fs, srcFile);
- assertPathExists(fs, dstFile);
-
- os.close();
- is.close();
- }
-
- /** @throws Exception If failed. */
- public void _testRenameDirectoryIfDstPathExists() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path srcDir = new Path(fsHome, "/tmp/");
- Path dstDir = new Path(fsHome, "/tmpNew/");
-
- FSDataOutputStream os = fs.create(new Path(srcDir, "file1"), EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- os = fs.create(new Path(dstDir, "file2"), EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- fs.rename(srcDir, dstDir);
-
- assertPathExists(fs, dstDir);
- assertPathExists(fs, new Path(fsHome, "/tmpNew/tmp"));
- assertPathExists(fs, new Path(fsHome, "/tmpNew/tmp/file1"));
- }
-
- /** @throws Exception If failed. */
- public void testRenameDirectory() throws Exception {
- Path fsHome = new Path(primaryFsUri);
- Path dir = new Path(fsHome, "/tmp/");
- Path newDir = new Path(fsHome, "/tmpNew/");
-
- FSDataOutputStream os = fs.create(new Path(dir, "myFile"), EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.close();
-
- fs.rename(dir, newDir);
-
- assertPathDoesNotExist(fs, dir);
- assertPathExists(fs, newDir);
- }
-
- /** @throws Exception If failed. */
- public void testListStatusIfPathIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.listStatus(null);
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: f");
- }
-
- /** @throws Exception If failed. */
- public void testListStatusIfPathDoesNotExist() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.listStatus(new Path("/someDir"));
- }
- }, FileNotFoundException.class, null);
- }
-
- /**
- * Test directory listing.
- *
- * @throws Exception If failed.
- */
- public void testListStatus() throws Exception {
- Path igfsHome = new Path(primaryFsUri);
-
- // Test listing of an empty directory.
- Path dir = new Path(igfsHome, "dir");
-
- fs.mkdir(dir, FsPermission.getDefault(), true);
-
- FileStatus[] list = fs.listStatus(dir);
-
- assert list.length == 0;
-
- // Test listing of a not empty directory.
- Path subDir = new Path(dir, "subDir");
-
- fs.mkdir(subDir, FsPermission.getDefault(), true);
-
- Path file = new Path(dir, "file");
-
- FSDataOutputStream fos = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- fos.close();
-
- list = fs.listStatus(dir);
-
- assert list.length == 2;
-
- String listRes1 = list[0].getPath().getName();
- String listRes2 = list[1].getPath().getName();
-
- assert "subDir".equals(listRes1) && "file".equals(listRes2) || "subDir".equals(listRes2) &&
- "file".equals(listRes1);
-
- // Test listing of a file.
- list = fs.listStatus(file);
-
- assert list.length == 1;
-
- assert "file".equals(list[0].getPath().getName());
- }
-
- /** @throws Exception If failed. */
- public void testMkdirsIfPathIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.mkdir(null, FsPermission.getDefault(), true);
-
- return null;
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: f");
- }
-
- /** @throws Exception If failed. */
- public void testMkdirsIfPermissionIsNull() throws Exception {
- Path dir = new Path("/tmp");
-
- fs.mkdir(dir, null, true);
-
- assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir).getPermission());
- }
-
- /** @throws Exception If failed. */
- @SuppressWarnings("OctalInteger")
- public void testMkdirs() throws Exception {
- Path fsHome = new Path(primaryFileSystemUriPath());
- Path dir = new Path(fsHome, "/tmp/staging");
- Path nestedDir = new Path(dir, "nested");
-
- FsPermission dirPerm = FsPermission.createImmutable((short)0700);
- FsPermission nestedDirPerm = FsPermission.createImmutable((short)111);
-
- fs.mkdir(dir, dirPerm, true);
- fs.mkdir(nestedDir, nestedDirPerm, true);
-
- assertEquals(dirPerm, fs.getFileStatus(dir).getPermission());
- assertEquals(nestedDirPerm, fs.getFileStatus(nestedDir).getPermission());
- }
-
- /** @throws Exception If failed. */
- public void testGetFileStatusIfPathIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.getFileStatus(null);
- }
- }, NullPointerException.class, "Ouch! Argument cannot be null: f");
- }
-
- /** @throws Exception If failed. */
- public void testGetFileStatusIfPathDoesNotExist() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.getFileStatus(new Path("someDir"));
- }
- }, FileNotFoundException.class, "File not found: someDir");
- }
-
- /** @throws Exception If failed. */
- public void testGetFileBlockLocationsIfFileStatusIsNull() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- // Argument is checked by Hadoop.
- return fs.getFileBlockLocations(null, 1, 2);
- }
- }, NullPointerException.class, null);
- }
-
- /** @throws Exception If failed. */
- public void testGetFileBlockLocationsIfFileStatusReferenceNotExistingPath() throws Exception {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.getFileBlockLocations(new Path("/someFile"), 1, 2);
- }
- }, FileNotFoundException.class, null);
- }
-
- /** @throws Exception If failed. */
- public void testGetFileBlockLocations() throws Exception {
- Path igfsHome = new Path(primaryFsUri);
-
- Path file = new Path(igfsHome, "someFile");
-
- try (OutputStream out = new BufferedOutputStream(fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault())))) {
- byte[] data = new byte[128 * 1024];
-
- for (int i = 0; i < 100; i++)
- out.write(data);
-
- out.flush();
- }
-
- try (FSDataInputStream in = fs.open(file, 1024 * 1024)) {
- byte[] data = new byte[128 * 1024];
-
- int read;
-
- do {
- read = in.read(data);
- }
- while (read > 0);
- }
-
- FileStatus status = fs.getFileStatus(file);
-
- int grpLen = 128 * 512 * 1024;
-
- int grpCnt = (int)((status.getLen() + grpLen - 1) / grpLen);
-
- BlockLocation[] locations = fs.getFileBlockLocations(file, 0, status.getLen());
-
- assertEquals(grpCnt, locations.length);
- }
-
- /** @throws Exception If failed. */
- public void testZeroReplicationFactor() throws Exception {
- // This test doesn't make sense for any mode except of PRIMARY.
- if (mode == PRIMARY) {
- Path igfsHome = new Path(primaryFsUri);
-
- Path file = new Path(igfsHome, "someFile");
-
- try (FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()), Options.CreateOpts.repFac((short)1))) {
- out.write(new byte[1024 * 1024]);
- }
-
- IgniteFs igfs = grid(0).fileSystem("igfs");
-
- IgfsPath filePath = new IgfsPath("/someFile");
-
- IgfsFile fileInfo = igfs.info(filePath);
-
- Collection<IgfsBlockLocation> locations = igfs.affinity(filePath, 0, fileInfo.length());
-
- assertEquals(1, locations.size());
-
- IgfsBlockLocation location = F.first(locations);
-
- assertEquals(1, location.nodeIds().size());
- }
- }
-
- /**
- * Ensure that when running in multithreaded mode only one create() operation succeed.
- *
- * @throws Exception If failed.
- */
- public void testMultithreadedCreate() throws Exception {
- Path dir = new Path(new Path(primaryFsUri), "/dir");
-
- fs.mkdir(dir, FsPermission.getDefault(), true);
-
- final Path file = new Path(dir, "file");
-
- fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault())).close();
-
- final AtomicInteger cnt = new AtomicInteger();
-
- final Collection<Integer> errs = new GridConcurrentHashSet<>(THREAD_CNT, 1.0f, THREAD_CNT);
-
- multithreaded(new Runnable() {
- @Override public void run() {
- int idx = cnt.getAndIncrement();
-
- byte[] data = new byte[256];
-
- Arrays.fill(data, (byte)idx);
-
- FSDataOutputStream os = null;
-
- try {
- os = fs.create(file, EnumSet.of(CreateFlag.OVERWRITE),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.write(data);
- }
- catch (IOException ignore) {
- errs.add(idx);
- }
- finally {
- U.awaitQuiet(barrier);
-
- U.closeQuiet(os);
- }
- }
- }, THREAD_CNT);
-
- // Only one thread could obtain write lock on the file.
- assert errs.size() == THREAD_CNT - 1 : "Invalid errors count [expected=" + (THREAD_CNT - 1) + ", actual=" +
- errs.size() + ']';
-
- int idx = -1;
-
- for (int i = 0; i < THREAD_CNT; i++) {
- if (!errs.remove(i)) {
- idx = i;
-
- break;
- }
- }
-
- byte[] expData = new byte[256];
-
- Arrays.fill(expData, (byte)idx);
-
- FSDataInputStream is = fs.open(file);
-
- byte[] data = new byte[256];
-
- is.read(data);
-
- is.close();
-
- assert Arrays.equals(expData, data);
- }
-
- /**
- * Ensure that when running in multithreaded mode only one append() operation succeed.
- *
- * @throws Exception If failed.
- */
- public void testMultithreadedAppend() throws Exception {
- Path dir = new Path(new Path(primaryFsUri), "/dir");
-
- fs.mkdir(dir, FsPermission.getDefault(), true);
-
- final Path file = new Path(dir, "file");
-
- fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault())).close();
-
- final AtomicInteger cnt = new AtomicInteger();
-
- final Collection<Integer> errs = new GridConcurrentHashSet<>(THREAD_CNT, 1.0f, THREAD_CNT);
-
- multithreaded(new Runnable() {
- @Override public void run() {
- int idx = cnt.getAndIncrement();
-
- byte[] data = new byte[256];
-
- Arrays.fill(data, (byte)idx);
-
- U.awaitQuiet(barrier);
-
- FSDataOutputStream os = null;
-
- try {
- os = fs.create(file, EnumSet.of(CreateFlag.APPEND),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- os.write(data);
- }
- catch (IOException ignore) {
- errs.add(idx);
- }
- finally {
- U.awaitQuiet(barrier);
-
- U.closeQuiet(os);
- }
- }
- }, THREAD_CNT);
-
- // Only one thread could obtain write lock on the file.
- assert errs.size() == THREAD_CNT - 1;
-
- int idx = -1;
-
- for (int i = 0; i < THREAD_CNT; i++) {
- if (!errs.remove(i)) {
- idx = i;
-
- break;
- }
- }
-
- byte[] expData = new byte[256];
-
- Arrays.fill(expData, (byte)idx);
-
- FSDataInputStream is = fs.open(file);
-
- byte[] data = new byte[256];
-
- is.read(data);
-
- is.close();
-
- assert Arrays.equals(expData, data);
- }
-
- /**
- * Test concurrent reads within the file.
- *
- * @throws Exception If failed.
- */
- public void testMultithreadedOpen() throws Exception {
- final byte[] dataChunk = new byte[256];
-
- for (int i = 0; i < dataChunk.length; i++)
- dataChunk[i] = (byte)i;
-
- Path dir = new Path(new Path(primaryFsUri), "/dir");
-
- fs.mkdir(dir, FsPermission.getDefault(), true);
-
- final Path file = new Path(dir, "file");
-
- FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault()));
-
- // Write 256 * 2048 = 512Kb of data.
- for (int i = 0; i < 2048; i++)
- os.write(dataChunk);
-
- os.close();
-
- final AtomicBoolean err = new AtomicBoolean();
-
- multithreaded(new Runnable() {
- @Override
- public void run() {
- FSDataInputStream is = null;
-
- try {
- int pos = ThreadLocalRandom8.current().nextInt(2048);
-
- try {
- is = fs.open(file);
- }
- finally {
- U.awaitQuiet(barrier);
- }
-
- is.seek(256 * pos);
-
- byte[] buf = new byte[256];
-
- for (int i = pos; i < 2048; i++) {
- // First perform normal read.
- int read = is.read(buf);
-
- assert read == 256;
-
- Arrays.equals(dataChunk, buf);
- }
-
- int res = is.read(buf);
-
- assert res == -1;
- }
- catch (IOException ignore) {
- err.set(true);
- }
- finally {
- U.closeQuiet(is);
- }
- }
- }, THREAD_CNT);
-
- assert !err.get();
- }
-
- /**
- * Test concurrent creation of multiple directories.
- *
- * @throws Exception If failed.
- */
- public void testMultithreadedMkdirs() throws Exception {
- final Path dir = new Path(new Path("igfs:///"), "/dir");
-
- fs.mkdir(dir, FsPermission.getDefault(), true);
-
- final int depth = 3;
- final int entryCnt = 5;
-
- final AtomicBoolean err = new AtomicBoolean();
-
- multithreaded(new Runnable() {
- @Override public void run() {
- Deque<IgniteBiTuple<Integer, Path>> queue = new ArrayDeque<>();
-
- queue.add(F.t(0, dir));
-
- U.awaitQuiet(barrier);
-
- while (!queue.isEmpty()) {
- IgniteBiTuple<Integer, Path> t = queue.pollFirst();
-
- int curDepth = t.getKey();
- Path curPath = t.getValue();
-
- if (curDepth <= depth) {
- int newDepth = curDepth + 1;
-
- // Create directories.
- for (int i = 0; i < entryCnt; i++) {
- Path subDir = new Path(curPath, "dir-" + newDepth + "-" + i);
-
- try {
- fs.mkdir(subDir, FsPermission.getDefault(), true);
- }
- catch (IOException ignore) {
- err.set(true);
- }
-
- queue.addLast(F.t(newDepth, subDir));
- }
- }
- }
- }
- }, THREAD_CNT);
-
- // Ensure there were no errors.
- assert !err.get();
-
- // Ensure correct folders structure.
- Deque<IgniteBiTuple<Integer, Path>> queue = new ArrayDeque<>();
-
- queue.add(F.t(0, dir));
-
- while (!queue.isEmpty()) {
- IgniteBiTuple<Integer, Path> t = queue.pollFirst();
-
- int curDepth = t.getKey();
- Path curPath = t.getValue();
-
- if (curDepth <= depth) {
- int newDepth = curDepth + 1;
-
- // Create directories.
- for (int i = 0; i < entryCnt; i++) {
- Path subDir = new Path(curPath, "dir-" + newDepth + "-" + i);
-
- assertNotNull(fs.getFileStatus(subDir));
-
- queue.add(F.t(newDepth, subDir));
- }
- }
- }
- }
-
- /**
- * Test concurrent deletion of the same directory with advanced structure.
- *
- * @throws Exception If failed.
- */
- @SuppressWarnings("TooBroadScope")
- public void testMultithreadedDelete() throws Exception {
- final Path dir = new Path(new Path(primaryFsUri), "/dir");
-
- fs.mkdir(dir, FsPermission.getDefault(), true);
-
- int depth = 3;
- int entryCnt = 5;
-
- Deque<IgniteBiTuple<Integer, Path>> queue = new ArrayDeque<>();
-
- queue.add(F.t(0, dir));
-
- while (!queue.isEmpty()) {
- IgniteBiTuple<Integer, Path> t = queue.pollFirst();
-
- int curDepth = t.getKey();
- Path curPath = t.getValue();
-
- if (curDepth < depth) {
- int newDepth = curDepth + 1;
-
- // Create directories.
- for (int i = 0; i < entryCnt; i++) {
- Path subDir = new Path(curPath, "dir-" + newDepth + "-" + i);
-
- fs.mkdir(subDir, FsPermission.getDefault(), true);
-
- queue.addLast(F.t(newDepth, subDir));
- }
- }
- else {
- // Create files.
- for (int i = 0; i < entryCnt; i++) {
- Path file = new Path(curPath, "file " + i);
-
- fs.create(file, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault())).close();
- }
- }
- }
-
- final AtomicBoolean err = new AtomicBoolean();
-
- multithreaded(new Runnable() {
- @Override public void run() {
- try {
- U.awaitQuiet(barrier);
-
- fs.delete(dir, true);
- }
- catch (FileNotFoundException ignore) {
- // No-op.
- }
- catch (IOException ignore) {
- err.set(true);
- }
- }
- }, THREAD_CNT);
-
- // Ensure there were no errors.
- assert !err.get();
-
- // Ensure the directory was actually deleted.
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.getFileStatus(dir);
-
- return null;
- }
- }, FileNotFoundException.class, null);
- }
-
- /** @throws Exception If failed. */
- public void testConsistency() throws Exception {
- // Default buffers values
- checkConsistency(-1, 1, -1, -1, 1, -1);
- checkConsistency(-1, 10, -1, -1, 10, -1);
- checkConsistency(-1, 100, -1, -1, 100, -1);
- checkConsistency(-1, 1000, -1, -1, 1000, -1);
- checkConsistency(-1, 10000, -1, -1, 10000, -1);
- checkConsistency(-1, 100000, -1, -1, 100000, -1);
-
- checkConsistency(65 * 1024 + 13, 100000, -1, -1, 100000, -1);
-
- checkConsistency(-1, 100000, 2 * 4 * 1024 + 17, -1, 100000, -1);
-
- checkConsistency(-1, 100000, -1, 65 * 1024 + 13, 100000, -1);
-
- checkConsistency(-1, 100000, -1, -1, 100000, 2 * 4 * 1024 + 17);
-
- checkConsistency(65 * 1024 + 13, 100000, 2 * 4 * 1024 + 13, 65 * 1024 + 149, 100000, 2 * 4 * 1024 + 157);
- }
-
- /**
- * Verifies that client reconnects after connection to the server has been lost.
- *
- * @throws Exception If error occurs.
- */
- public void testClientReconnect() throws Exception {
- final Path igfsHome = new Path(primaryFsUri);
-
- final Path filePath = new Path(igfsHome, "someFile");
-
- final FSDataOutputStream s = fs.create(filePath, EnumSet.noneOf(CreateFlag.class),
- Options.CreateOpts.perms(FsPermission.getDefault())); // Open stream before stopping IGFS.
-
- try {
- G.stopAll(true); // Stop the server.
-
- startNodes(); // Start server again.
-
- // Check that client is again operational.
- fs.mkdir(new Path("igfs:///dir1/dir2"), FsPermission.getDefault(), true);
-
- // However, the streams, opened before disconnect, should not be valid.
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Nullable @Override public Object call() throws Exception {
- s.write("test".getBytes());
-
- s.flush();
-
- return null;
- }
- }, IOException.class, null);
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fs.getFileStatus(filePath);
-
- return null;
- }
- }, FileNotFoundException.class, null);
- }
- finally {
- U.closeQuiet(s);
- }
- }
-
- /**
- * Verifies that client reconnects after connection to the server has been lost (multithreaded mode).
- *
- * @throws Exception If error occurs.
- */
- public void testClientReconnectMultithreaded() throws Exception {
- final ConcurrentLinkedQueue<FileSystem> q = new ConcurrentLinkedQueue<>();
-
- Configuration cfg = new Configuration();
-
- for (Map.Entry<String, String> entry : primaryFsCfg)
- cfg.set(entry.getKey(), entry.getValue());
-
- cfg.setBoolean("fs.igfs.impl.disable.cache", true);
-
- final int nClients = 16;
-
- // Initialize clients.
- for (int i = 0; i < nClients; i++)
- q.add(FileSystem.get(primaryFsUri, cfg));
-
- G.stopAll(true); // Stop the server.
-
- startNodes(); // Start server again.
-
- GridTestUtils.runMultiThreaded(new Callable<Object>() {
- @Override public Object call() throws Exception {
- FileSystem fs = q.poll();
-
- try {
- // Check that client is again operational.
- assertTrue(fs.mkdirs(new Path("igfs:///" + Thread.currentThread().getName())));
-
- return true;
- }
- finally {
- U.closeQuiet(fs);
- }
- }
- }, nClients, "test-client");
- }
-
- /**
- * Checks consistency of create --> open --> append --> open operations with different buffer sizes.
- *
- * @param createBufSize Buffer size used for file creation.
- * @param writeCntsInCreate Count of times to write in file creation.
- * @param openAfterCreateBufSize Buffer size used for file opening after creation.
- * @param appendBufSize Buffer size used for file appending.
- * @param writeCntsInAppend Count of times to write in file appending.
- * @param openAfterAppendBufSize Buffer size used for file opening after appending.
- * @throws Exception If failed.
- */
- private void checkConsistency(int createBufSize, int writeCntsInCreate, int openAfterCreateBufSize,
- int appendBufSize, int writeCntsInAppend, int openAfterAppendBufSize) throws Exception {
- final Path igfsHome = new Path(primaryFsUri);
-
- Path file = new Path(igfsHome, "/someDir/someInnerDir/someFile");
-
- if (createBufSize == -1)
- createBufSize = fs.getServerDefaults().getFileBufferSize();
-
- if (appendBufSize == -1)
- appendBufSize = fs.getServerDefaults().getFileBufferSize();
-
- FSDataOutputStream os = fs.create(file, EnumSet.of(CreateFlag.OVERWRITE),
- Options.CreateOpts.perms(FsPermission.getDefault()), Options.CreateOpts.bufferSize(createBufSize));
-
- for (int i = 0; i < writeCntsInCreate; i++)
- os.writeInt(i);
-
- os.close();
-
- FSDataInputStream is = fs.open(file, openAfterCreateBufSize);
-
- for (int i = 0; i < writeCntsInCreate; i++)
- assertEquals(i, is.readInt());
-
- is.close();
-
- os = fs.create(file, EnumSet.of(CreateFlag.APPEND),
- Options.CreateOpts.perms(FsPermission.getDefault()), Options.CreateOpts.bufferSize(appendBufSize));
-
- for (int i = writeCntsInCreate; i < writeCntsInCreate + writeCntsInAppend; i++)
- os.writeInt(i);
-
- os.close();
-
- is = fs.open(file, openAfterAppendBufSize);
-
- for (int i = 0; i < writeCntsInCreate + writeCntsInAppend; i++)
- assertEquals(i, is.readInt());
-
- is.close();
- }
-
- /**
- * Test expected failures for 'close' operation.
- *
- * @param fs File system to test.
- * @param msg Expected exception message.
- */
- public void assertCloseFails(final FileSystem fs, String msg) {
- GridTestUtils.assertThrows(log, new Callable() {
- @Override public Object call() throws Exception {
- fs.close();
-
- return null;
- }
- }, IOException.class, msg);
- }
-
- /**
- * Test expected failures for 'get content summary' operation.
- *
- * @param fs File system to test.
- * @param path Path to evaluate content summary for.
- */
- private void assertContentSummaryFails(final FileSystem fs, final Path path) {
- GridTestUtils.assertThrows(log, new Callable<ContentSummary>() {
- @Override public ContentSummary call() throws Exception {
- return fs.getContentSummary(path);
- }
- }, FileNotFoundException.class, null);
- }
-
- /**
- * Assert that a given path exists in a given FileSystem.
- *
- * @param fs FileSystem to check.
- * @param p Path to check.
- * @throws IOException if the path does not exist.
- */
- private void assertPathExists(AbstractFileSystem fs, Path p) throws IOException {
- FileStatus fileStatus = fs.getFileStatus(p);
-
- assertEquals(p, fileStatus.getPath());
- assertNotSame(0, fileStatus.getModificationTime());
- }
-
- /**
- * Check path does not exist in a given FileSystem.
- *
- * @param fs FileSystem to check.
- * @param path Path to check.
- */
- private void assertPathDoesNotExist(final AbstractFileSystem fs, final Path path) {
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return fs.getFileStatus(path);
- }
- }, FileNotFoundException.class, null);
- }
-
- /** Helper class to encapsulate source and destination folders. */
- @SuppressWarnings({"PublicInnerClass", "PublicField"})
- public static final class Config {
- /** Source file system. */
- public final AbstractFileSystem srcFs;
-
- /** Source path to work with. */
- public final Path src;
-
- /** Destination file system. */
- public final AbstractFileSystem destFs;
-
- /** Destination path to work with. */
- public final Path dest;
-
- /**
- * Copying task configuration.
- *
- * @param srcFs Source file system.
- * @param src Source path.
- * @param destFs Destination file system.
- * @param dest Destination path.
- */
- public Config(AbstractFileSystem srcFs, Path src, AbstractFileSystem destFs, Path dest) {
- this.srcFs = srcFs;
- this.src = src;
- this.destFs = destFs;
- this.dest = dest;
- }
- }
-
- /**
- * Convert path for exception message testing purposes.
- *
- * @param path Path.
- * @return Converted path.
- * @throws Exception If failed.
- */
- private Path convertPath(Path path) throws Exception {
- if (mode != PROXY)
- return path;
- else {
- URI secondaryUri = new URI(secondaryFileSystemUriPath());
-
- URI pathUri = path.toUri();
-
- return new Path(new URI(pathUri.getScheme() != null ? secondaryUri.getScheme() : null,
- pathUri.getAuthority() != null ? secondaryUri.getAuthority() : null, pathUri.getPath(), null, null));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoop20FileSystemLoopbackPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoop20FileSystemLoopbackPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoop20FileSystemLoopbackPrimarySelfTest.java
deleted file mode 100644
index 47e1c86..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoop20FileSystemLoopbackPrimarySelfTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import java.util.*;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
-
-/**
- * Tests Hadoop 2.x file system in primary mode.
- */
-public class IgfsHadoop20FileSystemLoopbackPrimarySelfTest extends IgfsHadoop20FileSystemAbstractSelfTest {
- /**
- * Creates test in primary mode.
- */
- public IgfsHadoop20FileSystemLoopbackPrimarySelfTest() {
- super(PRIMARY);
- }
-
- /** {@inheritDoc} */
- @Override protected String primaryFileSystemUriPath() {
- return "igfs://igfs:" + getTestGridName(0) + "@/";
- }
-
- /** {@inheritDoc} */
- @Override protected String primaryFileSystemConfigPath() {
- return "/modules/core/src/test/config/hadoop/core-site-loopback.xml";
- }
-
- /** {@inheritDoc} */
- @Override protected Map<String, String> primaryIpcEndpointConfiguration(final String gridName) {
- return new HashMap<String, String>() {{
- put("type", "tcp");
- put("port", String.valueOf(DFLT_IPC_PORT + getTestGridIndex(gridName)));
- }};
- }
-
- /** {@inheritDoc} */
- @Override protected String secondaryFileSystemUriPath() {
- assert false;
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override protected String secondaryFileSystemConfigPath() {
- assert false;
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override protected Map<String, String> secondaryIpcEndpointConfiguration() {
- assert false;
-
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoop20FileSystemShmemPrimarySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoop20FileSystemShmemPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoop20FileSystemShmemPrimarySelfTest.java
deleted file mode 100644
index 631f188..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoop20FileSystemShmemPrimarySelfTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import java.util.*;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*;
-
-/**
- * Tests Hadoop 2.x file system in primary mode.
- */
-public class IgfsHadoop20FileSystemShmemPrimarySelfTest extends IgfsHadoop20FileSystemAbstractSelfTest {
- /**
- * Creates test in primary mode.
- */
- public IgfsHadoop20FileSystemShmemPrimarySelfTest() {
- super(PRIMARY);
- }
-
- /** {@inheritDoc} */
- @Override protected String primaryFileSystemUriPath() {
- return "igfs://igfs:" + getTestGridName(0) + "@/";
- }
-
- /** {@inheritDoc} */
- @Override protected String primaryFileSystemConfigPath() {
- return "/modules/core/src/test/config/hadoop/core-site.xml";
- }
-
- /** {@inheritDoc} */
- @Override protected Map<String, String> primaryIpcEndpointConfiguration(final String gridName) {
- return new HashMap<String, String>() {{
- put("type", "shmem");
- put("port", String.valueOf(DFLT_IPC_PORT + getTestGridIndex(gridName)));
- }};
- }
-
- /** {@inheritDoc} */
- @Override protected String secondaryFileSystemUriPath() {
- assert false;
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override protected String secondaryFileSystemConfigPath() {
- assert false;
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override protected Map<String, String> secondaryIpcEndpointConfiguration() {
- assert false;
-
- return null;
- }
-}
[27/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
new file mode 100644
index 0000000..39f42b2
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
@@ -0,0 +1,1626 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.jobtracker;
+
+import org.apache.ignite.*;
+import org.apache.ignite.events.*;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.managers.eventstorage.*;
+import org.apache.ignite.internal.processors.cache.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.future.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
+import org.jdk8.backport.*;
+import org.jetbrains.annotations.*;
+
+import javax.cache.event.*;
+import javax.cache.expiry.*;
+import javax.cache.processor.*;
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+
+import static java.util.concurrent.TimeUnit.*;
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobPhase.*;
+import static org.apache.ignite.internal.processors.hadoop.HadoopTaskType.*;
+import static org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopTaskState.*;
+
+/**
+ * Hadoop job tracker.
+ */
+public class HadoopJobTracker extends HadoopComponent {
+ /** */
+ private final GridMutex mux = new GridMutex();
+
+ /** */
+ private volatile GridCacheProjectionEx<HadoopJobId, HadoopJobMetadata> jobMetaPrj;
+
+ /** Projection with expiry policy for finished job updates. */
+ private volatile GridCacheProjectionEx<HadoopJobId, HadoopJobMetadata> finishedJobMetaPrj;
+
+ /** Map-reduce execution planner. */
+ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
+ private HadoopMapReducePlanner mrPlanner;
+
+ /** All the known jobs. */
+ private final ConcurrentMap<HadoopJobId, GridFutureAdapterEx<HadoopJob>> jobs = new ConcurrentHashMap8<>();
+
+ /** Locally active jobs. */
+ private final ConcurrentMap<HadoopJobId, JobLocalState> activeJobs = new ConcurrentHashMap8<>();
+
+ /** Locally requested finish futures. */
+ private final ConcurrentMap<HadoopJobId, GridFutureAdapter<HadoopJobId>> activeFinishFuts =
+ new ConcurrentHashMap8<>();
+
+ /** Event processing service. */
+ private ExecutorService evtProcSvc;
+
+ /** Component busy lock. */
+ private GridSpinReadWriteLock busyLock;
+
+ /** Closure to check result of async transform of system cache. */
+ private final IgniteInClosure<IgniteInternalFuture<?>> failsLog = new CI1<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> gridFut) {
+ try {
+ gridFut.get();
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to transform system cache.", e);
+ }
+ }
+ };
+
+ /** {@inheritDoc} */
+ @Override public void start(HadoopContext ctx) throws IgniteCheckedException {
+ super.start(ctx);
+
+ busyLock = new GridSpinReadWriteLock();
+
+ evtProcSvc = Executors.newFixedThreadPool(1);
+ }
+
+ /**
+ * @return Job meta projection.
+ */
+ @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext")
+ private GridCacheProjectionEx<HadoopJobId, HadoopJobMetadata> jobMetaCache() {
+ GridCacheProjectionEx<HadoopJobId, HadoopJobMetadata> prj = jobMetaPrj;
+
+ if (prj == null) {
+ synchronized (mux) {
+ if ((prj = jobMetaPrj) == null) {
+ CacheProjection<Object, Object> sysCache = ctx.kernalContext().cache()
+ .cache(CU.SYS_CACHE_HADOOP_MR);
+
+ assert sysCache != null;
+
+ mrPlanner = ctx.planner();
+
+ try {
+ ctx.kernalContext().resource().injectGeneric(mrPlanner);
+ }
+ catch (IgniteCheckedException e) { // Must not happen.
+ U.error(log, "Failed to inject resources.", e);
+
+ throw new IllegalStateException(e);
+ }
+
+ jobMetaPrj = prj = (GridCacheProjectionEx<HadoopJobId, HadoopJobMetadata>)
+ sysCache.projection(HadoopJobId.class, HadoopJobMetadata.class);
+
+ if (ctx.configuration().getFinishedJobInfoTtl() > 0) {
+ ExpiryPolicy finishedJobPlc = new ModifiedExpiryPolicy(
+ new Duration(MILLISECONDS, ctx.configuration().getFinishedJobInfoTtl()));
+
+ finishedJobMetaPrj = prj.withExpiryPolicy(finishedJobPlc);
+ }
+ else
+ finishedJobMetaPrj = jobMetaPrj;
+ }
+ }
+ }
+
+ return prj;
+ }
+
+ /**
+ * @return Projection with expiry policy for finished job updates.
+ */
+ private GridCacheProjectionEx<HadoopJobId, HadoopJobMetadata> finishedJobMetaCache() {
+ GridCacheProjectionEx<HadoopJobId, HadoopJobMetadata> prj = finishedJobMetaPrj;
+
+ if (prj == null) {
+ jobMetaCache();
+
+ prj = finishedJobMetaPrj;
+
+ assert prj != null;
+ }
+
+ return prj;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("deprecation")
+ @Override public void onKernalStart() throws IgniteCheckedException {
+ super.onKernalStart();
+
+ jobMetaCache().context().continuousQueries().executeInternalQuery(
+ new CacheEntryUpdatedListener<HadoopJobId, HadoopJobMetadata>() {
+ @Override public void onUpdated(final Iterable<CacheEntryEvent<? extends HadoopJobId,
+ ? extends HadoopJobMetadata>> evts) {
+ if (!busyLock.tryReadLock())
+ return;
+
+ try {
+ // Must process query callback in a separate thread to avoid deadlocks.
+ evtProcSvc.submit(new EventHandler() {
+ @Override protected void body() throws IgniteCheckedException {
+ processJobMetadataUpdates(evts);
+ }
+ });
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+ },
+ null,
+ true,
+ true
+ );
+
+ ctx.kernalContext().event().addLocalEventListener(new GridLocalEventListener() {
+ @Override public void onEvent(final Event evt) {
+ if (!busyLock.tryReadLock())
+ return;
+
+ try {
+ // Must process discovery callback in a separate thread to avoid deadlock.
+ evtProcSvc.submit(new EventHandler() {
+ @Override protected void body() {
+ processNodeLeft((DiscoveryEvent)evt);
+ }
+ });
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+ }, EventType.EVT_NODE_FAILED, EventType.EVT_NODE_LEFT);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onKernalStop(boolean cancel) {
+ super.onKernalStop(cancel);
+
+ busyLock.writeLock();
+
+ evtProcSvc.shutdown();
+
+ // Fail all pending futures.
+ for (GridFutureAdapter<HadoopJobId> fut : activeFinishFuts.values())
+ fut.onDone(new IgniteCheckedException("Failed to execute Hadoop map-reduce job (grid is stopping)."));
+ }
+
+ /**
+ * Submits execution of Hadoop job to grid.
+ *
+ * @param jobId Job ID.
+ * @param info Job info.
+ * @return Job completion future.
+ */
+ @SuppressWarnings("unchecked")
+ public IgniteInternalFuture<HadoopJobId> submit(HadoopJobId jobId, HadoopJobInfo info) {
+ if (!busyLock.tryReadLock()) {
+ return new GridFinishedFutureEx<>(new IgniteCheckedException("Failed to execute map-reduce job " +
+ "(grid is stopping): " + info));
+ }
+
+ try {
+ long jobPrepare = U.currentTimeMillis();
+
+ if (jobs.containsKey(jobId) || jobMetaCache().containsKey(jobId))
+ throw new IgniteCheckedException("Failed to submit job. Job with the same ID already exists: " + jobId);
+
+ HadoopJob job = job(jobId, info);
+
+ HadoopMapReducePlan mrPlan = mrPlanner.preparePlan(job, ctx.nodes(), null);
+
+ HadoopJobMetadata meta = new HadoopJobMetadata(ctx.localNodeId(), jobId, info);
+
+ meta.mapReducePlan(mrPlan);
+
+ meta.pendingSplits(allSplits(mrPlan));
+ meta.pendingReducers(allReducers(mrPlan));
+
+ GridFutureAdapter<HadoopJobId> completeFut = new GridFutureAdapter<>();
+
+ GridFutureAdapter<HadoopJobId> old = activeFinishFuts.put(jobId, completeFut);
+
+ assert old == null : "Duplicate completion future [jobId=" + jobId + ", old=" + old + ']';
+
+ if (log.isDebugEnabled())
+ log.debug("Submitting job metadata [jobId=" + jobId + ", meta=" + meta + ']');
+
+ long jobStart = U.currentTimeMillis();
+
+ HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(meta.counters(),
+ ctx.localNodeId());
+
+ perfCntr.clientSubmissionEvents(info);
+ perfCntr.onJobPrepare(jobPrepare);
+ perfCntr.onJobStart(jobStart);
+
+ if (jobMetaCache().putIfAbsent(jobId, meta) != null)
+ throw new IgniteCheckedException("Failed to submit job. Job with the same ID already exists: " + jobId);
+
+ return completeFut;
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to submit job: " + jobId, e);
+
+ return new GridFinishedFutureEx<>(e);
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+
+ /**
+ * Convert Hadoop job metadata to job status.
+ *
+ * @param meta Metadata.
+ * @return Status.
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public static HadoopJobStatus status(HadoopJobMetadata meta) {
+ HadoopJobInfo jobInfo = meta.jobInfo();
+
+ return new HadoopJobStatus(
+ meta.jobId(),
+ jobInfo.jobName(),
+ jobInfo.user(),
+ meta.pendingSplits() != null ? meta.pendingSplits().size() : 0,
+ meta.pendingReducers() != null ? meta.pendingReducers().size() : 0,
+ meta.mapReducePlan().mappers(),
+ meta.mapReducePlan().reducers(),
+ meta.phase(),
+ meta.failCause() != null,
+ meta.version()
+ );
+ }
+
+ /**
+ * Gets hadoop job status for given job ID.
+ *
+ * @param jobId Job ID to get status for.
+ * @return Job status for given job ID or {@code null} if job was not found.
+ */
+ @Nullable public HadoopJobStatus status(HadoopJobId jobId) throws IgniteCheckedException {
+ if (!busyLock.tryReadLock())
+ return null; // Grid is stopping.
+
+ try {
+ HadoopJobMetadata meta = jobMetaCache().get(jobId);
+
+ return meta != null ? status(meta) : null;
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+
+ /**
+ * Gets job finish future.
+ *
+ * @param jobId Job ID.
+ * @return Finish future or {@code null}.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable public IgniteInternalFuture<?> finishFuture(HadoopJobId jobId) throws IgniteCheckedException {
+ if (!busyLock.tryReadLock())
+ return null; // Grid is stopping.
+
+ try {
+ HadoopJobMetadata meta = jobMetaCache().get(jobId);
+
+ if (meta == null)
+ return null;
+
+ if (log.isTraceEnabled())
+ log.trace("Got job metadata for status check [locNodeId=" + ctx.localNodeId() + ", meta=" + meta + ']');
+
+ if (meta.phase() == PHASE_COMPLETE) {
+ if (log.isTraceEnabled())
+ log.trace("Job is complete, returning finished future: " + jobId);
+
+ return new GridFinishedFutureEx<>(jobId, meta.failCause());
+ }
+
+ GridFutureAdapter<HadoopJobId> fut = F.addIfAbsent(activeFinishFuts, jobId,
+ new GridFutureAdapter<HadoopJobId>());
+
+ // Get meta from cache one more time to close the window.
+ meta = jobMetaCache().get(jobId);
+
+ if (log.isTraceEnabled())
+ log.trace("Re-checking job metadata [locNodeId=" + ctx.localNodeId() + ", meta=" + meta + ']');
+
+ if (meta == null) {
+ fut.onDone();
+
+ activeFinishFuts.remove(jobId , fut);
+ }
+ else if (meta.phase() == PHASE_COMPLETE) {
+ fut.onDone(jobId, meta.failCause());
+
+ activeFinishFuts.remove(jobId , fut);
+ }
+
+ return fut;
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+
+ /**
+ * Gets job plan by job ID.
+ *
+ * @param jobId Job ID.
+ * @return Job plan.
+ * @throws IgniteCheckedException If failed.
+ */
+ public HadoopMapReducePlan plan(HadoopJobId jobId) throws IgniteCheckedException {
+ if (!busyLock.tryReadLock())
+ return null;
+
+ try {
+ HadoopJobMetadata meta = jobMetaCache().get(jobId);
+
+ if (meta != null)
+ return meta.mapReducePlan();
+
+ return null;
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+
+ /**
+ * Callback from task executor invoked when a task has been finished.
+ *
+ * @param info Task info.
+ * @param status Task status.
+ */
+ @SuppressWarnings({"ConstantConditions", "ThrowableResultOfMethodCallIgnored"})
+ public void onTaskFinished(HadoopTaskInfo info, HadoopTaskStatus status) {
+ if (!busyLock.tryReadLock())
+ return;
+
+ try {
+ assert status.state() != RUNNING;
+
+ if (log.isDebugEnabled())
+ log.debug("Received task finished callback [info=" + info + ", status=" + status + ']');
+
+ JobLocalState state = activeJobs.get(info.jobId());
+
+ // Task CRASHes with null fail cause.
+ assert (status.state() != FAILED) || status.failCause() != null :
+ "Invalid task status [info=" + info + ", status=" + status + ']';
+
+ assert state != null || (ctx.jobUpdateLeader() && (info.type() == COMMIT || info.type() == ABORT)):
+ "Missing local state for finished task [info=" + info + ", status=" + status + ']';
+
+ StackedProcessor incrCntrs = null;
+
+ if (status.state() == COMPLETED)
+ incrCntrs = new IncrementCountersProcessor(null, status.counters());
+
+ switch (info.type()) {
+ case SETUP: {
+ state.onSetupFinished(info, status, incrCntrs);
+
+ break;
+ }
+
+ case MAP: {
+ state.onMapFinished(info, status, incrCntrs);
+
+ break;
+ }
+
+ case REDUCE: {
+ state.onReduceFinished(info, status, incrCntrs);
+
+ break;
+ }
+
+ case COMBINE: {
+ state.onCombineFinished(info, status, incrCntrs);
+
+ break;
+ }
+
+ case COMMIT:
+ case ABORT: {
+ GridCacheProjectionEx<HadoopJobId, HadoopJobMetadata> cache = finishedJobMetaCache();
+
+ cache.invokeAsync(info.jobId(), new UpdatePhaseProcessor(incrCntrs, PHASE_COMPLETE)).
+ listenAsync(failsLog);
+
+ break;
+ }
+ }
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+
+ /**
+ * @param jobId Job id.
+ * @param c Closure of operation.
+ */
+ private void transform(HadoopJobId jobId, EntryProcessor<HadoopJobId, HadoopJobMetadata, Void> c) {
+ jobMetaCache().invokeAsync(jobId, c).listenAsync(failsLog);
+ }
+
+ /**
+ * Callback from task executor called when process is ready to received shuffle messages.
+ *
+ * @param jobId Job ID.
+ * @param reducers Reducers.
+ * @param desc Process descriptor.
+ */
+ public void onExternalMappersInitialized(HadoopJobId jobId, Collection<Integer> reducers,
+ HadoopProcessDescriptor desc) {
+ transform(jobId, new InitializeReducersProcessor(null, reducers, desc));
+ }
+
+ /**
+ * Gets all input splits for given hadoop map-reduce plan.
+ *
+ * @param plan Map-reduce plan.
+ * @return Collection of all input splits that should be processed.
+ */
+ @SuppressWarnings("ConstantConditions")
+ private Map<HadoopInputSplit, Integer> allSplits(HadoopMapReducePlan plan) {
+ Map<HadoopInputSplit, Integer> res = new HashMap<>();
+
+ int taskNum = 0;
+
+ for (UUID nodeId : plan.mapperNodeIds()) {
+ for (HadoopInputSplit split : plan.mappers(nodeId)) {
+ if (res.put(split, taskNum++) != null)
+ throw new IllegalStateException("Split duplicate.");
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Gets all reducers for this job.
+ *
+ * @param plan Map-reduce plan.
+ * @return Collection of reducers.
+ */
+ private Collection<Integer> allReducers(HadoopMapReducePlan plan) {
+ Collection<Integer> res = new HashSet<>();
+
+ for (int i = 0; i < plan.reducers(); i++)
+ res.add(i);
+
+ return res;
+ }
+
+ /**
+ * Processes node leave (or fail) event.
+ *
+ * @param evt Discovery event.
+ */
+ @SuppressWarnings("ConstantConditions")
+ private void processNodeLeft(DiscoveryEvent evt) {
+ if (log.isDebugEnabled())
+ log.debug("Processing discovery event [locNodeId=" + ctx.localNodeId() + ", evt=" + evt + ']');
+
+ // Check only if this node is responsible for job status updates.
+ if (ctx.jobUpdateLeader()) {
+ boolean checkSetup = evt.eventNode().order() < ctx.localNodeOrder();
+
+ // Iteration over all local entries is correct since system cache is REPLICATED.
+ for (Object metaObj : jobMetaCache().values()) {
+ HadoopJobMetadata meta = (HadoopJobMetadata)metaObj;
+
+ HadoopJobId jobId = meta.jobId();
+
+ HadoopMapReducePlan plan = meta.mapReducePlan();
+
+ HadoopJobPhase phase = meta.phase();
+
+ try {
+ if (checkSetup && phase == PHASE_SETUP && !activeJobs.containsKey(jobId)) {
+ // Failover setup task.
+ HadoopJob job = job(jobId, meta.jobInfo());
+
+ Collection<HadoopTaskInfo> setupTask = setupTask(jobId);
+
+ assert setupTask != null;
+
+ ctx.taskExecutor().run(job, setupTask);
+ }
+ else if (phase == PHASE_MAP || phase == PHASE_REDUCE) {
+ // Must check all nodes, even that are not event node ID due to
+ // multiple node failure possibility.
+ Collection<HadoopInputSplit> cancelSplits = null;
+
+ for (UUID nodeId : plan.mapperNodeIds()) {
+ if (ctx.kernalContext().discovery().node(nodeId) == null) {
+ // Node has left the grid.
+ Collection<HadoopInputSplit> mappers = plan.mappers(nodeId);
+
+ if (cancelSplits == null)
+ cancelSplits = new HashSet<>();
+
+ cancelSplits.addAll(mappers);
+ }
+ }
+
+ Collection<Integer> cancelReducers = null;
+
+ for (UUID nodeId : plan.reducerNodeIds()) {
+ if (ctx.kernalContext().discovery().node(nodeId) == null) {
+ // Node has left the grid.
+ int[] reducers = plan.reducers(nodeId);
+
+ if (cancelReducers == null)
+ cancelReducers = new HashSet<>();
+
+ for (int rdc : reducers)
+ cancelReducers.add(rdc);
+ }
+ }
+
+ if (cancelSplits != null || cancelReducers != null)
+ jobMetaCache().invoke(meta.jobId(), new CancelJobProcessor(null, new IgniteCheckedException(
+ "One or more nodes participating in map-reduce job execution failed."), cancelSplits,
+ cancelReducers));
+ }
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to cancel job: " + meta, e);
+ }
+ }
+ }
+ }
+
+ /**
+ * @param updated Updated cache entries.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void processJobMetadataUpdates(
+ Iterable<CacheEntryEvent<? extends HadoopJobId, ? extends HadoopJobMetadata>> updated)
+ throws IgniteCheckedException {
+ UUID locNodeId = ctx.localNodeId();
+
+ for (CacheEntryEvent<? extends HadoopJobId, ? extends HadoopJobMetadata> entry : updated) {
+ HadoopJobId jobId = entry.getKey();
+ HadoopJobMetadata meta = entry.getValue();
+
+ if (meta == null || !ctx.isParticipating(meta))
+ continue;
+
+ if (log.isDebugEnabled())
+ log.debug("Processing job metadata update callback [locNodeId=" + locNodeId +
+ ", meta=" + meta + ']');
+
+ try {
+ ctx.taskExecutor().onJobStateChanged(meta);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to process job state changed callback (will fail the job) " +
+ "[locNodeId=" + locNodeId + ", jobId=" + jobId + ", meta=" + meta + ']', e);
+
+ transform(jobId, new CancelJobProcessor(null, e));
+
+ continue;
+ }
+
+ processJobMetaUpdate(jobId, meta, locNodeId);
+ }
+ }
+
+ /**
+ * @param jobId Job ID.
+ * @param plan Map-reduce plan.
+ */
+ private void printPlan(HadoopJobId jobId, HadoopMapReducePlan plan) {
+ log.info("Plan for " + jobId);
+
+ SB b = new SB();
+
+ b.a(" Map: ");
+
+ for (UUID nodeId : plan.mapperNodeIds())
+ b.a(nodeId).a("=").a(plan.mappers(nodeId).size()).a(' ');
+
+ log.info(b.toString());
+
+ b = new SB();
+
+ b.a(" Reduce: ");
+
+ for (UUID nodeId : plan.reducerNodeIds())
+ b.a(nodeId).a("=").a(Arrays.toString(plan.reducers(nodeId))).a(' ');
+
+ log.info(b.toString());
+ }
+
+ /**
+ * @param jobId Job ID.
+ * @param meta Job metadata.
+ * @param locNodeId Local node ID.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void processJobMetaUpdate(HadoopJobId jobId, HadoopJobMetadata meta, UUID locNodeId)
+ throws IgniteCheckedException {
+ JobLocalState state = activeJobs.get(jobId);
+
+ HadoopJob job = job(jobId, meta.jobInfo());
+
+ HadoopMapReducePlan plan = meta.mapReducePlan();
+
+ switch (meta.phase()) {
+ case PHASE_SETUP: {
+ if (ctx.jobUpdateLeader()) {
+ Collection<HadoopTaskInfo> setupTask = setupTask(jobId);
+
+ if (setupTask != null)
+ ctx.taskExecutor().run(job, setupTask);
+ }
+
+ break;
+ }
+
+ case PHASE_MAP: {
+ // Check if we should initiate new task on local node.
+ Collection<HadoopTaskInfo> tasks = mapperTasks(plan.mappers(locNodeId), meta);
+
+ if (tasks != null)
+ ctx.taskExecutor().run(job, tasks);
+
+ break;
+ }
+
+ case PHASE_REDUCE: {
+ if (meta.pendingReducers().isEmpty() && ctx.jobUpdateLeader()) {
+ HadoopTaskInfo info = new HadoopTaskInfo(COMMIT, jobId, 0, 0, null);
+
+ if (log.isDebugEnabled())
+ log.debug("Submitting COMMIT task for execution [locNodeId=" + locNodeId +
+ ", jobId=" + jobId + ']');
+
+ ctx.taskExecutor().run(job, Collections.singletonList(info));
+
+ break;
+ }
+
+ Collection<HadoopTaskInfo> tasks = reducerTasks(plan.reducers(locNodeId), job);
+
+ if (tasks != null)
+ ctx.taskExecutor().run(job, tasks);
+
+ break;
+ }
+
+ case PHASE_CANCELLING: {
+ // Prevent multiple task executor notification.
+ if (state != null && state.onCancel()) {
+ if (log.isDebugEnabled())
+ log.debug("Cancelling local task execution for job: " + meta);
+
+ ctx.taskExecutor().cancelTasks(jobId);
+ }
+
+ if (meta.pendingSplits().isEmpty() && meta.pendingReducers().isEmpty()) {
+ if (ctx.jobUpdateLeader()) {
+ if (state == null)
+ state = initState(jobId);
+
+ // Prevent running multiple abort tasks.
+ if (state.onAborted()) {
+ HadoopTaskInfo info = new HadoopTaskInfo(ABORT, jobId, 0, 0, null);
+
+ if (log.isDebugEnabled())
+ log.debug("Submitting ABORT task for execution [locNodeId=" + locNodeId +
+ ", jobId=" + jobId + ']');
+
+ ctx.taskExecutor().run(job, Collections.singletonList(info));
+ }
+ }
+
+ break;
+ }
+ else {
+ // Check if there are unscheduled mappers or reducers.
+ Collection<HadoopInputSplit> cancelMappers = new ArrayList<>();
+ Collection<Integer> cancelReducers = new ArrayList<>();
+
+ Collection<HadoopInputSplit> mappers = plan.mappers(ctx.localNodeId());
+
+ if (mappers != null) {
+ for (HadoopInputSplit b : mappers) {
+ if (state == null || !state.mapperScheduled(b))
+ cancelMappers.add(b);
+ }
+ }
+
+ int[] rdc = plan.reducers(ctx.localNodeId());
+
+ if (rdc != null) {
+ for (int r : rdc) {
+ if (state == null || !state.reducerScheduled(r))
+ cancelReducers.add(r);
+ }
+ }
+
+ if (!cancelMappers.isEmpty() || !cancelReducers.isEmpty())
+ transform(jobId, new CancelJobProcessor(null, cancelMappers, cancelReducers));
+ }
+
+ break;
+ }
+
+ case PHASE_COMPLETE: {
+ if (log.isDebugEnabled())
+ log.debug("Job execution is complete, will remove local state from active jobs " +
+ "[jobId=" + jobId + ", meta=" + meta + ']');
+
+ if (state != null) {
+ state = activeJobs.remove(jobId);
+
+ assert state != null;
+
+ ctx.shuffle().jobFinished(jobId);
+ }
+
+ GridFutureAdapter<HadoopJobId> finishFut = activeFinishFuts.remove(jobId);
+
+ if (finishFut != null) {
+ if (log.isDebugEnabled())
+ log.debug("Completing job future [locNodeId=" + locNodeId + ", meta=" + meta + ']');
+
+ finishFut.onDone(jobId, meta.failCause());
+ }
+
+ if (ctx.jobUpdateLeader())
+ job.cleanupStagingDirectory();
+
+ jobs.remove(jobId);
+
+ job.dispose(false);
+
+ if (ctx.jobUpdateLeader()) {
+ ClassLoader ldr = job.getClass().getClassLoader();
+
+ try {
+ String statWriterClsName = job.info().property(HadoopUtils.JOB_COUNTER_WRITER_PROPERTY);
+
+ if (statWriterClsName != null) {
+ Class<?> cls = ldr.loadClass(statWriterClsName);
+
+ HadoopCounterWriter writer = (HadoopCounterWriter)cls.newInstance();
+
+ HadoopCounters cntrs = meta.counters();
+
+ writer.write(job.info(), jobId, cntrs);
+ }
+ }
+ catch (Exception e) {
+ log.error("Can't write statistic due to: ", e);
+ }
+ }
+
+ break;
+ }
+
+ default:
+ throw new IllegalStateException("Unknown phase: " + meta.phase());
+ }
+ }
+
+ /**
+ * Creates setup task based on job information.
+ *
+ * @param jobId Job ID.
+ * @return Setup task wrapped in collection.
+ */
+ @Nullable private Collection<HadoopTaskInfo> setupTask(HadoopJobId jobId) {
+ if (activeJobs.containsKey(jobId))
+ return null;
+ else {
+ initState(jobId);
+
+ return Collections.singleton(new HadoopTaskInfo(SETUP, jobId, 0, 0, null));
+ }
+ }
+
+ /**
+ * Creates mapper tasks based on job information.
+ *
+ * @param mappers Mapper blocks.
+ * @param meta Job metadata.
+ * @return Collection of created task infos or {@code null} if no mapper tasks scheduled for local node.
+ */
+ private Collection<HadoopTaskInfo> mapperTasks(Iterable<HadoopInputSplit> mappers, HadoopJobMetadata meta) {
+ UUID locNodeId = ctx.localNodeId();
+ HadoopJobId jobId = meta.jobId();
+
+ JobLocalState state = activeJobs.get(jobId);
+
+ Collection<HadoopTaskInfo> tasks = null;
+
+ if (mappers != null) {
+ if (state == null)
+ state = initState(jobId);
+
+ for (HadoopInputSplit split : mappers) {
+ if (state.addMapper(split)) {
+ if (log.isDebugEnabled())
+ log.debug("Submitting MAP task for execution [locNodeId=" + locNodeId +
+ ", split=" + split + ']');
+
+ HadoopTaskInfo taskInfo = new HadoopTaskInfo(MAP, jobId, meta.taskNumber(split), 0, split);
+
+ if (tasks == null)
+ tasks = new ArrayList<>();
+
+ tasks.add(taskInfo);
+ }
+ }
+ }
+
+ return tasks;
+ }
+
+ /**
+ * Creates reducer tasks based on job information.
+ *
+ * @param reducers Reducers (may be {@code null}).
+ * @param job Job instance.
+ * @return Collection of task infos.
+ */
+ private Collection<HadoopTaskInfo> reducerTasks(int[] reducers, HadoopJob job) {
+ UUID locNodeId = ctx.localNodeId();
+ HadoopJobId jobId = job.id();
+
+ JobLocalState state = activeJobs.get(jobId);
+
+ Collection<HadoopTaskInfo> tasks = null;
+
+ if (reducers != null) {
+ if (state == null)
+ state = initState(job.id());
+
+ for (int rdc : reducers) {
+ if (state.addReducer(rdc)) {
+ if (log.isDebugEnabled())
+ log.debug("Submitting REDUCE task for execution [locNodeId=" + locNodeId +
+ ", rdc=" + rdc + ']');
+
+ HadoopTaskInfo taskInfo = new HadoopTaskInfo(REDUCE, jobId, rdc, 0, null);
+
+ if (tasks == null)
+ tasks = new ArrayList<>();
+
+ tasks.add(taskInfo);
+ }
+ }
+ }
+
+ return tasks;
+ }
+
+ /**
+ * Initializes local state for given job metadata.
+ *
+ * @param jobId Job ID.
+ * @return Local state.
+ */
+ private JobLocalState initState(HadoopJobId jobId) {
+ return F.addIfAbsent(activeJobs, jobId, new JobLocalState());
+ }
+
+ /**
+ * Gets or creates job instance.
+ *
+ * @param jobId Job ID.
+ * @param jobInfo Job info.
+ * @return Job.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable public HadoopJob job(HadoopJobId jobId, @Nullable HadoopJobInfo jobInfo) throws IgniteCheckedException {
+ GridFutureAdapterEx<HadoopJob> fut = jobs.get(jobId);
+
+ if (fut != null || (fut = jobs.putIfAbsent(jobId, new GridFutureAdapterEx<HadoopJob>())) != null)
+ return fut.get();
+
+ fut = jobs.get(jobId);
+
+ HadoopJob job = null;
+
+ try {
+ if (jobInfo == null) {
+ HadoopJobMetadata meta = jobMetaCache().get(jobId);
+
+ if (meta == null)
+ throw new IgniteCheckedException("Failed to find job metadata for ID: " + jobId);
+
+ jobInfo = meta.jobInfo();
+ }
+
+ job = jobInfo.createJob(jobId, log);
+
+ job.initialize(false, ctx.localNodeId());
+
+ fut.onDone(job);
+
+ return job;
+ }
+ catch (IgniteCheckedException e) {
+ fut.onDone(e);
+
+ jobs.remove(jobId, fut);
+
+ if (job != null) {
+ try {
+ job.dispose(false);
+ }
+ catch (IgniteCheckedException e0) {
+ U.error(log, "Failed to dispose job: " + jobId, e0);
+ }
+ }
+
+ throw e;
+ }
+ }
+
+ /**
+ * Kills job.
+ *
+ * @param jobId Job ID.
+ * @return {@code True} if job was killed.
+ * @throws IgniteCheckedException If failed.
+ */
+ public boolean killJob(HadoopJobId jobId) throws IgniteCheckedException {
+ if (!busyLock.tryReadLock())
+ return false; // Grid is stopping.
+
+ try {
+ HadoopJobMetadata meta = jobMetaCache().get(jobId);
+
+ if (meta != null && meta.phase() != PHASE_COMPLETE && meta.phase() != PHASE_CANCELLING) {
+ HadoopTaskCancelledException err = new HadoopTaskCancelledException("Job cancelled.");
+
+ jobMetaCache().invoke(jobId, new CancelJobProcessor(null, err));
+ }
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+
+ IgniteInternalFuture<?> fut = finishFuture(jobId);
+
+ if (fut != null) {
+ try {
+ fut.get();
+ }
+ catch (Throwable e) {
+ if (e.getCause() instanceof HadoopTaskCancelledException)
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Returns job counters.
+ *
+ * @param jobId Job identifier.
+ * @return Job counters or {@code null} if job cannot be found.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable public HadoopCounters jobCounters(HadoopJobId jobId) throws IgniteCheckedException {
+ if (!busyLock.tryReadLock())
+ return null;
+
+ try {
+ final HadoopJobMetadata meta = jobMetaCache().get(jobId);
+
+ return meta != null ? meta.counters() : null;
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+
+ /**
+ * Event handler protected by busy lock.
+ */
+ private abstract class EventHandler implements Runnable {
+ /** {@inheritDoc} */
+ @Override public void run() {
+ if (!busyLock.tryReadLock())
+ return;
+
+ try {
+ body();
+ }
+ catch (Throwable e) {
+ U.error(log, "Unhandled exception while processing event.", e);
+ }
+ finally {
+ busyLock.readUnlock();
+ }
+ }
+
+ /**
+ * Handler body.
+ */
+ protected abstract void body() throws Exception;
+ }
+
+ /**
+ *
+ */
+ private class JobLocalState {
+ /** Mappers. */
+ private final Collection<HadoopInputSplit> currMappers = new HashSet<>();
+
+ /** Reducers. */
+ private final Collection<Integer> currReducers = new HashSet<>();
+
+ /** Number of completed mappers. */
+ private final AtomicInteger completedMappersCnt = new AtomicInteger();
+
+ /** Cancelled flag. */
+ private boolean cancelled;
+
+ /** Aborted flag. */
+ private boolean aborted;
+
+ /**
+ * @param mapSplit Map split to add.
+ * @return {@code True} if mapper was added.
+ */
+ private boolean addMapper(HadoopInputSplit mapSplit) {
+ return currMappers.add(mapSplit);
+ }
+
+ /**
+ * @param rdc Reducer number to add.
+ * @return {@code True} if reducer was added.
+ */
+ private boolean addReducer(int rdc) {
+ return currReducers.add(rdc);
+ }
+
+ /**
+ * Checks whether this split was scheduled for given attempt.
+ *
+ * @param mapSplit Map split to check.
+ * @return {@code True} if mapper was scheduled.
+ */
+ public boolean mapperScheduled(HadoopInputSplit mapSplit) {
+ return currMappers.contains(mapSplit);
+ }
+
+ /**
+ * Checks whether this split was scheduled for given attempt.
+ *
+ * @param rdc Reducer number to check.
+ * @return {@code True} if reducer was scheduled.
+ */
+ public boolean reducerScheduled(int rdc) {
+ return currReducers.contains(rdc);
+ }
+
+ /**
+ * @param taskInfo Task info.
+ * @param status Task status.
+ * @param prev Previous closure.
+ */
+ private void onSetupFinished(final HadoopTaskInfo taskInfo, HadoopTaskStatus status, StackedProcessor prev) {
+ final HadoopJobId jobId = taskInfo.jobId();
+
+ if (status.state() == FAILED || status.state() == CRASHED)
+ transform(jobId, new CancelJobProcessor(prev, status.failCause()));
+ else
+ transform(jobId, new UpdatePhaseProcessor(prev, PHASE_MAP));
+ }
+
+ /**
+ * @param taskInfo Task info.
+ * @param status Task status.
+ * @param prev Previous closure.
+ */
+ private void onMapFinished(final HadoopTaskInfo taskInfo, HadoopTaskStatus status,
+ final StackedProcessor prev) {
+ final HadoopJobId jobId = taskInfo.jobId();
+
+ boolean lastMapperFinished = completedMappersCnt.incrementAndGet() == currMappers.size();
+
+ if (status.state() == FAILED || status.state() == CRASHED) {
+ // Fail the whole job.
+ transform(jobId, new RemoveMappersProcessor(prev, taskInfo.inputSplit(), status.failCause()));
+
+ return;
+ }
+
+ IgniteInClosure<IgniteInternalFuture<?>> cacheUpdater = new CIX1<IgniteInternalFuture<?>>() {
+ @Override public void applyx(IgniteInternalFuture<?> f) {
+ Throwable err = null;
+
+ if (f != null) {
+ try {
+ f.get();
+ }
+ catch (IgniteCheckedException e) {
+ err = e;
+ }
+ }
+
+ transform(jobId, new RemoveMappersProcessor(prev, taskInfo.inputSplit(), err));
+ }
+ };
+
+ if (lastMapperFinished)
+ ctx.shuffle().flush(jobId).listenAsync(cacheUpdater);
+ else
+ cacheUpdater.apply(null);
+ }
+
+ /**
+ * @param taskInfo Task info.
+ * @param status Task status.
+ * @param prev Previous closure.
+ */
+ private void onReduceFinished(HadoopTaskInfo taskInfo, HadoopTaskStatus status, StackedProcessor prev) {
+ HadoopJobId jobId = taskInfo.jobId();
+ if (status.state() == FAILED || status.state() == CRASHED)
+ // Fail the whole job.
+ transform(jobId, new RemoveReducerProcessor(prev, taskInfo.taskNumber(), status.failCause()));
+ else
+ transform(jobId, new RemoveReducerProcessor(prev, taskInfo.taskNumber()));
+ }
+
+ /**
+ * @param taskInfo Task info.
+ * @param status Task status.
+ * @param prev Previous closure.
+ */
+ private void onCombineFinished(HadoopTaskInfo taskInfo, HadoopTaskStatus status,
+ final StackedProcessor prev) {
+ final HadoopJobId jobId = taskInfo.jobId();
+
+ if (status.state() == FAILED || status.state() == CRASHED)
+ // Fail the whole job.
+ transform(jobId, new RemoveMappersProcessor(prev, currMappers, status.failCause()));
+ else {
+ ctx.shuffle().flush(jobId).listenAsync(new CIX1<IgniteInternalFuture<?>>() {
+ @Override public void applyx(IgniteInternalFuture<?> f) {
+ Throwable err = null;
+
+ if (f != null) {
+ try {
+ f.get();
+ }
+ catch (IgniteCheckedException e) {
+ err = e;
+ }
+ }
+
+ transform(jobId, new RemoveMappersProcessor(prev, currMappers, err));
+ }
+ });
+ }
+ }
+
+ /**
+ * @return {@code True} if job was cancelled by this (first) call.
+ */
+ public boolean onCancel() {
+ if (!cancelled && !aborted) {
+ cancelled = true;
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @return {@code True} if job was aborted this (first) call.
+ */
+ public boolean onAborted() {
+ if (!aborted) {
+ aborted = true;
+
+ return true;
+ }
+
+ return false;
+ }
+ }
+
+ /**
+ * Update job phase transform closure.
+ */
+ private static class UpdatePhaseProcessor extends StackedProcessor {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Phase to update. */
+ private final HadoopJobPhase phase;
+
+ /**
+ * @param prev Previous closure.
+ * @param phase Phase to update.
+ */
+ private UpdatePhaseProcessor(@Nullable StackedProcessor prev, HadoopJobPhase phase) {
+ super(prev);
+
+ this.phase = phase;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void update(HadoopJobMetadata meta, HadoopJobMetadata cp) {
+ cp.phase(phase);
+ }
+ }
+
+ /**
+ * Remove mapper transform closure.
+ */
+ private static class RemoveMappersProcessor extends StackedProcessor {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Mapper split to remove. */
+ private final Collection<HadoopInputSplit> splits;
+
+ /** Error. */
+ private final Throwable err;
+
+ /**
+ * @param prev Previous closure.
+ * @param split Mapper split to remove.
+ * @param err Error.
+ */
+ private RemoveMappersProcessor(@Nullable StackedProcessor prev, HadoopInputSplit split, Throwable err) {
+ this(prev, Collections.singletonList(split), err);
+ }
+
+ /**
+ * @param prev Previous closure.
+ * @param splits Mapper splits to remove.
+ * @param err Error.
+ */
+ private RemoveMappersProcessor(@Nullable StackedProcessor prev, Collection<HadoopInputSplit> splits,
+ Throwable err) {
+ super(prev);
+
+ this.splits = splits;
+ this.err = err;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void update(HadoopJobMetadata meta, HadoopJobMetadata cp) {
+ Map<HadoopInputSplit, Integer> splitsCp = new HashMap<>(cp.pendingSplits());
+
+ for (HadoopInputSplit s : splits)
+ splitsCp.remove(s);
+
+ cp.pendingSplits(splitsCp);
+
+ if (cp.phase() != PHASE_CANCELLING && err != null)
+ cp.failCause(err);
+
+ if (err != null)
+ cp.phase(PHASE_CANCELLING);
+
+ if (splitsCp.isEmpty()) {
+ if (cp.phase() != PHASE_CANCELLING)
+ cp.phase(PHASE_REDUCE);
+ }
+ }
+ }
+
+ /**
+ * Remove reducer transform closure.
+ */
+ private static class RemoveReducerProcessor extends StackedProcessor {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Mapper split to remove. */
+ private final int rdc;
+
+ /** Error. */
+ private Throwable err;
+
+ /**
+ * @param prev Previous closure.
+ * @param rdc Reducer to remove.
+ */
+ private RemoveReducerProcessor(@Nullable StackedProcessor prev, int rdc) {
+ super(prev);
+
+ this.rdc = rdc;
+ }
+
+ /**
+ * @param prev Previous closure.
+ * @param rdc Reducer to remove.
+ * @param err Error.
+ */
+ private RemoveReducerProcessor(@Nullable StackedProcessor prev, int rdc, Throwable err) {
+ super(prev);
+
+ this.rdc = rdc;
+ this.err = err;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void update(HadoopJobMetadata meta, HadoopJobMetadata cp) {
+ Collection<Integer> rdcCp = new HashSet<>(cp.pendingReducers());
+
+ rdcCp.remove(rdc);
+
+ cp.pendingReducers(rdcCp);
+
+ if (err != null) {
+ cp.phase(PHASE_CANCELLING);
+ cp.failCause(err);
+ }
+ }
+ }
+
+ /**
+ * Initialize reducers.
+ */
+ private static class InitializeReducersProcessor extends StackedProcessor {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Reducers. */
+ private final Collection<Integer> rdc;
+
+ /** Process descriptor for reducers. */
+ private final HadoopProcessDescriptor desc;
+
+ /**
+ * @param prev Previous closure.
+ * @param rdc Reducers to initialize.
+ * @param desc External process descriptor.
+ */
+ private InitializeReducersProcessor(@Nullable StackedProcessor prev,
+ Collection<Integer> rdc,
+ HadoopProcessDescriptor desc) {
+ super(prev);
+
+ assert !F.isEmpty(rdc);
+ assert desc != null;
+
+ this.rdc = rdc;
+ this.desc = desc;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void update(HadoopJobMetadata meta, HadoopJobMetadata cp) {
+ Map<Integer, HadoopProcessDescriptor> oldMap = meta.reducersAddresses();
+
+ Map<Integer, HadoopProcessDescriptor> rdcMap = oldMap == null ?
+ new HashMap<Integer, HadoopProcessDescriptor>() : new HashMap<>(oldMap);
+
+ for (Integer r : rdc)
+ rdcMap.put(r, desc);
+
+ cp.reducersAddresses(rdcMap);
+ }
+ }
+
+ /**
+ * Remove reducer transform closure.
+ */
+ private static class CancelJobProcessor extends StackedProcessor {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Mapper split to remove. */
+ private final Collection<HadoopInputSplit> splits;
+
+ /** Reducers to remove. */
+ private final Collection<Integer> rdc;
+
+ /** Error. */
+ private final Throwable err;
+
+ /**
+ * @param prev Previous closure.
+ * @param err Fail cause.
+ */
+ private CancelJobProcessor(@Nullable StackedProcessor prev, Throwable err) {
+ this(prev, err, null, null);
+ }
+
+ /**
+ * @param prev Previous closure.
+ * @param splits Splits to remove.
+ * @param rdc Reducers to remove.
+ */
+ private CancelJobProcessor(@Nullable StackedProcessor prev,
+ Collection<HadoopInputSplit> splits,
+ Collection<Integer> rdc) {
+ this(prev, null, splits, rdc);
+ }
+
+ /**
+ * @param prev Previous closure.
+ * @param err Error.
+ * @param splits Splits to remove.
+ * @param rdc Reducers to remove.
+ */
+ private CancelJobProcessor(@Nullable StackedProcessor prev,
+ Throwable err,
+ Collection<HadoopInputSplit> splits,
+ Collection<Integer> rdc) {
+ super(prev);
+
+ this.splits = splits;
+ this.rdc = rdc;
+ this.err = err;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void update(HadoopJobMetadata meta, HadoopJobMetadata cp) {
+ assert meta.phase() == PHASE_CANCELLING || err != null: "Invalid phase for cancel: " + meta;
+
+ Collection<Integer> rdcCp = new HashSet<>(cp.pendingReducers());
+
+ if (rdc != null)
+ rdcCp.removeAll(rdc);
+
+ cp.pendingReducers(rdcCp);
+
+ Map<HadoopInputSplit, Integer> splitsCp = new HashMap<>(cp.pendingSplits());
+
+ if (splits != null) {
+ for (HadoopInputSplit s : splits)
+ splitsCp.remove(s);
+ }
+
+ cp.pendingSplits(splitsCp);
+
+ cp.phase(PHASE_CANCELLING);
+
+ if (err != null)
+ cp.failCause(err);
+ }
+ }
+
+ /**
+ * Increment counter values closure.
+ */
+ private static class IncrementCountersProcessor extends StackedProcessor {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final HadoopCounters counters;
+
+ /**
+ * @param prev Previous closure.
+ * @param counters Task counters to add into job counters.
+ */
+ private IncrementCountersProcessor(@Nullable StackedProcessor prev, HadoopCounters counters) {
+ super(prev);
+
+ assert counters != null;
+
+ this.counters = counters;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void update(HadoopJobMetadata meta, HadoopJobMetadata cp) {
+ HadoopCounters cntrs = new HadoopCountersImpl(cp.counters());
+
+ cntrs.merge(counters);
+
+ cp.counters(cntrs);
+ }
+ }
+
+ /**
+ * Abstract stacked closure.
+ */
+ private abstract static class StackedProcessor implements
+ EntryProcessor<HadoopJobId, HadoopJobMetadata, Void>, Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final StackedProcessor prev;
+
+ /**
+ * @param prev Previous closure.
+ */
+ private StackedProcessor(@Nullable StackedProcessor prev) {
+ this.prev = prev;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Void process(MutableEntry<HadoopJobId, HadoopJobMetadata> e, Object... args) {
+ HadoopJobMetadata val = apply(e.getValue());
+
+ if (val != null)
+ e.setValue(val);
+ else
+ e.remove();;
+
+ return null;
+ }
+
+ /**
+ * @param meta Old value.
+ * @return New value.
+ */
+ private HadoopJobMetadata apply(HadoopJobMetadata meta) {
+ if (meta == null)
+ return null;
+
+ HadoopJobMetadata cp = prev != null ? prev.apply(meta) : new HadoopJobMetadata(meta);
+
+ update(meta, cp);
+
+ return cp;
+ }
+
+ /**
+ * Update given job metadata object.
+ *
+ * @param meta Initial job metadata.
+ * @param cp Copy.
+ */
+ protected abstract void update(HadoopJobMetadata meta, HadoopJobMetadata cp);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/GridHadoopMessage.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/GridHadoopMessage.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/GridHadoopMessage.java
deleted file mode 100644
index 1670a8a..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/GridHadoopMessage.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.message;
-
-import java.io.*;
-
-/**
- * Marker interface for all hadoop messages.
- */
-public interface GridHadoopMessage extends Externalizable {
- // No-op.
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java
new file mode 100644
index 0000000..cab6138
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.message;
+
+import java.io.*;
+
+/**
+ * Marker interface for all hadoop messages.
+ */
+public interface HadoopMessage extends Externalizable {
+ // No-op.
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlan.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlan.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlan.java
deleted file mode 100644
index 7988403..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlan.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.planner;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.jetbrains.annotations.*;
-
-import java.util.*;
-
-/**
- * Map-reduce plan.
- */
-public class GridHadoopDefaultMapReducePlan implements GridHadoopMapReducePlan {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Mappers map. */
- private Map<UUID, Collection<GridHadoopInputSplit>> mappers;
-
- /** Reducers map. */
- private Map<UUID, int[]> reducers;
-
- /** Mappers count. */
- private int mappersCnt;
-
- /** Reducers count. */
- private int reducersCnt;
-
- /**
- * @param mappers Mappers map.
- * @param reducers Reducers map.
- */
- public GridHadoopDefaultMapReducePlan(Map<UUID, Collection<GridHadoopInputSplit>> mappers,
- Map<UUID, int[]> reducers) {
- this.mappers = mappers;
- this.reducers = reducers;
-
- if (mappers != null) {
- for (Collection<GridHadoopInputSplit> splits : mappers.values())
- mappersCnt += splits.size();
- }
-
- if (reducers != null) {
- for (int[] rdcrs : reducers.values())
- reducersCnt += rdcrs.length;
- }
- }
-
- /** {@inheritDoc} */
- @Override public int mappers() {
- return mappersCnt;
- }
-
- /** {@inheritDoc} */
- @Override public int reducers() {
- return reducersCnt;
- }
-
- /** {@inheritDoc} */
- @Override public UUID nodeForReducer(int reducer) {
- assert reducer >= 0 && reducer < reducersCnt : reducer;
-
- for (Map.Entry<UUID, int[]> entry : reducers.entrySet()) {
- for (int r : entry.getValue()) {
- if (r == reducer)
- return entry.getKey();
- }
- }
-
- throw new IllegalStateException("Not found reducer index: " + reducer);
- }
-
- /** {@inheritDoc} */
- @Override @Nullable public Collection<GridHadoopInputSplit> mappers(UUID nodeId) {
- return mappers.get(nodeId);
- }
-
- /** {@inheritDoc} */
- @Override @Nullable public int[] reducers(UUID nodeId) {
- return reducers.get(nodeId);
- }
-
- /** {@inheritDoc} */
- @Override public Collection<UUID> mapperNodeIds() {
- return mappers.keySet();
- }
-
- /** {@inheritDoc} */
- @Override public Collection<UUID> reducerNodeIds() {
- return reducers.keySet();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlanner.java
deleted file mode 100644
index 342cbab..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlanner.java
+++ /dev/null
@@ -1,434 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.planner;
-
-import org.apache.ignite.*;
-import org.apache.ignite.cluster.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.igfs.hadoop.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.resources.*;
-import org.jetbrains.annotations.*;
-
-import java.util.*;
-
-import static org.apache.ignite.IgniteFs.*;
-
-/**
- * Default map-reduce planner implementation.
- */
-public class GridHadoopDefaultMapReducePlanner implements GridHadoopMapReducePlanner {
- /** Injected grid. */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** Logger. */
- @SuppressWarnings("UnusedDeclaration")
- @LoggerResource
- private IgniteLogger log;
-
- /** {@inheritDoc} */
- @Override public GridHadoopMapReducePlan preparePlan(GridHadoopJob job, Collection<ClusterNode> top,
- @Nullable GridHadoopMapReducePlan oldPlan) throws IgniteCheckedException {
- // Convert collection of topology nodes to collection of topology node IDs.
- Collection<UUID> topIds = new HashSet<>(top.size(), 1.0f);
-
- for (ClusterNode topNode : top)
- topIds.add(topNode.id());
-
- Map<UUID, Collection<GridHadoopInputSplit>> mappers = mappers(top, topIds, job.input());
-
- int rdcCnt = job.info().reducers();
-
- if (rdcCnt < 0)
- throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + rdcCnt);
-
- Map<UUID, int[]> reducers = reducers(top, mappers, rdcCnt);
-
- return new GridHadoopDefaultMapReducePlan(mappers, reducers);
- }
-
- /**
- * Create plan for mappers.
- *
- * @param top Topology nodes.
- * @param topIds Topology node IDs.
- * @param splits Splits.
- * @return Mappers map.
- * @throws IgniteCheckedException If failed.
- */
- private Map<UUID, Collection<GridHadoopInputSplit>> mappers(Collection<ClusterNode> top, Collection<UUID> topIds,
- Iterable<GridHadoopInputSplit> splits) throws IgniteCheckedException {
- Map<UUID, Collection<GridHadoopInputSplit>> mappers = new HashMap<>();
-
- Map<String, Collection<UUID>> nodes = hosts(top);
-
- Map<UUID, Integer> nodeLoads = new HashMap<>(top.size(), 1.0f); // Track node load.
-
- for (UUID nodeId : topIds)
- nodeLoads.put(nodeId, 0);
-
- for (GridHadoopInputSplit split : splits) {
- UUID nodeId = nodeForSplit(split, topIds, nodes, nodeLoads);
-
- if (log.isDebugEnabled())
- log.debug("Mapped split to node [split=" + split + ", nodeId=" + nodeId + ']');
-
- Collection<GridHadoopInputSplit> nodeSplits = mappers.get(nodeId);
-
- if (nodeSplits == null) {
- nodeSplits = new ArrayList<>();
-
- mappers.put(nodeId, nodeSplits);
- }
-
- nodeSplits.add(split);
-
- // Updated node load.
- nodeLoads.put(nodeId, nodeLoads.get(nodeId) + 1);
- }
-
- return mappers;
- }
-
- /**
- * Groups nodes by host names.
- *
- * @param top Topology to group.
- * @return Map.
- */
- private static Map<String, Collection<UUID>> hosts(Collection<ClusterNode> top) {
- Map<String, Collection<UUID>> grouped = U.newHashMap(top.size());
-
- for (ClusterNode node : top) {
- for (String host : node.hostNames()) {
- Collection<UUID> nodeIds = grouped.get(host);
-
- if (nodeIds == null) {
- // Expecting 1-2 nodes per host.
- nodeIds = new ArrayList<>(2);
-
- grouped.put(host, nodeIds);
- }
-
- nodeIds.add(node.id());
- }
- }
-
- return grouped;
- }
-
- /**
- * Determine the best node for this split.
- *
- * @param split Split.
- * @param topIds Topology node IDs.
- * @param nodes Nodes.
- * @param nodeLoads Node load tracker.
- * @return Node ID.
- */
- @SuppressWarnings("unchecked")
- private UUID nodeForSplit(GridHadoopInputSplit split, Collection<UUID> topIds, Map<String, Collection<UUID>> nodes,
- Map<UUID, Integer> nodeLoads) throws IgniteCheckedException {
- if (split instanceof GridHadoopFileBlock) {
- GridHadoopFileBlock split0 = (GridHadoopFileBlock)split;
-
- if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
- IgfsHadoopEndpoint endpoint = new IgfsHadoopEndpoint(split0.file().getAuthority());
-
- IgfsEx igfs = null;
-
- if (F.eq(ignite.name(), endpoint.grid()))
- igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs());
-
- if (igfs != null && !igfs.isProxy(split0.file())) {
- Collection<IgfsBlockLocation> blocks;
-
- try {
- blocks = igfs.affinity(new IgfsPath(split0.file()), split0.start(), split0.length());
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
-
- assert blocks != null;
-
- if (blocks.size() == 1)
- // Fast-path, split consists of one IGFS block (as in most cases).
- return bestNode(blocks.iterator().next().nodeIds(), topIds, nodeLoads, false);
- else {
- // Slow-path, file consists of multiple IGFS blocks. First, find the most co-located nodes.
- Map<UUID, Long> nodeMap = new HashMap<>();
-
- List<UUID> bestNodeIds = null;
- long bestLen = -1L;
-
- for (IgfsBlockLocation block : blocks) {
- for (UUID blockNodeId : block.nodeIds()) {
- if (topIds.contains(blockNodeId)) {
- Long oldLen = nodeMap.get(blockNodeId);
- long newLen = oldLen == null ? block.length() : oldLen + block.length();
-
- nodeMap.put(blockNodeId, newLen);
-
- if (bestNodeIds == null || bestLen < newLen) {
- bestNodeIds = new ArrayList<>(1);
-
- bestNodeIds.add(blockNodeId);
-
- bestLen = newLen;
- }
- else if (bestLen == newLen) {
- assert !F.isEmpty(bestNodeIds);
-
- bestNodeIds.add(blockNodeId);
- }
- }
- }
- }
-
- if (bestNodeIds != null) {
- return bestNodeIds.size() == 1 ? bestNodeIds.get(0) :
- bestNode(bestNodeIds, topIds, nodeLoads, true);
- }
- }
- }
- }
- }
-
- // Cannot use local IGFS for some reason, try selecting the node by host.
- Collection<UUID> blockNodes = null;
-
- for (String host : split.hosts()) {
- Collection<UUID> hostNodes = nodes.get(host);
-
- if (!F.isEmpty(hostNodes)) {
- if (blockNodes == null)
- blockNodes = new ArrayList<>(hostNodes);
- else
- blockNodes.addAll(hostNodes);
- }
- }
-
- return bestNode(blockNodes, topIds, nodeLoads, false);
- }
-
- /**
- * Finds the best (the least loaded) node among the candidates.
- *
- * @param candidates Candidates.
- * @param topIds Topology node IDs.
- * @param nodeLoads Known node loads.
- * @param skipTopCheck Whether to skip topology check.
- * @return The best node.
- */
- private UUID bestNode(@Nullable Collection<UUID> candidates, Collection<UUID> topIds, Map<UUID, Integer> nodeLoads,
- boolean skipTopCheck) {
- UUID bestNode = null;
- int bestLoad = Integer.MAX_VALUE;
-
- if (candidates != null) {
- for (UUID candidate : candidates) {
- if (skipTopCheck || topIds.contains(candidate)) {
- int load = nodeLoads.get(candidate);
-
- if (bestNode == null || bestLoad > load) {
- bestNode = candidate;
- bestLoad = load;
-
- if (bestLoad == 0)
- break; // Minimum load possible, no need for further iterations.
- }
- }
- }
- }
-
- if (bestNode == null) {
- // Blocks are located on nodes which are not Hadoop-enabled, assign to the least loaded one.
- bestLoad = Integer.MAX_VALUE;
-
- for (UUID nodeId : topIds) {
- int load = nodeLoads.get(nodeId);
-
- if (bestNode == null || bestLoad > load) {
- bestNode = nodeId;
- bestLoad = load;
-
- if (bestLoad == 0)
- break; // Minimum load possible, no need for further iterations.
- }
- }
- }
-
- assert bestNode != null;
-
- return bestNode;
- }
-
- /**
- * Create plan for reducers.
- *
- * @param top Topology.
- * @param mappers Mappers map.
- * @param reducerCnt Reducers count.
- * @return Reducers map.
- */
- private Map<UUID, int[]> reducers(Collection<ClusterNode> top,
- Map<UUID, Collection<GridHadoopInputSplit>> mappers, int reducerCnt) {
- // Determine initial node weights.
- int totalWeight = 0;
-
- List<WeightedNode> nodes = new ArrayList<>(top.size());
-
- for (ClusterNode node : top) {
- Collection<GridHadoopInputSplit> split = mappers.get(node.id());
-
- int weight = reducerNodeWeight(node, split != null ? split.size() : 0);
-
- nodes.add(new WeightedNode(node.id(), weight, weight));
-
- totalWeight += weight;
- }
-
- // Adjust weights.
- int totalAdjustedWeight = 0;
-
- for (WeightedNode node : nodes) {
- node.floatWeight = ((float)node.weight * reducerCnt) / totalWeight;
-
- node.weight = Math.round(node.floatWeight);
-
- totalAdjustedWeight += node.weight;
- }
-
- // Apply redundant/lost reducers.
- Collections.sort(nodes);
-
- if (totalAdjustedWeight > reducerCnt) {
- // Too much reducers set.
- ListIterator<WeightedNode> iter = nodes.listIterator(nodes.size() - 1);
-
- while (totalAdjustedWeight != reducerCnt) {
- if (!iter.hasPrevious())
- iter = nodes.listIterator(nodes.size() - 1);
-
- WeightedNode node = iter.previous();
-
- if (node.weight > 0) {
- node.weight -= 1;
-
- totalAdjustedWeight--;
- }
- }
- }
- else if (totalAdjustedWeight < reducerCnt) {
- // Not enough reducers set.
- ListIterator<WeightedNode> iter = nodes.listIterator(0);
-
- while (totalAdjustedWeight != reducerCnt) {
- if (!iter.hasNext())
- iter = nodes.listIterator(0);
-
- WeightedNode node = iter.next();
-
- if (node.floatWeight > 0.0f) {
- node.weight += 1;
-
- totalAdjustedWeight++;
- }
- }
- }
-
- int idx = 0;
-
- Map<UUID, int[]> reducers = new HashMap<>(nodes.size(), 1.0f);
-
- for (WeightedNode node : nodes) {
- if (node.weight > 0) {
- int[] arr = new int[node.weight];
-
- for (int i = 0; i < arr.length; i++)
- arr[i] = idx++;
-
- reducers.put(node.nodeId, arr);
- }
- }
-
- return reducers;
- }
-
- /**
- * Calculate node weight based on node metrics and data co-location.
- *
- * @param node Node.
- * @param splitCnt Splits mapped to this node.
- * @return Node weight.
- */
- @SuppressWarnings("UnusedParameters")
- protected int reducerNodeWeight(ClusterNode node, int splitCnt) {
- return splitCnt;
- }
-
- /**
- * Weighted node.
- */
- private static class WeightedNode implements Comparable<WeightedNode> {
- /** Node ID. */
- private final UUID nodeId;
-
- /** Weight. */
- private int weight;
-
- /** Floating point weight. */
- private float floatWeight;
-
- /**
- * Constructor.
- *
- * @param nodeId Node ID.
- * @param weight Weight.
- * @param floatWeight Floating point weight.
- */
- private WeightedNode(UUID nodeId, int weight, float floatWeight) {
- this.nodeId = nodeId;
- this.weight = weight;
- this.floatWeight = floatWeight;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object obj) {
- return obj != null && obj instanceof WeightedNode && F.eq(nodeId, ((WeightedNode)obj).nodeId);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return nodeId.hashCode();
- }
-
- /** {@inheritDoc} */
- @Override public int compareTo(@NotNull WeightedNode other) {
- float res = other.floatWeight - floatWeight;
-
- return res > 0.0f ? 1 : res < 0.0f ? -1 : nodeId.compareTo(other.nodeId);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopDefaultMapReducePlan.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopDefaultMapReducePlan.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopDefaultMapReducePlan.java
new file mode 100644
index 0000000..1413612
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopDefaultMapReducePlan.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.planner;
+
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.jetbrains.annotations.*;
+
+import java.util.*;
+
+/**
+ * Map-reduce plan.
+ */
+public class HadoopDefaultMapReducePlan implements HadoopMapReducePlan {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Mappers map. */
+ private Map<UUID, Collection<HadoopInputSplit>> mappers;
+
+ /** Reducers map. */
+ private Map<UUID, int[]> reducers;
+
+ /** Mappers count. */
+ private int mappersCnt;
+
+ /** Reducers count. */
+ private int reducersCnt;
+
+ /**
+ * @param mappers Mappers map.
+ * @param reducers Reducers map.
+ */
+ public HadoopDefaultMapReducePlan(Map<UUID, Collection<HadoopInputSplit>> mappers,
+ Map<UUID, int[]> reducers) {
+ this.mappers = mappers;
+ this.reducers = reducers;
+
+ if (mappers != null) {
+ for (Collection<HadoopInputSplit> splits : mappers.values())
+ mappersCnt += splits.size();
+ }
+
+ if (reducers != null) {
+ for (int[] rdcrs : reducers.values())
+ reducersCnt += rdcrs.length;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public int mappers() {
+ return mappersCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int reducers() {
+ return reducersCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public UUID nodeForReducer(int reducer) {
+ assert reducer >= 0 && reducer < reducersCnt : reducer;
+
+ for (Map.Entry<UUID, int[]> entry : reducers.entrySet()) {
+ for (int r : entry.getValue()) {
+ if (r == reducer)
+ return entry.getKey();
+ }
+ }
+
+ throw new IllegalStateException("Not found reducer index: " + reducer);
+ }
+
+ /** {@inheritDoc} */
+ @Override @Nullable public Collection<HadoopInputSplit> mappers(UUID nodeId) {
+ return mappers.get(nodeId);
+ }
+
+ /** {@inheritDoc} */
+ @Override @Nullable public int[] reducers(UUID nodeId) {
+ return reducers.get(nodeId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<UUID> mapperNodeIds() {
+ return mappers.keySet();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<UUID> reducerNodeIds() {
+ return reducers.keySet();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobCountersTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobCountersTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobCountersTask.java
deleted file mode 100644
index 37073d9..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobCountersTask.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.proto;
-
-import org.apache.ignite.*;
-import org.apache.ignite.compute.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-
-import java.util.*;
-
-/**
- * Task to get job counters.
- */
-public class GridHadoopProtocolJobCountersTask extends GridHadoopProtocolTaskAdapter<GridHadoopCounters> {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** {@inheritDoc} */
- @Override public GridHadoopCounters run(ComputeJobContext jobCtx, GridHadoop hadoop,
- GridHadoopProtocolTaskArguments args) throws IgniteCheckedException {
-
- UUID nodeId = UUID.fromString(args.<String>get(0));
- Integer id = args.get(1);
-
- assert nodeId != null;
- assert id != null;
-
- return hadoop.counters(new GridHadoopJobId(nodeId, id));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobStatusTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobStatusTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobStatusTask.java
deleted file mode 100644
index de4f89c..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobStatusTask.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.proto;
-
-import org.apache.ignite.*;
-import org.apache.ignite.compute.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.lang.*;
-
-import java.util.*;
-
-/**
- * Job status task.
- */
-public class GridHadoopProtocolJobStatusTask extends GridHadoopProtocolTaskAdapter<GridHadoopJobStatus> {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Default poll delay */
- private static final long DFLT_POLL_DELAY = 100L;
-
- /** Attribute for held status. */
- private static final String ATTR_HELD = "held";
-
- /** {@inheritDoc} */
- @Override public GridHadoopJobStatus run(final ComputeJobContext jobCtx, GridHadoop hadoop,
- GridHadoopProtocolTaskArguments args) throws IgniteCheckedException {
- UUID nodeId = UUID.fromString(args.<String>get(0));
- Integer id = args.get(1);
- Long pollDelay = args.get(2);
-
- assert nodeId != null;
- assert id != null;
-
- GridHadoopJobId jobId = new GridHadoopJobId(nodeId, id);
-
- if (pollDelay == null)
- pollDelay = DFLT_POLL_DELAY;
-
- if (pollDelay > 0) {
- IgniteInternalFuture<?> fut = hadoop.finishFuture(jobId);
-
- if (fut != null) {
- if (fut.isDone() || F.eq(jobCtx.getAttribute(ATTR_HELD), true))
- return hadoop.status(jobId);
- else {
- fut.listenAsync(new IgniteInClosure<IgniteInternalFuture<?>>() {
- @Override public void apply(IgniteInternalFuture<?> fut0) {
- jobCtx.callcc();
- }
- });
-
- jobCtx.setAttribute(ATTR_HELD, true);
-
- return jobCtx.holdcc(pollDelay);
- }
- }
- else
- return null;
- }
- else
- return hadoop.status(jobId);
- }
-}
[35/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopInputStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopInputStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopInputStream.java
deleted file mode 100644
index efc5264..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopInputStream.java
+++ /dev/null
@@ -1,626 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.commons.logging.*;
-import org.apache.hadoop.fs.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.igfs.common.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * IGFS input stream wrapper for hadoop interfaces.
- */
-@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
-public final class IgfsHadoopInputStream extends InputStream implements Seekable, PositionedReadable,
- IgfsHadoopStreamEventListener {
- /** Minimum buffer size. */
- private static final int MIN_BUF_SIZE = 4 * 1024;
-
- /** Server stream delegate. */
- private IgfsHadoopStreamDelegate delegate;
-
- /** Stream ID used by logger. */
- private long logStreamId;
-
- /** Stream position. */
- private long pos;
-
- /** Stream read limit. */
- private long limit;
-
- /** Mark position. */
- private long markPos = -1;
-
- /** Prefetch buffer. */
- private DoubleFetchBuffer buf = new DoubleFetchBuffer();
-
- /** Buffer half size for double-buffering. */
- private int bufHalfSize;
-
- /** Closed flag. */
- private volatile boolean closed;
-
- /** Flag set if stream was closed due to connection breakage. */
- private boolean connBroken;
-
- /** Logger. */
- private Log log;
-
- /** Client logger. */
- private IgfsLogger clientLog;
-
- /** Read time. */
- private long readTime;
-
- /** User time. */
- private long userTime;
-
- /** Last timestamp. */
- private long lastTs;
-
- /** Amount of read bytes. */
- private long total;
-
- /**
- * Creates input stream.
- *
- * @param delegate Server stream delegate.
- * @param limit Read limit.
- * @param bufSize Buffer size.
- * @param log Log.
- * @param clientLog Client logger.
- */
- public IgfsHadoopInputStream(IgfsHadoopStreamDelegate delegate, long limit, int bufSize, Log log,
- IgfsLogger clientLog, long logStreamId) {
- assert limit >= 0;
-
- this.delegate = delegate;
- this.limit = limit;
- this.log = log;
- this.clientLog = clientLog;
- this.logStreamId = logStreamId;
-
- bufHalfSize = Math.max(bufSize, MIN_BUF_SIZE);
-
- lastTs = System.nanoTime();
-
- delegate.hadoop().addEventListener(delegate, this);
- }
-
- /**
- * Read start.
- */
- private void readStart() {
- long now = System.nanoTime();
-
- userTime += now - lastTs;
-
- lastTs = now;
- }
-
- /**
- * Read end.
- */
- private void readEnd() {
- long now = System.nanoTime();
-
- readTime += now - lastTs;
-
- lastTs = now;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int read() throws IOException {
- checkClosed();
-
- readStart();
-
- try {
- if (eof())
- return -1;
-
- buf.refreshAhead(pos);
-
- int res = buf.atPosition(pos);
-
- pos++;
- total++;
-
- buf.refreshAhead(pos);
-
- return res;
- }
- catch (IgniteCheckedException e) {
- throw IgfsHadoopUtils.cast(e);
- }
- finally {
- readEnd();
- }
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int read(@NotNull byte[] b, int off, int len) throws IOException {
- checkClosed();
-
- if (eof())
- return -1;
-
- readStart();
-
- try {
- long remaining = limit - pos;
-
- int read = buf.flatten(b, pos, off, len);
-
- pos += read;
- total += read;
- remaining -= read;
-
- if (remaining > 0 && read != len) {
- int readAmt = (int)Math.min(remaining, len - read);
-
- delegate.hadoop().readData(delegate, pos, readAmt, b, off + read, len - read).get();
-
- read += readAmt;
- pos += readAmt;
- total += readAmt;
- }
-
- buf.refreshAhead(pos);
-
- return read;
- }
- catch (IgniteCheckedException e) {
- throw IgfsHadoopUtils.cast(e);
- }
- finally {
- readEnd();
- }
- }
-
- /** {@inheritDoc} */
- @Override public synchronized long skip(long n) throws IOException {
- checkClosed();
-
- if (clientLog.isLogEnabled())
- clientLog.logSkip(logStreamId, n);
-
- long oldPos = pos;
-
- if (pos + n <= limit)
- pos += n;
- else
- pos = limit;
-
- buf.refreshAhead(pos);
-
- return pos - oldPos;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int available() throws IOException {
- checkClosed();
-
- int available = buf.available(pos);
-
- assert available >= 0;
-
- return available;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void close() throws IOException {
- if (!closed) {
- readStart();
-
- if (log.isDebugEnabled())
- log.debug("Closing input stream: " + delegate);
-
- delegate.hadoop().closeStream(delegate);
-
- readEnd();
-
- if (clientLog.isLogEnabled())
- clientLog.logCloseIn(logStreamId, userTime, readTime, total);
-
- markClosed(false);
-
- if (log.isDebugEnabled())
- log.debug("Closed stream [delegate=" + delegate + ", readTime=" + readTime +
- ", userTime=" + userTime + ']');
- }
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void mark(int readLimit) {
- markPos = pos;
-
- if (clientLog.isLogEnabled())
- clientLog.logMark(logStreamId, readLimit);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void reset() throws IOException {
- checkClosed();
-
- if (clientLog.isLogEnabled())
- clientLog.logReset(logStreamId);
-
- if (markPos == -1)
- throw new IOException("Stream was not marked.");
-
- pos = markPos;
-
- buf.refreshAhead(pos);
- }
-
- /** {@inheritDoc} */
- @Override public boolean markSupported() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int read(long position, byte[] buf, int off, int len) throws IOException {
- long remaining = limit - position;
-
- int read = (int)Math.min(len, remaining);
-
- // Return -1 at EOF.
- if (read == 0)
- return -1;
-
- readFully(position, buf, off, read);
-
- return read;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void readFully(long position, byte[] buf, int off, int len) throws IOException {
- long remaining = limit - position;
-
- checkClosed();
-
- if (len > remaining)
- throw new EOFException("End of stream reached before data was fully read.");
-
- readStart();
-
- try {
- int read = this.buf.flatten(buf, position, off, len);
-
- total += read;
-
- if (read != len) {
- int readAmt = len - read;
-
- delegate.hadoop().readData(delegate, position + read, readAmt, buf, off + read, readAmt).get();
-
- total += readAmt;
- }
-
- if (clientLog.isLogEnabled())
- clientLog.logRandomRead(logStreamId, position, len);
- }
- catch (IgniteCheckedException e) {
- throw IgfsHadoopUtils.cast(e);
- }
- finally {
- readEnd();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void readFully(long position, byte[] buf) throws IOException {
- readFully(position, buf, 0, buf.length);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void seek(long pos) throws IOException {
- A.ensure(pos >= 0, "position must be non-negative");
-
- checkClosed();
-
- if (clientLog.isLogEnabled())
- clientLog.logSeek(logStreamId, pos);
-
- if (pos > limit)
- pos = limit;
-
- if (log.isDebugEnabled())
- log.debug("Seek to position [delegate=" + delegate + ", pos=" + pos + ", oldPos=" + this.pos + ']');
-
- this.pos = pos;
-
- buf.refreshAhead(pos);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized long getPos() {
- return pos;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized boolean seekToNewSource(long targetPos) {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public void onClose() {
- markClosed(true);
- }
-
- /** {@inheritDoc} */
- @Override public void onError(String errMsg) {
- // No-op.
- }
-
- /**
- * Marks stream as closed.
- *
- * @param connBroken {@code True} if connection with server was lost.
- */
- private void markClosed(boolean connBroken) {
- // It is ok to have race here.
- if (!closed) {
- closed = true;
-
- this.connBroken = connBroken;
-
- delegate.hadoop().removeEventListener(delegate);
- }
- }
-
- /**
- * @throws IOException If check failed.
- */
- private void checkClosed() throws IOException {
- if (closed) {
- if (connBroken)
- throw new IOException("Server connection was lost.");
- else
- throw new IOException("Stream is closed.");
- }
- }
-
- /**
- * @return {@code True} if end of stream reached.
- */
- private boolean eof() {
- return limit == pos;
- }
-
- /**
- * Asynchronous prefetch buffer.
- */
- private static class FetchBufferPart {
- /** Read future. */
- private GridPlainFuture<byte[]> readFut;
-
- /** Position of cached chunk in file. */
- private long pos;
-
- /** Prefetch length. Need to store as read future result might be not available yet. */
- private int len;
-
- /**
- * Creates fetch buffer part.
- *
- * @param readFut Read future for this buffer.
- * @param pos Read position.
- * @param len Chunk length.
- */
- private FetchBufferPart(GridPlainFuture<byte[]> readFut, long pos, int len) {
- this.readFut = readFut;
- this.pos = pos;
- this.len = len;
- }
-
- /**
- * Copies cached data if specified position matches cached region.
- *
- * @param dst Destination buffer.
- * @param pos Read position in file.
- * @param dstOff Offset in destination buffer from which start writing.
- * @param len Maximum number of bytes to copy.
- * @return Number of bytes copied.
- * @throws IgniteCheckedException If read future failed.
- */
- public int flatten(byte[] dst, long pos, int dstOff, int len) throws IgniteCheckedException {
- // If read start position is within cached boundaries.
- if (contains(pos)) {
- byte[] data = readFut.get();
-
- int srcPos = (int)(pos - this.pos);
- int cpLen = Math.min(len, data.length - srcPos);
-
- U.arrayCopy(data, srcPos, dst, dstOff, cpLen);
-
- return cpLen;
- }
-
- return 0;
- }
-
- /**
- * @return {@code True} if data is ready to be read.
- */
- public boolean ready() {
- return readFut.isDone();
- }
-
- /**
- * Checks if current buffer part contains given position.
- *
- * @param pos Position to check.
- * @return {@code True} if position matches buffer region.
- */
- public boolean contains(long pos) {
- return this.pos <= pos && this.pos + len > pos;
- }
- }
-
- private class DoubleFetchBuffer {
- /** */
- private FetchBufferPart first;
-
- /** */
- private FetchBufferPart second;
-
- /**
- * Copies fetched data from both buffers to destination array if cached region matched read position.
- *
- * @param dst Destination buffer.
- * @param pos Read position in file.
- * @param dstOff Destination buffer offset.
- * @param len Maximum number of bytes to copy.
- * @return Number of bytes copied.
- * @throws IgniteCheckedException If any read operation failed.
- */
- public int flatten(byte[] dst, long pos, int dstOff, int len) throws IgniteCheckedException {
- assert dstOff >= 0;
- assert dstOff + len <= dst.length : "Invalid indices [dst.length=" + dst.length + ", dstOff=" + dstOff +
- ", len=" + len + ']';
-
- int bytesCopied = 0;
-
- if (first != null) {
- bytesCopied += first.flatten(dst, pos, dstOff, len);
-
- if (bytesCopied != len && second != null) {
- assert second.pos == first.pos + first.len;
-
- bytesCopied += second.flatten(dst, pos + bytesCopied, dstOff + bytesCopied, len - bytesCopied);
- }
- }
-
- return bytesCopied;
- }
-
- /**
- * Gets byte at specified position in buffer.
- *
- * @param pos Stream position.
- * @return Read byte.
- * @throws IgniteCheckedException If read failed.
- */
- public int atPosition(long pos) throws IgniteCheckedException {
- // Should not reach here if stream contains no data.
- assert first != null;
-
- if (first.contains(pos)) {
- byte[] bytes = first.readFut.get();
-
- return bytes[((int)(pos - first.pos))] & 0xFF;
- }
- else {
- assert second != null;
- assert second.contains(pos);
-
- byte[] bytes = second.readFut.get();
-
- return bytes[((int)(pos - second.pos))] & 0xFF;
- }
- }
-
- /**
- * Starts asynchronous buffer refresh if needed, depending on current position.
- *
- * @param pos Current stream position.
- */
- public void refreshAhead(long pos) {
- if (fullPrefetch(pos)) {
- first = fetch(pos, bufHalfSize);
- second = fetch(pos + bufHalfSize, bufHalfSize);
- }
- else if (needFlip(pos)) {
- first = second;
-
- second = fetch(first.pos + first.len, bufHalfSize);
- }
- }
-
- /**
- * @param pos Position from which read is expected.
- * @return Number of bytes available to be read without blocking.
- */
- public int available(long pos) {
- int available = 0;
-
- if (first != null) {
- if (first.contains(pos)) {
- if (first.ready()) {
- available += (pos - first.pos);
-
- if (second != null && second.ready())
- available += second.len;
- }
- }
- else {
- if (second != null && second.contains(pos) && second.ready())
- available += (pos - second.pos);
- }
- }
-
- return available;
- }
-
- /**
- * Checks if position shifted enough to forget previous buffer.
- *
- * @param pos Current position.
- * @return {@code True} if need flip buffers.
- */
- private boolean needFlip(long pos) {
- // Return true if we read more then half of second buffer.
- return second != null && second.contains(pos);
- }
-
- /**
- * Determines if all cached bytes should be discarded and new region should be
- * prefetched.
- *
- * @param curPos Current stream position.
- * @return {@code True} if need to refresh both blocks.
- */
- private boolean fullPrefetch(long curPos) {
- // If no data was prefetched yet, return true.
- return first == null || curPos < first.pos || (second != null && curPos >= second.pos + second.len);
- }
-
- /**
- * Starts asynchronous fetch for given region.
- *
- * @param pos Position to read from.
- * @param size Number of bytes to read.
- * @return Fetch buffer part.
- */
- private FetchBufferPart fetch(long pos, int size) {
- long remaining = limit - pos;
-
- size = (int)Math.min(size, remaining);
-
- return size <= 0 ? null :
- new FetchBufferPart(delegate.hadoop().readData(delegate, pos, size, null, 0, 0), pos, size);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopIo.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopIo.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopIo.java
deleted file mode 100644
index 46f5a6c..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopIo.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.igfs.common.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.jetbrains.annotations.*;
-
-/**
- * IO abstraction layer for IGFS client. Two kind of messages are expected to be sent: requests with response
- * and request without response.
- */
-public interface IgfsHadoopIo {
- /**
- * Sends given IGFS client message and asynchronously awaits for response.
- *
- * @param msg Message to send.
- * @return Future that will be completed.
- * @throws IgniteCheckedException If a message cannot be sent (connection is broken or client was closed).
- */
- public GridPlainFuture<IgfsMessage> send(IgfsMessage msg) throws IgniteCheckedException;
-
- /**
- * Sends given IGFS client message and asynchronously awaits for response. When IO detects response
- * beginning for given message it stops reading data and passes input stream to closure which can read
- * response in a specific way.
- *
- * @param msg Message to send.
- * @param outBuf Output buffer. If {@code null}, the output buffer is not used.
- * @param outOff Output buffer offset.
- * @param outLen Output buffer length.
- * @return Future that will be completed when response is returned from closure.
- * @throws IgniteCheckedException If a message cannot be sent (connection is broken or client was closed).
- */
- public <T> GridPlainFuture<T> send(IgfsMessage msg, @Nullable byte[] outBuf, int outOff, int outLen)
- throws IgniteCheckedException;
-
- /**
- * Sends given message and does not wait for response.
- *
- * @param msg Message to send.
- * @throws IgniteCheckedException If send failed.
- */
- public void sendPlain(IgfsMessage msg) throws IgniteCheckedException;
-
- /**
- * Adds event listener that will be invoked when connection with server is lost or remote error has occurred.
- * If connection is closed already, callback will be invoked synchronously inside this method.
- *
- * @param lsnr Event listener.
- */
- public void addEventListener(IgfsHadoopIpcIoListener lsnr);
-
- /**
- * Removes event listener that will be invoked when connection with server is lost or remote error has occurred.
- *
- * @param lsnr Event listener.
- */
- public void removeEventListener(IgfsHadoopIpcIoListener lsnr);
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopIpcIo.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopIpcIo.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopIpcIo.java
deleted file mode 100644
index d07f34d..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopIpcIo.java
+++ /dev/null
@@ -1,599 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.commons.logging.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.igfs.common.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.ipc.*;
-import org.apache.ignite.internal.util.ipc.shmem.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jdk8.backport.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-import java.util.concurrent.locks.*;
-
-/**
- * IO layer implementation based on blocking IPC streams.
- */
-@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
-public class IgfsHadoopIpcIo implements IgfsHadoopIo {
- /** Logger. */
- private Log log;
-
- /** Request futures map. */
- private ConcurrentMap<Long, IgfsHadoopFuture> reqMap =
- new ConcurrentHashMap8<>();
-
- /** Request ID counter. */
- private AtomicLong reqIdCnt = new AtomicLong();
-
- /** Endpoint. */
- private IpcEndpoint endpoint;
-
- /** Endpoint output stream. */
- private IgfsDataOutputStream out;
-
- /** Protocol. */
- private final IgfsMarshaller marsh;
-
- /** Client reader thread. */
- private Thread reader;
-
- /** Lock for graceful shutdown. */
- private final ReadWriteLock busyLock = new ReentrantReadWriteLock();
-
- /** Stopping flag. */
- private volatile boolean stopping;
-
- /** Server endpoint address. */
- private final String endpointAddr;
-
- /** Number of open file system sessions. */
- private final AtomicInteger activeCnt = new AtomicInteger(1);
-
- /** Event listeners. */
- private final Collection<IgfsHadoopIpcIoListener> lsnrs =
- new GridConcurrentHashSet<>();
-
- /** Cached connections. */
- private static final ConcurrentMap<String, IgfsHadoopIpcIo> ipcCache =
- new ConcurrentHashMap8<>();
-
- /** Striped lock that prevents multiple instance creation in {@link #get(Log, String)}. */
- private static final GridStripedLock initLock = new GridStripedLock(32);
-
- /**
- * @param endpointAddr Endpoint.
- * @param marsh Protocol.
- * @param log Logger to use.
- */
- public IgfsHadoopIpcIo(String endpointAddr, IgfsMarshaller marsh, Log log) {
- assert endpointAddr != null;
- assert marsh != null;
-
- this.endpointAddr = endpointAddr;
- this.marsh = marsh;
- this.log = log;
- }
-
- /**
- * Returns a started and valid instance of this class
- * for a given endpoint.
- *
- * @param log Logger to use for new instance.
- * @param endpoint Endpoint string.
- * @return New or existing cached instance, which is started and operational.
- * @throws IOException If new instance was created but failed to start.
- */
- public static IgfsHadoopIpcIo get(Log log, String endpoint) throws IOException {
- while (true) {
- IgfsHadoopIpcIo clientIo = ipcCache.get(endpoint);
-
- if (clientIo != null) {
- if (clientIo.acquire())
- return clientIo;
- else
- // If concurrent close.
- ipcCache.remove(endpoint, clientIo);
- }
- else {
- Lock lock = initLock.getLock(endpoint);
-
- lock.lock();
-
- try {
- clientIo = ipcCache.get(endpoint);
-
- if (clientIo != null) { // Perform double check.
- if (clientIo.acquire())
- return clientIo;
- else
- // If concurrent close.
- ipcCache.remove(endpoint, clientIo);
- }
-
- // Otherwise try creating a new one.
- clientIo = new IgfsHadoopIpcIo(endpoint, new IgfsMarshaller(), log);
-
- try {
- clientIo.start();
- }
- catch (IgniteCheckedException e) {
- throw new IOException(e.getMessage(), e);
- }
-
- IgfsHadoopIpcIo old = ipcCache.putIfAbsent(endpoint, clientIo);
-
- // Put in exclusive lock.
- assert old == null;
-
- return clientIo;
- }
- finally {
- lock.unlock();
- }
- }
- }
- }
-
- /**
- * Increases usage count for this instance.
- *
- * @return {@code true} if usage count is greater than zero.
- */
- private boolean acquire() {
- while (true) {
- int cnt = activeCnt.get();
-
- if (cnt == 0) {
- if (log.isDebugEnabled())
- log.debug("IPC IO not acquired (count was 0): " + this);
-
- return false;
- }
-
- // Need to make sure that no-one decremented count in between.
- if (activeCnt.compareAndSet(cnt, cnt + 1)) {
- if (log.isDebugEnabled())
- log.debug("IPC IO acquired: " + this);
-
- return true;
- }
- }
- }
-
- /**
- * Releases this instance, decrementing usage count.
- * <p>
- * If usage count becomes zero, the instance is stopped
- * and removed from cache.
- */
- public void release() {
- while (true) {
- int cnt = activeCnt.get();
-
- if (cnt == 0) {
- if (log.isDebugEnabled())
- log.debug("IPC IO not released (count was 0): " + this);
-
- return;
- }
-
- if (activeCnt.compareAndSet(cnt, cnt - 1)) {
- if (cnt == 1) {
- ipcCache.remove(endpointAddr, this);
-
- if (log.isDebugEnabled())
- log.debug("IPC IO stopping as unused: " + this);
-
- stop();
- }
- else if (log.isDebugEnabled())
- log.debug("IPC IO released: " + this);
-
- return;
- }
- }
- }
-
- /**
- * Closes this IO instance, removing it from cache.
- */
- public void forceClose() {
- if (ipcCache.remove(endpointAddr, this))
- stop();
- }
-
- /**
- * Starts the IO.
- *
- * @throws IgniteCheckedException If failed to connect the endpoint.
- */
- private void start() throws IgniteCheckedException {
- boolean success = false;
-
- try {
- endpoint = IpcEndpointFactory.connectEndpoint(
- endpointAddr, new GridLoggerProxy(new IgfsHadoopJclLogger(log), null, null, ""));
-
- out = new IgfsDataOutputStream(new BufferedOutputStream(endpoint.outputStream()));
-
- reader = new ReaderThread();
-
- // Required for Hadoop 2.x
- reader.setDaemon(true);
-
- reader.start();
-
- success = true;
- }
- catch (IgniteCheckedException e) {
- IpcOutOfSystemResourcesException resEx = e.getCause(IpcOutOfSystemResourcesException.class);
-
- if (resEx != null)
- throw new IgniteCheckedException(IpcSharedMemoryServerEndpoint.OUT_OF_RESOURCES_MSG, resEx);
-
- throw e;
- }
- finally {
- if (!success)
- stop();
- }
- }
-
- /**
- * Shuts down the IO. No send requests will be accepted anymore, all pending futures will be failed.
- * Close listeners will be invoked as if connection is closed by server.
- */
- private void stop() {
- close0(null);
-
- if (reader != null) {
- try {
- U.interrupt(reader);
- U.join(reader);
-
- reader = null;
- }
- catch (IgniteInterruptedCheckedException ignored) {
- Thread.currentThread().interrupt();
-
- log.warn("Got interrupted while waiting for reader thread to shut down (will return).");
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public void addEventListener(IgfsHadoopIpcIoListener lsnr) {
- if (!busyLock.readLock().tryLock()) {
- lsnr.onClose();
-
- return;
- }
-
- boolean invokeNow = false;
-
- try {
- invokeNow = stopping;
-
- if (!invokeNow)
- lsnrs.add(lsnr);
- }
- finally {
- busyLock.readLock().unlock();
-
- if (invokeNow)
- lsnr.onClose();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void removeEventListener(IgfsHadoopIpcIoListener lsnr) {
- lsnrs.remove(lsnr);
- }
-
- /** {@inheritDoc} */
- @Override public GridPlainFuture<IgfsMessage> send(IgfsMessage msg) throws IgniteCheckedException {
- return send(msg, null, 0, 0);
- }
-
- /** {@inheritDoc} */
- @Override public <T> GridPlainFuture<T> send(IgfsMessage msg, @Nullable byte[] outBuf, int outOff,
- int outLen) throws IgniteCheckedException {
- assert outBuf == null || msg.command() == IgfsIpcCommand.READ_BLOCK;
-
- if (!busyLock.readLock().tryLock())
- throw new IgfsHadoopCommunicationException("Failed to send message (client is being concurrently " +
- "closed).");
-
- try {
- if (stopping)
- throw new IgfsHadoopCommunicationException("Failed to send message (client is being concurrently " +
- "closed).");
-
- long reqId = reqIdCnt.getAndIncrement();
-
- IgfsHadoopFuture<T> fut = new IgfsHadoopFuture<>();
-
- fut.outputBuffer(outBuf);
- fut.outputOffset(outOff);
- fut.outputLength(outLen);
- fut.read(msg.command() == IgfsIpcCommand.READ_BLOCK);
-
- IgfsHadoopFuture oldFut = reqMap.putIfAbsent(reqId, fut);
-
- assert oldFut == null;
-
- if (log.isDebugEnabled())
- log.debug("Sending IGFS message [reqId=" + reqId + ", msg=" + msg + ']');
-
- byte[] hdr = IgfsMarshaller.createHeader(reqId, msg.command());
-
- IgniteCheckedException err = null;
-
- try {
- synchronized (this) {
- marsh.marshall(msg, hdr, out);
-
- out.flush(); // Blocking operation + sometimes system call.
- }
- }
- catch (IgniteCheckedException e) {
- err = e;
- }
- catch (IOException e) {
- err = new IgfsHadoopCommunicationException(e);
- }
-
- if (err != null) {
- reqMap.remove(reqId, fut);
-
- fut.onDone(err);
- }
-
- return fut;
- }
- finally {
- busyLock.readLock().unlock();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void sendPlain(IgfsMessage msg) throws IgniteCheckedException {
- if (!busyLock.readLock().tryLock())
- throw new IgfsHadoopCommunicationException("Failed to send message (client is being " +
- "concurrently closed).");
-
- try {
- if (stopping)
- throw new IgfsHadoopCommunicationException("Failed to send message (client is being concurrently closed).");
-
- assert msg.command() == IgfsIpcCommand.WRITE_BLOCK;
-
- IgfsStreamControlRequest req = (IgfsStreamControlRequest)msg;
-
- byte[] hdr = IgfsMarshaller.createHeader(-1, IgfsIpcCommand.WRITE_BLOCK);
-
- U.longToBytes(req.streamId(), hdr, 12);
- U.intToBytes(req.length(), hdr, 20);
-
- synchronized (this) {
- out.write(hdr);
- out.write(req.data(), (int)req.position(), req.length());
-
- out.flush();
- }
- }
- catch (IOException e) {
- throw new IgfsHadoopCommunicationException(e);
- }
- finally {
- busyLock.readLock().unlock();
- }
- }
-
- /**
- * Closes client but does not wait.
- *
- * @param err Error.
- */
- private void close0(@Nullable Throwable err) {
- busyLock.writeLock().lock();
-
- try {
- if (stopping)
- return;
-
- stopping = true;
- }
- finally {
- busyLock.writeLock().unlock();
- }
-
- if (err == null)
- err = new IgniteCheckedException("Failed to perform request (connection was concurrently closed before response " +
- "is received).");
-
- // Clean up resources.
- U.closeQuiet(out);
-
- if (endpoint != null)
- endpoint.close();
-
- // Unwind futures. We can safely iterate here because no more futures will be added.
- Iterator<IgfsHadoopFuture> it = reqMap.values().iterator();
-
- while (it.hasNext()) {
- IgfsHadoopFuture fut = it.next();
-
- fut.onDone(err);
-
- it.remove();
- }
-
- for (IgfsHadoopIpcIoListener lsnr : lsnrs)
- lsnr.onClose();
- }
-
- /**
- * Do not extend {@code GridThread} to minimize class dependencies.
- */
- private class ReaderThread extends Thread {
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public void run() {
- // Error to fail pending futures.
- Throwable err = null;
-
- try {
- InputStream in = endpoint.inputStream();
-
- IgfsDataInputStream dis = new IgfsDataInputStream(in);
-
- byte[] hdr = new byte[IgfsMarshaller.HEADER_SIZE];
- byte[] msgHdr = new byte[IgfsControlResponse.RES_HEADER_SIZE];
-
- while (!Thread.currentThread().isInterrupted()) {
- dis.readFully(hdr);
-
- long reqId = U.bytesToLong(hdr, 0);
-
- // We don't wait for write responses, therefore reqId is -1.
- if (reqId == -1) {
- // We received a response which normally should not be sent. It must contain an error.
- dis.readFully(msgHdr);
-
- assert msgHdr[4] != 0;
-
- String errMsg = dis.readUTF();
-
- // Error code.
- dis.readInt();
-
- long streamId = dis.readLong();
-
- for (IgfsHadoopIpcIoListener lsnr : lsnrs)
- lsnr.onError(streamId, errMsg);
- }
- else {
- IgfsHadoopFuture<Object> fut = reqMap.remove(reqId);
-
- if (fut == null) {
- String msg = "Failed to read response from server: response closure is unavailable for " +
- "requestId (will close connection):" + reqId;
-
- log.warn(msg);
-
- err = new IgniteCheckedException(msg);
-
- break;
- }
- else {
- try {
- IgfsIpcCommand cmd = IgfsIpcCommand.valueOf(U.bytesToInt(hdr, 8));
-
- if (log.isDebugEnabled())
- log.debug("Received IGFS response [reqId=" + reqId + ", cmd=" + cmd + ']');
-
- Object res = null;
-
- if (fut.read()) {
- dis.readFully(msgHdr);
-
- boolean hasErr = msgHdr[4] != 0;
-
- if (hasErr) {
- String errMsg = dis.readUTF();
-
- // Error code.
- Integer errCode = dis.readInt();
-
- IgfsControlResponse.throwError(errCode, errMsg);
- }
-
- int blockLen = U.bytesToInt(msgHdr, 5);
-
- int readLen = Math.min(blockLen, fut.outputLength());
-
- if (readLen > 0) {
- assert fut.outputBuffer() != null;
-
- dis.readFully(fut.outputBuffer(), fut.outputOffset(), readLen);
- }
-
- if (readLen != blockLen) {
- byte[] buf = new byte[blockLen - readLen];
-
- dis.readFully(buf);
-
- res = buf;
- }
- }
- else
- res = marsh.unmarshall(cmd, hdr, dis);
-
- fut.onDone(res);
- }
- catch (IgniteCheckedException e) {
- if (log.isDebugEnabled())
- log.debug("Failed to apply response closure (will fail request future): " +
- e.getMessage());
-
- fut.onDone(e);
-
- err = e;
- }
- }
- }
- }
- }
- catch (EOFException ignored) {
- err = new IgniteCheckedException("Failed to read response from server (connection was closed by remote peer).");
- }
- catch (IOException e) {
- if (!stopping)
- log.error("Failed to read data (connection will be closed)", e);
-
- err = new IgfsHadoopCommunicationException(e);
- }
- catch (IgniteCheckedException e) {
- if (!stopping)
- log.error("Failed to obtain endpoint input stream (connection will be closed)", e);
-
- err = e;
- }
- finally {
- close0(err);
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return getClass().getSimpleName() + " [endpointAddr=" + endpointAddr + ", activeCnt=" + activeCnt +
- ", stopping=" + stopping + ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopIpcIoListener.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopIpcIoListener.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopIpcIoListener.java
deleted file mode 100644
index ffc58ba..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopIpcIoListener.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-/**
- * Listens to the events of {@link IgfsHadoopIpcIo}.
- */
-public interface IgfsHadoopIpcIoListener {
- /**
- * Callback invoked when the IO is being closed.
- */
- public void onClose();
-
- /**
- * Callback invoked when remote error occurs.
- *
- * @param streamId Stream ID.
- * @param errMsg Error message.
- */
- public void onError(long streamId, String errMsg);
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopJclLogger.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopJclLogger.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopJclLogger.java
deleted file mode 100644
index e43d77a..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopJclLogger.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.commons.logging.*;
-import org.apache.ignite.*;
-import org.jetbrains.annotations.*;
-
-/**
- * JCL logger wrapper for Hadoop.
- */
-public class IgfsHadoopJclLogger implements IgniteLogger {
- /** JCL implementation proxy. */
- private Log impl;
-
- /**
- * Constructor.
- *
- * @param impl JCL implementation to use.
- */
- IgfsHadoopJclLogger(Log impl) {
- assert impl != null;
-
- this.impl = impl;
- }
-
- /** {@inheritDoc} */
- @Override public IgniteLogger getLogger(Object ctgr) {
- return new IgfsHadoopJclLogger(LogFactory.getLog(
- ctgr instanceof Class ? ((Class)ctgr).getName() : String.valueOf(ctgr)));
- }
-
- /** {@inheritDoc} */
- @Override public void trace(String msg) {
- impl.trace(msg);
- }
-
- /** {@inheritDoc} */
- @Override public void debug(String msg) {
- impl.debug(msg);
- }
-
- /** {@inheritDoc} */
- @Override public void info(String msg) {
- impl.info(msg);
- }
-
- /** {@inheritDoc} */
- @Override public void warning(String msg) {
- impl.warn(msg);
- }
-
- /** {@inheritDoc} */
- @Override public void warning(String msg, @Nullable Throwable e) {
- impl.warn(msg, e);
- }
-
- /** {@inheritDoc} */
- @Override public void error(String msg) {
- impl.error(msg);
- }
-
- /** {@inheritDoc} */
- @Override public boolean isQuiet() {
- return !isInfoEnabled() && !isDebugEnabled();
- }
-
- /** {@inheritDoc} */
- @Override public void error(String msg, @Nullable Throwable e) {
- impl.error(msg, e);
- }
-
- /** {@inheritDoc} */
- @Override public boolean isTraceEnabled() {
- return impl.isTraceEnabled();
- }
-
- /** {@inheritDoc} */
- @Override public boolean isDebugEnabled() {
- return impl.isDebugEnabled();
- }
-
- /** {@inheritDoc} */
- @Override public boolean isInfoEnabled() {
- return impl.isInfoEnabled();
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public String fileName() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "IgfsHadoopJclLogger [impl=" + impl + ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopOutProc.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopOutProc.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopOutProc.java
deleted file mode 100644
index 31183a8..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopOutProc.java
+++ /dev/null
@@ -1,466 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.commons.logging.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.igfs.common.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.jdk8.backport.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-
-import static org.apache.ignite.internal.igfs.common.IgfsIpcCommand.*;
-
-/**
- * Communication with external process (TCP or shmem).
- */
-public class IgfsHadoopOutProc implements IgfsHadoopEx, IgfsHadoopIpcIoListener {
- /** Expected result is boolean. */
- private static final GridPlainClosure<GridPlainFuture<IgfsMessage>, Boolean> BOOL_RES = createClosure();
-
- /** Expected result is boolean. */
- private static final GridPlainClosure<GridPlainFuture<IgfsMessage>, Long> LONG_RES = createClosure();
-
- /** Expected result is {@code IgfsFile}. */
- private static final GridPlainClosure<GridPlainFuture<IgfsMessage>, IgfsFile> FILE_RES = createClosure();
-
- /** Expected result is {@code IgfsHandshakeResponse} */
- private static final GridPlainClosure<GridPlainFuture<IgfsMessage>,
- IgfsHandshakeResponse> HANDSHAKE_RES = createClosure();
-
- /** Expected result is {@code IgfsStatus} */
- private static final GridPlainClosure<GridPlainFuture<IgfsMessage>, IgfsStatus> STATUS_RES =
- createClosure();
-
- /** Expected result is {@code IgfsFile}. */
- private static final GridPlainClosure<GridPlainFuture<IgfsMessage>,
- IgfsInputStreamDescriptor> STREAM_DESCRIPTOR_RES = createClosure();
-
- /** Expected result is {@code IgfsFile}. */
- private static final GridPlainClosure<GridPlainFuture<IgfsMessage>,
- Collection<IgfsFile>> FILE_COL_RES = createClosure();
-
- /** Expected result is {@code IgfsFile}. */
- private static final GridPlainClosure<GridPlainFuture<IgfsMessage>,
- Collection<IgfsPath>> PATH_COL_RES = createClosure();
-
- /** Expected result is {@code IgfsPathSummary}. */
- private static final GridPlainClosure<GridPlainFuture<IgfsMessage>, IgfsPathSummary> SUMMARY_RES =
- createClosure();
-
- /** Expected result is {@code IgfsFile}. */
- private static final GridPlainClosure<GridPlainFuture<IgfsMessage>,
- Collection<IgfsBlockLocation>> BLOCK_LOCATION_COL_RES = createClosure();
-
- /** Grid name. */
- private final String grid;
-
- /** IGFS name. */
- private final String igfs;
-
- /** Client log. */
- private final Log log;
-
- /** Client IO. */
- private final IgfsHadoopIpcIo io;
-
- /** Event listeners. */
- private final Map<Long, IgfsHadoopStreamEventListener> lsnrs = new ConcurrentHashMap8<>();
-
- /**
- * Constructor for TCP endpoint.
- *
- * @param host Host.
- * @param port Port.
- * @param grid Grid name.
- * @param igfs IGFS name.
- * @param log Client logger.
- * @throws IOException If failed.
- */
- public IgfsHadoopOutProc(String host, int port, String grid, String igfs, Log log) throws IOException {
- this(host, port, grid, igfs, false, log);
- }
-
- /**
- * Constructor for shmem endpoint.
- *
- * @param port Port.
- * @param grid Grid name.
- * @param igfs IGFS name.
- * @param log Client logger.
- * @throws IOException If failed.
- */
- public IgfsHadoopOutProc(int port, String grid, String igfs, Log log) throws IOException {
- this(null, port, grid, igfs, true, log);
- }
-
- /**
- * Constructor.
- *
- * @param host Host.
- * @param port Port.
- * @param grid Grid name.
- * @param igfs IGFS name.
- * @param shmem Shared memory flag.
- * @param log Client logger.
- * @throws IOException If failed.
- */
- private IgfsHadoopOutProc(String host, int port, String grid, String igfs, boolean shmem, Log log)
- throws IOException {
- assert host != null && !shmem || host == null && shmem :
- "Invalid arguments [host=" + host + ", port=" + port + ", shmem=" + shmem + ']';
-
- String endpoint = host != null ? host + ":" + port : "shmem:" + port;
-
- this.grid = grid;
- this.igfs = igfs;
- this.log = log;
-
- io = IgfsHadoopIpcIo.get(log, endpoint);
-
- io.addEventListener(this);
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHandshakeResponse handshake(String logDir) throws IgniteCheckedException {
- final IgfsHandshakeRequest req = new IgfsHandshakeRequest();
-
- req.gridName(grid);
- req.igfsName(igfs);
- req.logDirectory(logDir);
-
- return io.send(req).chain(HANDSHAKE_RES).get();
- }
-
- /** {@inheritDoc} */
- @Override public void close(boolean force) {
- assert io != null;
-
- io.removeEventListener(this);
-
- if (force)
- io.forceClose();
- else
- io.release();
- }
-
- /** {@inheritDoc} */
- @Override public IgfsFile info(IgfsPath path) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(INFO);
- msg.path(path);
-
- return io.send(msg).chain(FILE_RES).get();
- }
-
- /** {@inheritDoc} */
- @Override public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(UPDATE);
- msg.path(path);
- msg.properties(props);
-
- return io.send(msg).chain(FILE_RES).get();
- }
-
- /** {@inheritDoc} */
- @Override public Boolean setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(SET_TIMES);
- msg.path(path);
- msg.accessTime(accessTime);
- msg.modificationTime(modificationTime);
-
- return io.send(msg).chain(BOOL_RES).get();
- }
-
- /** {@inheritDoc} */
- @Override public Boolean rename(IgfsPath src, IgfsPath dest) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(RENAME);
- msg.path(src);
- msg.destinationPath(dest);
-
- return io.send(msg).chain(BOOL_RES).get();
- }
-
- /** {@inheritDoc} */
- @Override public Boolean delete(IgfsPath path, boolean recursive) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(DELETE);
- msg.path(path);
- msg.flag(recursive);
-
- return io.send(msg).chain(BOOL_RES).get();
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len)
- throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(AFFINITY);
- msg.path(path);
- msg.start(start);
- msg.length(len);
-
- return io.send(msg).chain(BLOCK_LOCATION_COL_RES).get();
- }
-
- /** {@inheritDoc} */
- @Override public IgfsPathSummary contentSummary(IgfsPath path) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(PATH_SUMMARY);
- msg.path(path);
-
- return io.send(msg).chain(SUMMARY_RES).get();
- }
-
- /** {@inheritDoc} */
- @Override public Boolean mkdirs(IgfsPath path, Map<String, String> props) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(MAKE_DIRECTORIES);
- msg.path(path);
- msg.properties(props);
-
- return io.send(msg).chain(BOOL_RES).get();
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(LIST_FILES);
- msg.path(path);
-
- return io.send(msg).chain(FILE_COL_RES).get();
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(LIST_PATHS);
- msg.path(path);
-
- return io.send(msg).chain(PATH_COL_RES).get();
- }
-
- /** {@inheritDoc} */
- @Override public IgfsStatus fsStatus() throws IgniteCheckedException {
- return io.send(new IgfsStatusRequest()).chain(STATUS_RES).get();
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHadoopStreamDelegate open(IgfsPath path) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(OPEN_READ);
- msg.path(path);
- msg.flag(false);
-
- IgfsInputStreamDescriptor rmtDesc = io.send(msg).chain(STREAM_DESCRIPTOR_RES).get();
-
- return new IgfsHadoopStreamDelegate(this, rmtDesc.streamId(), rmtDesc.length());
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHadoopStreamDelegate open(IgfsPath path,
- int seqReadsBeforePrefetch) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(OPEN_READ);
- msg.path(path);
- msg.flag(true);
- msg.sequentialReadsBeforePrefetch(seqReadsBeforePrefetch);
-
- IgfsInputStreamDescriptor rmtDesc = io.send(msg).chain(STREAM_DESCRIPTOR_RES).get();
-
- return new IgfsHadoopStreamDelegate(this, rmtDesc.streamId(), rmtDesc.length());
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHadoopStreamDelegate create(IgfsPath path, boolean overwrite, boolean colocate,
- int replication, long blockSize, @Nullable Map<String, String> props) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(OPEN_CREATE);
- msg.path(path);
- msg.flag(overwrite);
- msg.colocate(colocate);
- msg.properties(props);
- msg.replication(replication);
- msg.blockSize(blockSize);
-
- Long streamId = io.send(msg).chain(LONG_RES).get();
-
- return new IgfsHadoopStreamDelegate(this, streamId);
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHadoopStreamDelegate append(IgfsPath path, boolean create,
- @Nullable Map<String, String> props) throws IgniteCheckedException {
- final IgfsPathControlRequest msg = new IgfsPathControlRequest();
-
- msg.command(OPEN_APPEND);
- msg.path(path);
- msg.flag(create);
- msg.properties(props);
-
- Long streamId = io.send(msg).chain(LONG_RES).get();
-
- return new IgfsHadoopStreamDelegate(this, streamId);
- }
-
- /** {@inheritDoc} */
- @Override public GridPlainFuture<byte[]> readData(IgfsHadoopStreamDelegate desc, long pos, int len,
- final @Nullable byte[] outBuf, final int outOff, final int outLen) {
- assert len > 0;
-
- final IgfsStreamControlRequest msg = new IgfsStreamControlRequest();
-
- msg.command(READ_BLOCK);
- msg.streamId((long) desc.target());
- msg.position(pos);
- msg.length(len);
-
- try {
- return io.send(msg, outBuf, outOff, outLen);
- }
- catch (IgniteCheckedException e) {
- return new GridPlainFutureAdapter<>(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void writeData(IgfsHadoopStreamDelegate desc, byte[] data, int off, int len)
- throws IOException {
- final IgfsStreamControlRequest msg = new IgfsStreamControlRequest();
-
- msg.command(WRITE_BLOCK);
- msg.streamId((long) desc.target());
- msg.data(data);
- msg.position(off);
- msg.length(len);
-
- try {
- io.sendPlain(msg);
- }
- catch (IgniteCheckedException e) {
- throw IgfsHadoopUtils.cast(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void flush(IgfsHadoopStreamDelegate delegate) throws IOException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void closeStream(IgfsHadoopStreamDelegate desc) throws IOException {
- final IgfsStreamControlRequest msg = new IgfsStreamControlRequest();
-
- msg.command(CLOSE);
- msg.streamId((long)desc.target());
-
- try {
- io.send(msg).chain(BOOL_RES).get();
- }
- catch (IgniteCheckedException e) {
- throw IgfsHadoopUtils.cast(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void addEventListener(IgfsHadoopStreamDelegate desc,
- IgfsHadoopStreamEventListener lsnr) {
- long streamId = desc.target();
-
- IgfsHadoopStreamEventListener lsnr0 = lsnrs.put(streamId, lsnr);
-
- assert lsnr0 == null || lsnr0 == lsnr;
-
- if (log.isDebugEnabled())
- log.debug("Added stream event listener [streamId=" + streamId + ']');
- }
-
- /** {@inheritDoc} */
- @Override public void removeEventListener(IgfsHadoopStreamDelegate desc) {
- long streamId = desc.target();
-
- IgfsHadoopStreamEventListener lsnr0 = lsnrs.remove(streamId);
-
- if (lsnr0 != null && log.isDebugEnabled())
- log.debug("Removed stream event listener [streamId=" + streamId + ']');
- }
-
- /** {@inheritDoc} */
- @Override public void onClose() {
- for (IgfsHadoopStreamEventListener lsnr : lsnrs.values()) {
- try {
- lsnr.onClose();
- }
- catch (IgniteCheckedException e) {
- log.warn("Got exception from stream event listener (will ignore): " + lsnr, e);
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public void onError(long streamId, String errMsg) {
- IgfsHadoopStreamEventListener lsnr = lsnrs.get(streamId);
-
- if (lsnr != null)
- lsnr.onError(errMsg);
- else
- log.warn("Received write error response for not registered output stream (will ignore) " +
- "[streamId= " + streamId + ']');
- }
-
- /**
- * Creates conversion closure for given type.
- *
- * @param <T> Type of expected result.
- * @return Conversion closure.
- */
- @SuppressWarnings("unchecked")
- private static <T> GridPlainClosure<GridPlainFuture<IgfsMessage>, T> createClosure() {
- return new GridPlainClosure<GridPlainFuture<IgfsMessage>, T>() {
- @Override public T apply(GridPlainFuture<IgfsMessage> fut) throws IgniteCheckedException {
- IgfsControlResponse res = (IgfsControlResponse)fut.get();
-
- if (res.hasError())
- res.throwError();
-
- return (T)res.response();
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopOutputStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopOutputStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopOutputStream.java
deleted file mode 100644
index ae5f980..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopOutputStream.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.commons.logging.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.igfs.common.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * IGFS Hadoop output stream implementation.
- */
-public class IgfsHadoopOutputStream extends OutputStream implements IgfsHadoopStreamEventListener {
- /** Log instance. */
- private Log log;
-
- /** Client logger. */
- private IgfsLogger clientLog;
-
- /** Log stream ID. */
- private long logStreamId;
-
- /** Server stream delegate. */
- private IgfsHadoopStreamDelegate delegate;
-
- /** Closed flag. */
- private volatile boolean closed;
-
- /** Flag set if stream was closed due to connection breakage. */
- private boolean connBroken;
-
- /** Error message. */
- private volatile String errMsg;
-
- /** Read time. */
- private long writeTime;
-
- /** User time. */
- private long userTime;
-
- /** Last timestamp. */
- private long lastTs;
-
- /** Amount of written bytes. */
- private long total;
-
- /**
- * Creates light output stream.
- *
- * @param delegate Server stream delegate.
- * @param log Logger to use.
- * @param clientLog Client logger.
- */
- public IgfsHadoopOutputStream(IgfsHadoopStreamDelegate delegate, Log log,
- IgfsLogger clientLog, long logStreamId) {
- this.delegate = delegate;
- this.log = log;
- this.clientLog = clientLog;
- this.logStreamId = logStreamId;
-
- lastTs = System.nanoTime();
-
- delegate.hadoop().addEventListener(delegate, this);
- }
-
- /**
- * Read start.
- */
- private void writeStart() {
- long now = System.nanoTime();
-
- userTime += now - lastTs;
-
- lastTs = now;
- }
-
- /**
- * Read end.
- */
- private void writeEnd() {
- long now = System.nanoTime();
-
- writeTime += now - lastTs;
-
- lastTs = now;
- }
-
- /** {@inheritDoc} */
- @Override public void write(@NotNull byte[] b, int off, int len) throws IOException {
- check();
-
- writeStart();
-
- try {
- delegate.hadoop().writeData(delegate, b, off, len);
-
- total += len;
- }
- finally {
- writeEnd();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void write(int b) throws IOException {
- write(new byte[] {(byte)b});
-
- total++;
- }
-
- /** {@inheritDoc} */
- @Override public void flush() throws IOException {
- delegate.hadoop().flush(delegate);
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IOException {
- if (!closed) {
- if (log.isDebugEnabled())
- log.debug("Closing output stream: " + delegate);
-
- writeStart();
-
- delegate.hadoop().closeStream(delegate);
-
- markClosed(false);
-
- writeEnd();
-
- if (clientLog.isLogEnabled())
- clientLog.logCloseOut(logStreamId, userTime, writeTime, total);
-
- if (log.isDebugEnabled())
- log.debug("Closed output stream [delegate=" + delegate + ", writeTime=" + writeTime / 1000 +
- ", userTime=" + userTime / 1000 + ']');
- }
- else if(connBroken)
- throw new IOException(
- "Failed to close stream, because connection was broken (data could have been lost).");
- }
-
- /**
- * Marks stream as closed.
- *
- * @param connBroken {@code True} if connection with server was lost.
- */
- private void markClosed(boolean connBroken) {
- // It is ok to have race here.
- if (!closed) {
- closed = true;
-
- delegate.hadoop().removeEventListener(delegate);
-
- this.connBroken = connBroken;
- }
- }
-
- /**
- * @throws IOException If check failed.
- */
- private void check() throws IOException {
- String errMsg0 = errMsg;
-
- if (errMsg0 != null)
- throw new IOException(errMsg0);
-
- if (closed) {
- if (connBroken)
- throw new IOException("Server connection was lost.");
- else
- throw new IOException("Stream is closed.");
- }
- }
-
- /** {@inheritDoc} */
- @Override public void onClose() throws IgniteCheckedException {
- markClosed(true);
- }
-
- /** {@inheritDoc} */
- @Override public void onError(String errMsg) {
- this.errMsg = errMsg;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopProxyInputStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopProxyInputStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopProxyInputStream.java
deleted file mode 100644
index 330537d..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopProxyInputStream.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.hadoop.fs.*;
-import org.apache.ignite.internal.igfs.common.*;
-
-import java.io.*;
-
-/**
- * Secondary Hadoop file system input stream wrapper.
- */
-public class IgfsHadoopProxyInputStream extends InputStream implements Seekable, PositionedReadable {
- /** Actual input stream to the secondary file system. */
- private final FSDataInputStream is;
-
- /** Client logger. */
- private final IgfsLogger clientLog;
-
- /** Log stream ID. */
- private final long logStreamId;
-
- /** Read time. */
- private long readTime;
-
- /** User time. */
- private long userTime;
-
- /** Last timestamp. */
- private long lastTs;
-
- /** Amount of read bytes. */
- private long total;
-
- /** Closed flag. */
- private boolean closed;
-
- /**
- * Constructor.
- *
- * @param is Actual input stream to the secondary file system.
- * @param clientLog Client log.
- */
- public IgfsHadoopProxyInputStream(FSDataInputStream is, IgfsLogger clientLog, long logStreamId) {
- assert is != null;
- assert clientLog != null;
-
- this.is = is;
- this.clientLog = clientLog;
- this.logStreamId = logStreamId;
-
- lastTs = System.nanoTime();
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int read(byte[] b) throws IOException {
- readStart();
-
- int res;
-
- try {
- res = is.read(b);
- }
- finally {
- readEnd();
- }
-
- if (res != -1)
- total += res;
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int read(byte[] b, int off, int len) throws IOException {
- readStart();
-
- int res;
-
- try {
- res = super.read(b, off, len);
- }
- finally {
- readEnd();
- }
-
- if (res != -1)
- total += res;
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized long skip(long n) throws IOException {
- readStart();
-
- long res;
-
- try {
- res = is.skip(n);
- }
- finally {
- readEnd();
- }
-
- if (clientLog.isLogEnabled())
- clientLog.logSkip(logStreamId, res);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int available() throws IOException {
- readStart();
-
- try {
- return is.available();
- }
- finally {
- readEnd();
- }
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void close() throws IOException {
- if (!closed) {
- closed = true;
-
- readStart();
-
- try {
- is.close();
- }
- finally {
- readEnd();
- }
-
- if (clientLog.isLogEnabled())
- clientLog.logCloseIn(logStreamId, userTime, readTime, total);
- }
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void mark(int readLimit) {
- readStart();
-
- try {
- is.mark(readLimit);
- }
- finally {
- readEnd();
- }
-
- if (clientLog.isLogEnabled())
- clientLog.logMark(logStreamId, readLimit);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void reset() throws IOException {
- readStart();
-
- try {
- is.reset();
- }
- finally {
- readEnd();
- }
-
- if (clientLog.isLogEnabled())
- clientLog.logReset(logStreamId);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized boolean markSupported() {
- readStart();
-
- try {
- return is.markSupported();
- }
- finally {
- readEnd();
- }
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int read() throws IOException {
- readStart();
-
- int res;
-
- try {
- res = is.read();
- }
- finally {
- readEnd();
- }
-
- if (res != -1)
- total++;
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int read(long pos, byte[] buf, int off, int len) throws IOException {
- readStart();
-
- int res;
-
- try {
- res = is.read(pos, buf, off, len);
- }
- finally {
- readEnd();
- }
-
- if (res != -1)
- total += res;
-
- if (clientLog.isLogEnabled())
- clientLog.logRandomRead(logStreamId, pos, res);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void readFully(long pos, byte[] buf, int off, int len) throws IOException {
- readStart();
-
- try {
- is.readFully(pos, buf, off, len);
- }
- finally {
- readEnd();
- }
-
- total += len;
-
- if (clientLog.isLogEnabled())
- clientLog.logRandomRead(logStreamId, pos, len);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void readFully(long pos, byte[] buf) throws IOException {
- readStart();
-
- try {
- is.readFully(pos, buf);
- }
- finally {
- readEnd();
- }
-
- total += buf.length;
-
- if (clientLog.isLogEnabled())
- clientLog.logRandomRead(logStreamId, pos, buf.length);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void seek(long pos) throws IOException {
- readStart();
-
- try {
- is.seek(pos);
- }
- finally {
- readEnd();
- }
-
- if (clientLog.isLogEnabled())
- clientLog.logSeek(logStreamId, pos);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized long getPos() throws IOException {
- readStart();
-
- try {
- return is.getPos();
- }
- finally {
- readEnd();
- }
- }
-
- /** {@inheritDoc} */
- @Override public synchronized boolean seekToNewSource(long targetPos) throws IOException {
- readStart();
-
- try {
- return is.seekToNewSource(targetPos);
- }
- finally {
- readEnd();
- }
- }
-
- /**
- * Read start.
- */
- private void readStart() {
- long now = System.nanoTime();
-
- userTime += now - lastTs;
-
- lastTs = now;
- }
-
- /**
- * Read end.
- */
- private void readEnd() {
- long now = System.nanoTime();
-
- readTime += now - lastTs;
-
- lastTs = now;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopProxyOutputStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopProxyOutputStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopProxyOutputStream.java
deleted file mode 100644
index 41e80eb..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopProxyOutputStream.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.hadoop.fs.*;
-import org.apache.ignite.internal.igfs.common.*;
-
-import java.io.*;
-
-/**
- * Secondary Hadoop file system output stream wrapper.
- */
-public class IgfsHadoopProxyOutputStream extends OutputStream {
- /** Actual output stream. */
- private FSDataOutputStream os;
-
- /** Client logger. */
- private final IgfsLogger clientLog;
-
- /** Log stream ID. */
- private final long logStreamId;
-
- /** Read time. */
- private long writeTime;
-
- /** User time. */
- private long userTime;
-
- /** Last timestamp. */
- private long lastTs;
-
- /** Amount of written bytes. */
- private long total;
-
- /** Closed flag. */
- private boolean closed;
-
- /**
- * Constructor.
- *
- * @param os Actual output stream.
- * @param clientLog Client logger.
- * @param logStreamId Log stream ID.
- */
- public IgfsHadoopProxyOutputStream(FSDataOutputStream os, IgfsLogger clientLog, long logStreamId) {
- assert os != null;
- assert clientLog != null;
-
- this.os = os;
- this.clientLog = clientLog;
- this.logStreamId = logStreamId;
-
- lastTs = System.nanoTime();
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void write(int b) throws IOException {
- writeStart();
-
- try {
- os.write(b);
- }
- finally {
- writeEnd();
- }
-
- total++;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void write(byte[] b) throws IOException {
- writeStart();
-
- try {
- os.write(b);
- }
- finally {
- writeEnd();
- }
-
- total += b.length;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void write(byte[] b, int off, int len) throws IOException {
- writeStart();
-
- try {
- os.write(b, off, len);
- }
- finally {
- writeEnd();
- }
-
- total += len;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void flush() throws IOException {
- writeStart();
-
- try {
- os.flush();
- }
- finally {
- writeEnd();
- }
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void close() throws IOException {
- if (!closed) {
- closed = true;
-
- writeStart();
-
- try {
- os.close();
- }
- finally {
- writeEnd();
- }
-
- if (clientLog.isLogEnabled())
- clientLog.logCloseOut(logStreamId, userTime, writeTime, total);
- }
- }
-
- /**
- * Read start.
- */
- private void writeStart() {
- long now = System.nanoTime();
-
- userTime += now - lastTs;
-
- lastTs = now;
- }
-
- /**
- * Read end.
- */
- private void writeEnd() {
- long now = System.nanoTime();
-
- writeTime += now - lastTs;
-
- lastTs = now;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopReader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopReader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopReader.java
deleted file mode 100644
index 3ab3acc..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopReader.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-
-/**
- * Secondary file system input stream wrapper which actually opens input stream only in case it is explicitly
- * requested.
- * <p>
- * The class is expected to be used only from synchronized context and therefore is not tread-safe.
- */
-public class IgfsHadoopReader implements IgfsReader {
- /** Secondary file system. */
- private final FileSystem fs;
-
- /** Path to the file to open. */
- private final Path path;
-
- /** Buffer size. */
- private final int bufSize;
-
- /** Actual input stream. */
- private FSDataInputStream in;
-
- /** Cached error occurred during output stream open. */
- private IOException err;
-
- /** Flag indicating that the stream was already opened. */
- private boolean opened;
-
- /**
- * Constructor.
- *
- * @param fs Secondary file system.
- * @param path Path to the file to open.
- * @param bufSize Buffer size.
- */
- public IgfsHadoopReader(FileSystem fs, Path path, int bufSize) {
- assert fs != null;
- assert path != null;
-
- this.fs = fs;
- this.path = path;
- this.bufSize = bufSize;
- }
-
- /** Get input stream. */
- private PositionedReadable in() throws IOException {
- if (opened) {
- if (err != null)
- throw err;
- }
- else {
- opened = true;
-
- try {
- in = fs.open(path, bufSize);
-
- if (in == null)
- throw new IOException("Failed to open input stream (file system returned null): " + path);
- }
- catch (IOException e) {
- err = e;
-
- throw err;
- }
- }
-
- return in;
- }
-
- /**
- * Close wrapped input stream in case it was previously opened.
- */
- @Override public void close() {
- U.closeQuiet(in);
- }
-
- /** {@inheritDoc} */
- @Override public int read(long pos, byte[] buf, int off, int len) throws IOException {
- return in().read(pos, buf, off, len);
- }
-}
[06/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobTrackerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobTrackerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobTrackerSelfTest.java
deleted file mode 100644
index 3aa74d0..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobTrackerSelfTest.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.hadoop.mapreduce.lib.output.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Job tracker self test.
- */
-public class GridHadoopJobTrackerSelfTest extends GridHadoopAbstractSelfTest {
- /** */
- private static final String PATH_OUTPUT = "/test-out";
-
- /** Test block count parameter name. */
- private static final int BLOCK_CNT = 10;
-
- /** */
- private static GridHadoopSharedMap m = GridHadoopSharedMap.map(GridHadoopJobTrackerSelfTest.class);
-
- /** Map task execution count. */
- private static final AtomicInteger mapExecCnt = m.put("mapExecCnt", new AtomicInteger());
-
- /** Reduce task execution count. */
- private static final AtomicInteger reduceExecCnt = m.put("reduceExecCnt", new AtomicInteger());
-
- /** Reduce task execution count. */
- private static final AtomicInteger combineExecCnt = m.put("combineExecCnt", new AtomicInteger());
-
- /** */
- private static final Map<String, CountDownLatch> latch = m.put("latch", new HashMap<String, CountDownLatch>());
-
- /** {@inheritDoc} */
- @Override protected boolean igfsEnabled() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
- startGrids(gridCount());
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- stopAllGrids();
-
- super.afterTestsStopped();
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- latch.put("mapAwaitLatch", new CountDownLatch(1));
- latch.put("reduceAwaitLatch", new CountDownLatch(1));
- latch.put("combineAwaitLatch", new CountDownLatch(1));
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- mapExecCnt.set(0);
- combineExecCnt.set(0);
- reduceExecCnt.set(0);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopConfiguration hadoopConfiguration(String gridName) {
- GridHadoopConfiguration cfg = super.hadoopConfiguration(gridName);
-
- cfg.setMapReducePlanner(new GridHadoopTestRoundRobinMrPlanner());
- cfg.setExternalExecution(false);
-
- return cfg;
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testSimpleTaskSubmit() throws Exception {
- try {
- UUID globalId = UUID.randomUUID();
-
- Job job = Job.getInstance();
- setupFileSystems(job.getConfiguration());
-
- job.setMapperClass(TestMapper.class);
- job.setReducerClass(TestReducer.class);
- job.setInputFormatClass(InFormat.class);
-
- FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT + "1"));
-
- GridHadoopJobId jobId = new GridHadoopJobId(globalId, 1);
-
- grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));
-
- checkStatus(jobId, false);
-
- info("Releasing map latch.");
-
- latch.get("mapAwaitLatch").countDown();
-
- checkStatus(jobId, false);
-
- info("Releasing reduce latch.");
-
- latch.get("reduceAwaitLatch").countDown();
-
- checkStatus(jobId, true);
-
- assertEquals(10, mapExecCnt.get());
- assertEquals(0, combineExecCnt.get());
- assertEquals(1, reduceExecCnt.get());
- }
- finally {
- // Safety.
- latch.get("mapAwaitLatch").countDown();
- latch.get("combineAwaitLatch").countDown();
- latch.get("reduceAwaitLatch").countDown();
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testTaskWithCombinerPerMap() throws Exception {
- try {
- UUID globalId = UUID.randomUUID();
-
- Job job = Job.getInstance();
- setupFileSystems(job.getConfiguration());
-
- job.setMapperClass(TestMapper.class);
- job.setReducerClass(TestReducer.class);
- job.setCombinerClass(TestCombiner.class);
- job.setInputFormatClass(InFormat.class);
-
- FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT + "2"));
-
- GridHadoopJobId jobId = new GridHadoopJobId(globalId, 1);
-
- grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));
-
- checkStatus(jobId, false);
-
- info("Releasing map latch.");
-
- latch.get("mapAwaitLatch").countDown();
-
- checkStatus(jobId, false);
-
- // All maps are completed. We have a combiner, so no reducers should be executed
- // before combiner latch is released.
-
- U.sleep(50);
-
- assertEquals(0, reduceExecCnt.get());
-
- info("Releasing combiner latch.");
-
- latch.get("combineAwaitLatch").countDown();
-
- checkStatus(jobId, false);
-
- info("Releasing reduce latch.");
-
- latch.get("reduceAwaitLatch").countDown();
-
- checkStatus(jobId, true);
-
- assertEquals(10, mapExecCnt.get());
- assertEquals(10, combineExecCnt.get());
- assertEquals(1, reduceExecCnt.get());
- }
- finally {
- // Safety.
- latch.get("mapAwaitLatch").countDown();
- latch.get("combineAwaitLatch").countDown();
- latch.get("reduceAwaitLatch").countDown();
- }
- }
-
- /**
- * Checks job execution status.
- *
- * @param jobId Job ID.
- * @param complete Completion status.
- * @throws Exception If failed.
- */
- private void checkStatus(GridHadoopJobId jobId, boolean complete) throws Exception {
- for (int i = 0; i < gridCount(); i++) {
- IgniteKernal kernal = (IgniteKernal)grid(i);
-
- GridHadoop hadoop = kernal.hadoop();
-
- GridHadoopJobStatus stat = hadoop.status(jobId);
-
- assert stat != null;
-
- IgniteInternalFuture<?> fut = hadoop.finishFuture(jobId);
-
- if (!complete)
- assertFalse(fut.isDone());
- else {
- info("Waiting for status future completion on node [idx=" + i + ", nodeId=" +
- kernal.getLocalNodeId() + ']');
-
- fut.get();
- }
- }
- }
-
- /**
- * Test input format
- */
- public static class InFormat extends InputFormat {
-
- @Override public List<InputSplit> getSplits(JobContext ctx) throws IOException, InterruptedException {
- List<InputSplit> res = new ArrayList<>(BLOCK_CNT);
-
- for (int i = 0; i < BLOCK_CNT; i++)
- try {
- res.add(new FileSplit(new Path(new URI("someFile")), i, i + 1, new String[] {"localhost"}));
- }
- catch (URISyntaxException e) {
- throw new IOException(e);
- }
-
- return res;
- }
-
- @Override public RecordReader createRecordReader(InputSplit split, TaskAttemptContext ctx) throws IOException, InterruptedException {
- return new RecordReader() {
- @Override public void initialize(InputSplit split, TaskAttemptContext ctx) {
- }
-
- @Override public boolean nextKeyValue() {
- return false;
- }
-
- @Override public Object getCurrentKey() {
- return null;
- }
-
- @Override public Object getCurrentValue() {
- return null;
- }
-
- @Override public float getProgress() {
- return 0;
- }
-
- @Override public void close() {
-
- }
- };
- }
- }
-
- /**
- * Test mapper.
- */
- private static class TestMapper extends Mapper {
- @Override public void run(Context ctx) throws IOException, InterruptedException {
- System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId());
-
- latch.get("mapAwaitLatch").await();
-
- mapExecCnt.incrementAndGet();
-
- System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId());
- }
- }
-
- /**
- * Test reducer.
- */
- private static class TestReducer extends Reducer {
- @Override public void run(Context ctx) throws IOException, InterruptedException {
- System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId());
-
- latch.get("reduceAwaitLatch").await();
-
- reduceExecCnt.incrementAndGet();
-
- System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId());
- }
- }
-
- /**
- * Test combiner.
- */
- private static class TestCombiner extends Reducer {
- @Override public void run(Context ctx) throws IOException, InterruptedException {
- System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId());
-
- latch.get("combineAwaitLatch").await();
-
- combineExecCnt.incrementAndGet();
-
- System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReduceEmbeddedSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReduceEmbeddedSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReduceEmbeddedSelfTest.java
deleted file mode 100644
index dda041c..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReduceEmbeddedSelfTest.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.io.serializer.*;
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.hadoop.examples.*;
-
-import java.util.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Tests map-reduce execution with embedded mode.
- */
-public class GridHadoopMapReduceEmbeddedSelfTest extends GridHadoopMapReduceTest {
- /** */
- private static Map<String, Boolean> flags = GridHadoopSharedMap.map(GridHadoopMapReduceEmbeddedSelfTest.class)
- .put("flags", new HashMap<String, Boolean>());
-
- /** {@inheritDoc} */
- @Override public GridHadoopConfiguration hadoopConfiguration(String gridName) {
- GridHadoopConfiguration cfg = super.hadoopConfiguration(gridName);
-
- cfg.setExternalExecution(false);
-
- return cfg;
- }
-
- /**
- * Tests whole job execution with all phases in old and new versions of API with definition of custom
- * Serialization, Partitioner and IO formats.
- * @throws Exception If fails.
- */
- public void testMultiReducerWholeMapReduceExecution() throws Exception {
- IgfsPath inDir = new IgfsPath(PATH_INPUT);
-
- igfs.mkdirs(inDir);
-
- IgfsPath inFile = new IgfsPath(inDir, GridHadoopWordCount2.class.getSimpleName() + "-input");
-
- generateTestFile(inFile.toString(), "key1", 10000, "key2", 20000, "key3", 15000, "key4", 7000, "key5", 12000,
- "key6", 18000 );
-
- for (int i = 0; i < 2; i++) {
- boolean useNewAPI = i == 1;
-
- igfs.delete(new IgfsPath(PATH_OUTPUT), true);
-
- flags.put("serializationWasConfigured", false);
- flags.put("partitionerWasConfigured", false);
- flags.put("inputFormatWasConfigured", false);
- flags.put("outputFormatWasConfigured", false);
-
- JobConf jobConf = new JobConf();
-
- jobConf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, CustomSerialization.class.getName());
-
- //To split into about 6-7 items for v2
- jobConf.setInt(FileInputFormat.SPLIT_MAXSIZE, 65000);
-
- //For v1
- jobConf.setInt("fs.local.block.size", 65000);
-
- // File system coordinates.
- setupFileSystems(jobConf);
-
- GridHadoopWordCount1.setTasksClasses(jobConf, !useNewAPI, !useNewAPI, !useNewAPI);
-
- if (!useNewAPI) {
- jobConf.setPartitionerClass(CustomV1Partitioner.class);
- jobConf.setInputFormat(CustomV1InputFormat.class);
- jobConf.setOutputFormat(CustomV1OutputFormat.class);
- }
-
- Job job = Job.getInstance(jobConf);
-
- GridHadoopWordCount2.setTasksClasses(job, useNewAPI, useNewAPI, useNewAPI);
-
- if (useNewAPI) {
- job.setPartitionerClass(CustomV2Partitioner.class);
- job.setInputFormatClass(CustomV2InputFormat.class);
- job.setOutputFormatClass(CustomV2OutputFormat.class);
- }
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- FileInputFormat.setInputPaths(job, new Path(igfsScheme() + inFile.toString()));
- FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT));
-
- job.setNumReduceTasks(3);
-
- job.setJarByClass(GridHadoopWordCount2.class);
-
- IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 1),
- createJobInfo(job.getConfiguration()));
-
- fut.get();
-
- assertTrue("Serialization was configured (new API is " + useNewAPI + ")",
- flags.get("serializationWasConfigured"));
-
- assertTrue("Partitioner was configured (new API is = " + useNewAPI + ")",
- flags.get("partitionerWasConfigured"));
-
- assertTrue("Input format was configured (new API is = " + useNewAPI + ")",
- flags.get("inputFormatWasConfigured"));
-
- assertTrue("Output format was configured (new API is = " + useNewAPI + ")",
- flags.get("outputFormatWasConfigured"));
-
- assertEquals("Use new API = " + useNewAPI,
- "key3\t15000\n" +
- "key6\t18000\n",
- readAndSortFile(PATH_OUTPUT + "/" + (useNewAPI ? "part-r-" : "part-") + "00000")
- );
-
- assertEquals("Use new API = " + useNewAPI,
- "key1\t10000\n" +
- "key4\t7000\n",
- readAndSortFile(PATH_OUTPUT + "/" + (useNewAPI ? "part-r-" : "part-") + "00001")
- );
-
- assertEquals("Use new API = " + useNewAPI,
- "key2\t20000\n" +
- "key5\t12000\n",
- readAndSortFile(PATH_OUTPUT + "/" + (useNewAPI ? "part-r-" : "part-") + "00002")
- );
-
- }
- }
-
- /**
- * Custom serialization class that inherits behaviour of native {@link WritableSerialization}.
- */
- protected static class CustomSerialization extends WritableSerialization {
- @Override public void setConf(Configuration conf) {
- super.setConf(conf);
-
- flags.put("serializationWasConfigured", true);
- }
- }
-
- /**
- * Custom implementation of Partitioner in v1 API.
- */
- private static class CustomV1Partitioner extends org.apache.hadoop.mapred.lib.HashPartitioner {
- /** {@inheritDoc} */
- @Override public void configure(JobConf job) {
- flags.put("partitionerWasConfigured", true);
- }
- }
-
- /**
- * Custom implementation of Partitioner in v2 API.
- */
- private static class CustomV2Partitioner extends org.apache.hadoop.mapreduce.lib.partition.HashPartitioner
- implements Configurable {
- /** {@inheritDoc} */
- @Override public void setConf(Configuration conf) {
- flags.put("partitionerWasConfigured", true);
- }
-
- /** {@inheritDoc} */
- @Override public Configuration getConf() {
- return null;
- }
- }
-
- /**
- * Custom implementation of InputFormat in v2 API.
- */
- private static class CustomV2InputFormat extends org.apache.hadoop.mapreduce.lib.input.TextInputFormat implements Configurable {
- /** {@inheritDoc} */
- @Override public void setConf(Configuration conf) {
- flags.put("inputFormatWasConfigured", true);
- }
-
- /** {@inheritDoc} */
- @Override public Configuration getConf() {
- return null;
- }
- }
-
- /**
- * Custom implementation of OutputFormat in v2 API.
- */
- private static class CustomV2OutputFormat extends org.apache.hadoop.mapreduce.lib.output.TextOutputFormat implements Configurable {
- /** {@inheritDoc} */
- @Override public void setConf(Configuration conf) {
- flags.put("outputFormatWasConfigured", true);
- }
-
- /** {@inheritDoc} */
- @Override public Configuration getConf() {
- return null;
- }
- }
-
- /**
- * Custom implementation of InputFormat in v1 API.
- */
- private static class CustomV1InputFormat extends org.apache.hadoop.mapred.TextInputFormat {
- /** {@inheritDoc} */
- @Override public void configure(JobConf job) {
- super.configure(job);
-
- flags.put("inputFormatWasConfigured", true);
- }
- }
-
- /**
- * Custom implementation of OutputFormat in v1 API.
- */
- private static class CustomV1OutputFormat extends org.apache.hadoop.mapred.TextOutputFormat implements JobConfigurable {
- /** {@inheritDoc} */
- @Override public void configure(JobConf job) {
- flags.put("outputFormatWasConfigured", true);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReduceTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReduceTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReduceTest.java
deleted file mode 100644
index 072e764..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReduceTest.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.hadoop.counter.*;
-import org.apache.ignite.internal.processors.hadoop.examples.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.testframework.*;
-
-import java.io.*;
-import java.util.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Test of whole cycle of map-reduce processing via Job tracker.
- */
-public class GridHadoopMapReduceTest extends GridHadoopAbstractWordCountTest {
- /** {@inheritDoc} */
- @Override protected int gridCount() {
- return 3;
- }
-
- /**
- * Tests whole job execution with all phases in all combination of new and old versions of API.
- * @throws Exception If fails.
- */
- public void testWholeMapReduceExecution() throws Exception {
- IgfsPath inDir = new IgfsPath(PATH_INPUT);
-
- igfs.mkdirs(inDir);
-
- IgfsPath inFile = new IgfsPath(inDir, GridHadoopWordCount2.class.getSimpleName() + "-input");
-
- generateTestFile(inFile.toString(), "red", 100000, "blue", 200000, "green", 150000, "yellow", 70000 );
-
- for (int i = 0; i < 8; i++) {
- igfs.delete(new IgfsPath(PATH_OUTPUT), true);
-
- boolean useNewMapper = (i & 1) == 0;
- boolean useNewCombiner = (i & 2) == 0;
- boolean useNewReducer = (i & 4) == 0;
-
- JobConf jobConf = new JobConf();
-
- jobConf.set(JOB_COUNTER_WRITER_PROPERTY, GridHadoopFSCounterWriter.class.getName());
- jobConf.setUser("yyy");
- jobConf.set(GridHadoopFSCounterWriter.COUNTER_WRITER_DIR_PROPERTY, "/xxx/${USER}/zzz");
-
- //To split into about 40 items for v2
- jobConf.setInt(FileInputFormat.SPLIT_MAXSIZE, 65000);
-
- //For v1
- jobConf.setInt("fs.local.block.size", 65000);
-
- // File system coordinates.
- setupFileSystems(jobConf);
-
- GridHadoopWordCount1.setTasksClasses(jobConf, !useNewMapper, !useNewCombiner, !useNewReducer);
-
- Job job = Job.getInstance(jobConf);
-
- GridHadoopWordCount2.setTasksClasses(job, useNewMapper, useNewCombiner, useNewReducer);
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- FileInputFormat.setInputPaths(job, new Path(igfsScheme() + inFile.toString()));
- FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT));
-
- job.setJarByClass(GridHadoopWordCount2.class);
-
- GridHadoopJobId jobId = new GridHadoopJobId(UUID.randomUUID(), 1);
-
- IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));
-
- fut.get();
-
- checkJobStatistics(jobId);
-
- assertEquals("Use new mapper: " + useNewMapper + ", new combiner: " + useNewCombiner + ", new reducer: " +
- useNewReducer,
- "blue\t200000\n" +
- "green\t150000\n" +
- "red\t100000\n" +
- "yellow\t70000\n",
- readAndSortFile(PATH_OUTPUT + "/" + (useNewReducer ? "part-r-" : "part-") + "00000")
- );
- }
- }
-
- /**
- * Simple test job statistics.
- *
- * @param jobId Job id.
- * @throws IgniteCheckedException
- */
- private void checkJobStatistics(GridHadoopJobId jobId) throws IgniteCheckedException, IOException {
- GridHadoopCounters cntrs = grid(0).hadoop().counters(jobId);
-
- GridHadoopPerformanceCounter perfCntr = GridHadoopPerformanceCounter.getCounter(cntrs, null);
-
- Map<String, SortedMap<Integer,Long>> tasks = new TreeMap<>();
-
- Map<String, Integer> phaseOrders = new HashMap<>();
- phaseOrders.put("submit", 0);
- phaseOrders.put("prepare", 1);
- phaseOrders.put("start", 2);
- phaseOrders.put("Cstart", 3);
- phaseOrders.put("finish", 4);
-
- String prevTaskId = null;
-
- long apiEvtCnt = 0;
-
- for (T2<String, Long> evt : perfCntr.evts()) {
- //We expect string pattern: COMBINE 1 run 7fa86a14-5a08-40e3-a7cb-98109b52a706
- String[] parsedEvt = evt.get1().split(" ");
-
- String taskId;
- String taskPhase;
-
- if ("JOB".equals(parsedEvt[0])) {
- taskId = parsedEvt[0];
- taskPhase = parsedEvt[1];
- }
- else {
- taskId = ("COMBINE".equals(parsedEvt[0]) ? "MAP" : parsedEvt[0].substring(0, 3)) + parsedEvt[1];
- taskPhase = ("COMBINE".equals(parsedEvt[0]) ? "C" : "") + parsedEvt[2];
- }
-
- if (!taskId.equals(prevTaskId))
- tasks.put(taskId, new TreeMap<Integer,Long>());
-
- Integer pos = phaseOrders.get(taskPhase);
-
- assertNotNull("Invalid phase " + taskPhase, pos);
-
- tasks.get(taskId).put(pos, evt.get2());
-
- prevTaskId = taskId;
-
- apiEvtCnt++;
- }
-
- for (Map.Entry<String ,SortedMap<Integer,Long>> task : tasks.entrySet()) {
- Map<Integer, Long> order = task.getValue();
-
- long prev = 0;
-
- for (Map.Entry<Integer, Long> phase : order.entrySet()) {
- assertTrue("Phase order of " + task.getKey() + " is invalid", phase.getValue() >= prev);
-
- prev = phase.getValue();
- }
- }
-
- final IgfsPath statPath = new IgfsPath("/xxx/yyy/zzz/" + jobId + "/performance");
-
- GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- return igfs.exists(statPath);
- }
- }, 10000);
-
- BufferedReader reader = new BufferedReader(new InputStreamReader(igfs.open(statPath)));
-
- assertEquals(apiEvtCnt, GridHadoopTestUtils.simpleCheckJobStatFile(reader));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopPopularWordsTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopPopularWordsTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopPopularWordsTest.java
deleted file mode 100644
index 3e8a95a..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopPopularWordsTest.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import com.google.common.collect.*;
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.hadoop.mapreduce.lib.output.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.Map.*;
-
-import static com.google.common.collect.Maps.*;
-import static com.google.common.collect.MinMaxPriorityQueue.*;
-import static java.util.Collections.*;
-
-/**
- * Hadoop-based 10 popular words example: all files in a given directory are tokenized and for each word longer than
- * 3 characters the number of occurrences ins calculated. Finally, 10 words with the highest occurrence count are
- * output.
- *
- * NOTE: in order to run this example on Windows please ensure that cygwin is installed and available in the system
- * path.
- */
-public class GridHadoopPopularWordsTest {
- /** Ignite home. */
- private static final String IGNITE_HOME = U.getIgniteHome();
-
- /** The path to the input directory. ALl files in that directory will be processed. */
- private static final Path BOOKS_LOCAL_DIR =
- new Path("file:" + IGNITE_HOME, "modules/tests/java/org/apache/ignite/grid/hadoop/books");
-
- /** The path to the output directory. THe result file will be written to this location. */
- private static final Path RESULT_LOCAL_DIR =
- new Path("file:" + IGNITE_HOME, "modules/tests/java/org/apache/ignite/grid/hadoop/output");
-
- /** Popular books source dir in DFS. */
- private static final Path BOOKS_DFS_DIR = new Path("tmp/word-count-example/in");
-
- /** Popular books source dir in DFS. */
- private static final Path RESULT_DFS_DIR = new Path("tmp/word-count-example/out");
-
- /** Path to the distributed file system configuration. */
- private static final String DFS_CFG = "examples/config/filesystem/core-site.xml";
-
- /** Top N words to select **/
- private static final int POPULAR_WORDS_CNT = 10;
-
- /**
- * For each token in the input string the mapper emits a {word, 1} pair.
- */
- private static class TokenizingMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
- /** Constant value. */
- private static final IntWritable ONE = new IntWritable(1);
-
- /** The word converted into the Text. */
- private Text word = new Text();
-
- /**
- * Emits a entry where the key is the word and the value is always 1.
- *
- * @param key the current position in the input file (not used here)
- * @param val the text string
- * @param ctx mapper context
- * @throws IOException
- * @throws InterruptedException
- */
- @Override protected void map(LongWritable key, Text val, Context ctx)
- throws IOException, InterruptedException {
- // Get the mapped object.
- final String line = val.toString();
-
- // Splits the given string to words.
- final String[] words = line.split("[^a-zA-Z0-9]");
-
- for (final String w : words) {
- // Only emit counts for longer words.
- if (w.length() <= 3)
- continue;
-
- word.set(w);
-
- // Write the word into the context with the initial count equals 1.
- ctx.write(word, ONE);
- }
- }
- }
-
- /**
- * The reducer uses a priority queue to rank the words based on its number of occurrences.
- */
- private static class TopNWordsReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
- private MinMaxPriorityQueue<Entry<Integer, String>> q;
-
- TopNWordsReducer() {
- q = orderedBy(reverseOrder(new Comparator<Entry<Integer, String>>() {
- @Override public int compare(Entry<Integer, String> o1, Entry<Integer, String> o2) {
- return o1.getKey().compareTo(o2.getKey());
- }
- })).expectedSize(POPULAR_WORDS_CNT).maximumSize(POPULAR_WORDS_CNT).create();
- }
-
- /**
- * This method doesn't emit anything, but just keeps track of the top N words.
- *
- * @param key The word.
- * @param vals The words counts.
- * @param ctx Reducer context.
- * @throws IOException If failed.
- * @throws InterruptedException If failed.
- */
- @Override public void reduce(Text key, Iterable<IntWritable> vals, Context ctx) throws IOException,
- InterruptedException {
- int sum = 0;
-
- for (IntWritable val : vals)
- sum += val.get();
-
- q.add(immutableEntry(sum, key.toString()));
- }
-
- /**
- * This method is called after all the word entries have been processed. It writes the accumulated
- * statistics to the job output file.
- *
- * @param ctx The job context.
- * @throws IOException If failed.
- * @throws InterruptedException If failed.
- */
- @Override protected void cleanup(Context ctx) throws IOException, InterruptedException {
- IntWritable i = new IntWritable();
-
- Text txt = new Text();
-
- // iterate in desc order
- while (!q.isEmpty()) {
- Entry<Integer, String> e = q.removeFirst();
-
- i.set(e.getKey());
-
- txt.set(e.getValue());
-
- ctx.write(txt, i);
- }
- }
- }
-
- /**
- * Configures the Hadoop MapReduce job.
- *
- * @return Instance of the Hadoop MapRed job.
- * @throws IOException If failed.
- */
- private Job createConfigBasedHadoopJob() throws IOException {
- Job jobCfg = new Job();
-
- Configuration cfg = jobCfg.getConfiguration();
-
- // Use explicit configuration of distributed file system, if provided.
- if (DFS_CFG != null)
- cfg.addResource(U.resolveIgniteUrl(DFS_CFG));
-
- jobCfg.setJobName("HadoopPopularWordExample");
- jobCfg.setJarByClass(GridHadoopPopularWordsTest.class);
- jobCfg.setInputFormatClass(TextInputFormat.class);
- jobCfg.setOutputKeyClass(Text.class);
- jobCfg.setOutputValueClass(IntWritable.class);
- jobCfg.setMapperClass(TokenizingMapper.class);
- jobCfg.setReducerClass(TopNWordsReducer.class);
-
- FileInputFormat.setInputPaths(jobCfg, BOOKS_DFS_DIR);
- FileOutputFormat.setOutputPath(jobCfg, RESULT_DFS_DIR);
-
- // Local job tracker allows the only task per wave, but text input format
- // replaces it with the calculated value based on input split size option.
- if ("local".equals(cfg.get("mapred.job.tracker", "local"))) {
- // Split job into tasks using 32MB split size.
- FileInputFormat.setMinInputSplitSize(jobCfg, 32 * 1024 * 1024);
- FileInputFormat.setMaxInputSplitSize(jobCfg, Long.MAX_VALUE);
- }
-
- return jobCfg;
- }
-
- /**
- * Runs the Hadoop job.
- *
- * @return {@code True} if succeeded, {@code false} otherwise.
- * @throws Exception If failed.
- */
- private boolean runWordCountConfigBasedHadoopJob() throws Exception {
- Job job = createConfigBasedHadoopJob();
-
- // Distributed file system this job will work with.
- FileSystem fs = FileSystem.get(job.getConfiguration());
-
- X.println(">>> Using distributed file system: " + fs.getHomeDirectory());
-
- // Prepare input and output job directories.
- prepareDirectories(fs);
-
- long time = System.currentTimeMillis();
-
- // Run job.
- boolean res = job.waitForCompletion(true);
-
- X.println(">>> Job execution time: " + (System.currentTimeMillis() - time) / 1000 + " sec.");
-
- // Move job results into local file system, so you can view calculated results.
- publishResults(fs);
-
- return res;
- }
-
- /**
- * Prepare job's data: cleanup result directories that might have left over
- * after previous runs, copy input files from the local file system into DFS.
- *
- * @param fs Distributed file system to use in job.
- * @throws IOException If failed.
- */
- private void prepareDirectories(FileSystem fs) throws IOException {
- X.println(">>> Cleaning up DFS result directory: " + RESULT_DFS_DIR);
-
- fs.delete(RESULT_DFS_DIR, true);
-
- X.println(">>> Cleaning up DFS input directory: " + BOOKS_DFS_DIR);
-
- fs.delete(BOOKS_DFS_DIR, true);
-
- X.println(">>> Copy local files into DFS input directory: " + BOOKS_DFS_DIR);
-
- fs.copyFromLocalFile(BOOKS_LOCAL_DIR, BOOKS_DFS_DIR);
- }
-
- /**
- * Publish job execution results into local file system, so you can view them.
- *
- * @param fs Distributed file sytem used in job.
- * @throws IOException If failed.
- */
- private void publishResults(FileSystem fs) throws IOException {
- X.println(">>> Cleaning up DFS input directory: " + BOOKS_DFS_DIR);
-
- fs.delete(BOOKS_DFS_DIR, true);
-
- X.println(">>> Cleaning up LOCAL result directory: " + RESULT_LOCAL_DIR);
-
- fs.delete(RESULT_LOCAL_DIR, true);
-
- X.println(">>> Moving job results into LOCAL result directory: " + RESULT_LOCAL_DIR);
-
- fs.copyToLocalFile(true, RESULT_DFS_DIR, RESULT_LOCAL_DIR);
- }
-
- /**
- * Executes a modified version of the Hadoop word count example. Here, in addition to counting the number of
- * occurrences of the word in the source files, the N most popular words are selected.
- *
- * @param args None.
- */
- public static void main(String[] args) {
- try {
- new GridHadoopPopularWordsTest().runWordCountConfigBasedHadoopJob();
- }
- catch (Exception e) {
- X.println(">>> Failed to run word count example: " + e.getMessage());
- }
-
- System.exit(0);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSerializationWrapperSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSerializationWrapperSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSerializationWrapperSelfTest.java
deleted file mode 100644
index 79b9965..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSerializationWrapperSelfTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.io.serializer.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-import org.apache.ignite.testframework.junits.common.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Test of wrapper of the native serialization.
- */
-public class GridHadoopSerializationWrapperSelfTest extends GridCommonAbstractTest {
- /**
- * Tests read/write of IntWritable via native WritableSerialization.
- * @throws Exception If fails.
- */
- public void testIntWritableSerialization() throws Exception {
- GridHadoopSerialization ser = new GridHadoopSerializationWrapper(new WritableSerialization(), IntWritable.class);
-
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
-
- DataOutput out = new DataOutputStream(buf);
-
- ser.write(out, new IntWritable(3));
- ser.write(out, new IntWritable(-5));
-
- assertEquals("[0, 0, 0, 3, -1, -1, -1, -5]", Arrays.toString(buf.toByteArray()));
-
- DataInput in = new DataInputStream(new ByteArrayInputStream(buf.toByteArray()));
-
- assertEquals(3, ((IntWritable)ser.read(in, null)).get());
- assertEquals(-5, ((IntWritable)ser.read(in, null)).get());
- }
-
- /**
- * Tests read/write of Integer via native JavaleSerialization.
- * @throws Exception If fails.
- */
- public void testIntJavaSerialization() throws Exception {
- GridHadoopSerialization ser = new GridHadoopSerializationWrapper(new JavaSerialization(), Integer.class);
-
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
-
- DataOutput out = new DataOutputStream(buf);
-
- ser.write(out, 3);
- ser.write(out, -5);
- ser.close();
-
- DataInput in = new DataInputStream(new ByteArrayInputStream(buf.toByteArray()));
-
- assertEquals(3, ((Integer)ser.read(in, null)).intValue());
- assertEquals(-5, ((Integer)ser.read(in, null)).intValue());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSharedMap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSharedMap.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSharedMap.java
deleted file mode 100644
index 689fb58..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSharedMap.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.jdk8.backport.*;
-
-import java.util.concurrent.*;
-
-/**
- * For tests.
- */
-public class GridHadoopSharedMap {
- /** */
- private static final ConcurrentMap<String, GridHadoopSharedMap> maps = new ConcurrentHashMap8<>();
-
- /** */
- private final ConcurrentMap<String, Object> map = new ConcurrentHashMap8<>();
-
- /**
- * Private.
- */
- private GridHadoopSharedMap() {
- // No-op.
- }
-
- /**
- * Puts object by key.
- *
- * @param key Key.
- * @param val Value.
- */
- public <T> T put(String key, T val) {
- Object old = map.putIfAbsent(key, val);
-
- return old == null ? val : (T)old;
- }
-
- /**
- * @param cls Class.
- * @return Map of static fields.
- */
- public static GridHadoopSharedMap map(Class<?> cls) {
- GridHadoopSharedMap m = maps.get(cls.getName());
-
- if (m != null)
- return m;
-
- GridHadoopSharedMap old = maps.putIfAbsent(cls.getName(), m = new GridHadoopSharedMap());
-
- return old == null ? m : old;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSortingExternalTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSortingExternalTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSortingExternalTest.java
deleted file mode 100644
index 23884ef..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSortingExternalTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-/**
- * External test for sorting.
- */
-public class GridHadoopSortingExternalTest extends GridHadoopSortingTest {
- /** {@inheritDoc} */
- @Override public GridHadoopConfiguration hadoopConfiguration(String gridName) {
- GridHadoopConfiguration cfg = super.hadoopConfiguration(gridName);
-
- cfg.setExternalExecution(true);
-
- return cfg;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSortingTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSortingTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSortingTest.java
deleted file mode 100644
index 3a2c397..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSortingTest.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.io.serializer.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.hadoop.mapreduce.lib.output.*;
-import org.apache.ignite.internal.util.typedef.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Tests correct sorting.
- */
-public class GridHadoopSortingTest extends GridHadoopAbstractSelfTest {
- /** */
- private static final String PATH_INPUT = "/test-in";
-
- /** */
- private static final String PATH_OUTPUT = "/test-out";
-
- /** {@inheritDoc} */
- @Override protected int gridCount() {
- return 3;
- }
-
- /**
- * @return {@code True} if IGFS is enabled on Hadoop nodes.
- */
- @Override protected boolean igfsEnabled() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- startGrids(gridCount());
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids(true);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopConfiguration hadoopConfiguration(String gridName) {
- GridHadoopConfiguration cfg = super.hadoopConfiguration(gridName);
-
- cfg.setExternalExecution(false);
-
- return cfg;
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testSortSimple() throws Exception {
- // Generate test data.
- Job job = Job.getInstance();
-
- job.setInputFormatClass(InFormat.class);
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(NullWritable.class);
-
- job.setMapperClass(Mapper.class);
- job.setNumReduceTasks(0);
-
- setupFileSystems(job.getConfiguration());
-
- FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_INPUT));
-
- X.printerrln("Data generation started.");
-
- grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 1),
- createJobInfo(job.getConfiguration())).get(180000);
-
- X.printerrln("Data generation complete.");
-
- // Run main map-reduce job.
- job = Job.getInstance();
-
- setupFileSystems(job.getConfiguration());
-
- job.getConfiguration().set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, JavaSerialization.class.getName() +
- "," + WritableSerialization.class.getName());
-
- FileInputFormat.setInputPaths(job, new Path(igfsScheme() + PATH_INPUT));
- FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT));
-
- job.setSortComparatorClass(JavaSerializationComparator.class);
-
- job.setMapperClass(MyMapper.class);
- job.setReducerClass(MyReducer.class);
-
- job.setNumReduceTasks(2);
-
- job.setMapOutputKeyClass(UUID.class);
- job.setMapOutputValueClass(NullWritable.class);
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(NullWritable.class);
-
- X.printerrln("Job started.");
-
- grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 2),
- createJobInfo(job.getConfiguration())).get(180000);
-
- X.printerrln("Job complete.");
-
- // Check result.
- Path outDir = new Path(igfsScheme() + PATH_OUTPUT);
-
- AbstractFileSystem fs = AbstractFileSystem.get(new URI(igfsScheme()), job.getConfiguration());
-
- for (FileStatus file : fs.listStatus(outDir)) {
- X.printerrln("__ file: " + file);
-
- if (file.getLen() == 0)
- continue;
-
- FSDataInputStream in = fs.open(file.getPath());
-
- Scanner sc = new Scanner(in);
-
- UUID prev = null;
-
- while(sc.hasNextLine()) {
- UUID next = UUID.fromString(sc.nextLine());
-
-// X.printerrln("___ check: " + next);
-
- if (prev != null)
- assertTrue(prev.compareTo(next) < 0);
-
- prev = next;
- }
- }
- }
-
- public static class InFormat extends InputFormat<Text, NullWritable> {
- /** {@inheritDoc} */
- @Override public List<InputSplit> getSplits(JobContext ctx) throws IOException, InterruptedException {
- List<InputSplit> res = new ArrayList<>();
-
- FakeSplit split = new FakeSplit(20);
-
- for (int i = 0; i < 10; i++)
- res.add(split);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public RecordReader<Text, NullWritable> createRecordReader(final InputSplit split,
- TaskAttemptContext ctx) throws IOException, InterruptedException {
- return new RecordReader<Text, NullWritable>() {
- /** */
- int cnt;
-
- /** */
- Text txt = new Text();
-
- @Override public void initialize(InputSplit split, TaskAttemptContext ctx) {
- // No-op.
- }
-
- @Override public boolean nextKeyValue() throws IOException, InterruptedException {
- return ++cnt <= split.getLength();
- }
-
- @Override public Text getCurrentKey() {
- txt.set(UUID.randomUUID().toString());
-
-// X.printerrln("___ read: " + txt);
-
- return txt;
- }
-
- @Override public NullWritable getCurrentValue() {
- return NullWritable.get();
- }
-
- @Override public float getProgress() throws IOException, InterruptedException {
- return (float)cnt / split.getLength();
- }
-
- @Override public void close() {
- // No-op.
- }
- };
- }
- }
-
- public static class MyMapper extends Mapper<LongWritable, Text, UUID, NullWritable> {
- /** {@inheritDoc} */
- @Override protected void map(LongWritable key, Text val, Context ctx) throws IOException, InterruptedException {
-// X.printerrln("___ map: " + val);
-
- ctx.write(UUID.fromString(val.toString()), NullWritable.get());
- }
- }
-
- public static class MyReducer extends Reducer<UUID, NullWritable, Text, NullWritable> {
- /** */
- private Text text = new Text();
-
- /** {@inheritDoc} */
- @Override protected void reduce(UUID key, Iterable<NullWritable> vals, Context ctx)
- throws IOException, InterruptedException {
-// X.printerrln("___ rdc: " + key);
-
- text.set(key.toString());
-
- ctx.write(text, NullWritable.get());
- }
- }
-
- public static class FakeSplit extends InputSplit implements Writable {
- /** */
- private static final String[] HOSTS = {"127.0.0.1"};
-
- /** */
- private int len;
-
- /**
- * @param len Length.
- */
- public FakeSplit(int len) {
- this.len = len;
- }
-
- /**
- *
- */
- public FakeSplit() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public long getLength() throws IOException, InterruptedException {
- return len;
- }
-
- /** {@inheritDoc} */
- @Override public String[] getLocations() throws IOException, InterruptedException {
- return HOSTS;
- }
-
- /** {@inheritDoc} */
- @Override public void write(DataOutput out) throws IOException {
- out.writeInt(len);
- }
-
- /** {@inheritDoc} */
- @Override public void readFields(DataInput in) throws IOException {
- len = in.readInt();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSplitWrapperSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSplitWrapperSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSplitWrapperSelfTest.java
deleted file mode 100644
index 0b15a2c..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSplitWrapperSelfTest.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-import org.apache.ignite.testframework.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-/**
- * Self test of {@link GridHadoopSplitWrapper}.
- */
-public class GridHadoopSplitWrapperSelfTest extends GridHadoopAbstractSelfTest {
- /**
- * Tests serialization of wrapper and the wrapped native split.
- * @throws Exception If fails.
- */
- public void testSerialization() throws Exception {
- FileSplit nativeSplit = new FileSplit(new Path("/path/to/file"), 100, 500, new String[]{"host1", "host2"});
-
- assertEquals("/path/to/file:100+500", nativeSplit.toString());
-
- GridHadoopSplitWrapper split = GridHadoopUtils.wrapSplit(10, nativeSplit, nativeSplit.getLocations());
-
- assertEquals("[host1, host2]", Arrays.toString(split.hosts()));
-
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
-
- ObjectOutput out = new ObjectOutputStream(buf);
-
- out.writeObject(split);
-
- ObjectInput in = new ObjectInputStream(new ByteArrayInputStream(buf.toByteArray()));
-
- final GridHadoopSplitWrapper res = (GridHadoopSplitWrapper)in.readObject();
-
- assertEquals("/path/to/file:100+500", GridHadoopUtils.unwrapSplit(res).toString());
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- res.hosts();
-
- return null;
- }
- }, AssertionError.class, null);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopStartup.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopStartup.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopStartup.java
deleted file mode 100644
index 6cc7635..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopStartup.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.conf.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem;
-import org.apache.ignite.internal.util.typedef.*;
-
-/**
- * Hadoop node startup.
- */
-public class GridHadoopStartup {
- /**
- * @param args Arguments.
- */
- public static void main(String[] args) {
- G.start("config/hadoop/default-config.xml");
- }
-
- /**
- * @return Configuration for job run.
- */
- @SuppressWarnings("UnnecessaryFullyQualifiedName")
- public static Configuration configuration() {
- Configuration cfg = new Configuration();
-
- cfg.set("fs.defaultFS", "igfs://igfs@localhost");
-
- cfg.set("fs.igfs.impl", org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem.class.getName());
- cfg.set("fs.AbstractFileSystem.igfs.impl", IgfsHadoopFileSystem.class.getName());
-
- cfg.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
-
- cfg.set("mapreduce.framework.name", "ignite");
- cfg.set("mapreduce.jobtracker.address", "localhost:11211");
-
- return cfg;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskExecutionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskExecutionSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskExecutionSelfTest.java
deleted file mode 100644
index 40546bb..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskExecutionSelfTest.java
+++ /dev/null
@@ -1,551 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.hadoop.mapreduce.lib.output.*;
-import org.apache.ignite.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.igfs.hadoop.v1.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.testframework.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Tests map-reduce task execution basics.
- */
-public class GridHadoopTaskExecutionSelfTest extends GridHadoopAbstractSelfTest {
- /** */
- private static GridHadoopSharedMap m = GridHadoopSharedMap.map(GridHadoopTaskExecutionSelfTest.class);
-
- /** Line count. */
- private static final AtomicInteger totalLineCnt = m.put("totalLineCnt", new AtomicInteger());
-
- /** Executed tasks. */
- private static final AtomicInteger executedTasks = m.put("executedTasks", new AtomicInteger());
-
- /** Cancelled tasks. */
- private static final AtomicInteger cancelledTasks = m.put("cancelledTasks", new AtomicInteger());
-
- /** Working directory of each task. */
- private static final Map<String, String> taskWorkDirs = m.put("taskWorkDirs",
- new ConcurrentHashMap<String, String>());
-
- /** Mapper id to fail. */
- private static final AtomicInteger failMapperId = m.put("failMapperId", new AtomicInteger());
-
- /** Number of splits of the current input. */
- private static final AtomicInteger splitsCount = m.put("splitsCount", new AtomicInteger());
-
- /** Test param. */
- private static final String MAP_WRITE = "test.map.write";
-
-
- /** {@inheritDoc} */
- @Override public IgfsConfiguration igfsConfiguration() {
- IgfsConfiguration cfg = super.igfsConfiguration();
-
- cfg.setFragmentizerEnabled(false);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected boolean igfsEnabled() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
- startGrids(gridCount());
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- stopAllGrids();
-
- super.afterTestsStopped();
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- grid(0).fileSystem(igfsName).format();
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopConfiguration hadoopConfiguration(String gridName) {
- GridHadoopConfiguration cfg = super.hadoopConfiguration(gridName);
-
- cfg.setMaxParallelTasks(5);
- cfg.setExternalExecution(false);
-
- return cfg;
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testMapRun() throws Exception {
- int lineCnt = 10000;
- String fileName = "/testFile";
-
- prepareFile(fileName, lineCnt);
-
- totalLineCnt.set(0);
- taskWorkDirs.clear();
-
- Configuration cfg = new Configuration();
-
- cfg.setStrings("fs.igfs.impl", IgfsHadoopFileSystem.class.getName());
-
- Job job = Job.getInstance(cfg);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- job.setMapperClass(TestMapper.class);
-
- job.setNumReduceTasks(0);
-
- job.setInputFormatClass(TextInputFormat.class);
-
- FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
- FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output/"));
-
- job.setJarByClass(getClass());
-
- IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 1),
- createJobInfo(job.getConfiguration()));
-
- fut.get();
-
- assertEquals(lineCnt, totalLineCnt.get());
-
- assertEquals(32, taskWorkDirs.size());
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testMapCombineRun() throws Exception {
- int lineCnt = 10001;
- String fileName = "/testFile";
-
- prepareFile(fileName, lineCnt);
-
- totalLineCnt.set(0);
- taskWorkDirs.clear();
-
- Configuration cfg = new Configuration();
-
- cfg.setStrings("fs.igfs.impl", IgfsHadoopFileSystem.class.getName());
- cfg.setBoolean(MAP_WRITE, true);
-
- Job job = Job.getInstance(cfg);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- job.setMapperClass(TestMapper.class);
- job.setCombinerClass(TestCombiner.class);
- job.setReducerClass(TestReducer.class);
-
- job.setNumReduceTasks(2);
-
- job.setInputFormatClass(TextInputFormat.class);
-
- FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
- FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output"));
-
- job.setJarByClass(getClass());
-
- GridHadoopJobId jobId = new GridHadoopJobId(UUID.randomUUID(), 2);
-
- IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));
-
- fut.get();
-
- assertEquals(lineCnt, totalLineCnt.get());
-
- assertEquals(34, taskWorkDirs.size());
-
- for (int g = 0; g < gridCount(); g++)
- grid(g).hadoop().finishFuture(jobId).get();
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testMapperException() throws Exception {
- prepareFile("/testFile", 1000);
-
- Configuration cfg = new Configuration();
-
- cfg.setStrings("fs.igfs.impl", IgfsHadoopFileSystem.class.getName());
-
- Job job = Job.getInstance(cfg);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- job.setMapperClass(FailMapper.class);
-
- job.setNumReduceTasks(0);
-
- job.setInputFormatClass(TextInputFormat.class);
-
- FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
- FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output/"));
-
- job.setJarByClass(getClass());
-
- final IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 3),
- createJobInfo(job.getConfiguration()));
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fut.get();
-
- return null;
- }
- }, IgniteCheckedException.class, null);
- }
-
- /**
- * @param fileName File name.
- * @param lineCnt Line count.
- * @throws Exception If failed.
- */
- private void prepareFile(String fileName, int lineCnt) throws Exception {
- IgniteFs igfs = grid(0).fileSystem(igfsName);
-
- try (OutputStream os = igfs.create(new IgfsPath(fileName), true)) {
- PrintWriter w = new PrintWriter(new OutputStreamWriter(os));
-
- for (int i = 0; i < lineCnt; i++)
- w.print("Hello, Hadoop map-reduce!\n");
-
- w.flush();
- }
- }
-
- /**
- * Prepare job with mappers to cancel.
- * @return Fully configured job.
- * @throws Exception If fails.
- */
- private Configuration prepareJobForCancelling() throws Exception {
- prepareFile("/testFile", 1500);
-
- executedTasks.set(0);
- cancelledTasks.set(0);
- failMapperId.set(0);
- splitsCount.set(0);
-
- Configuration cfg = new Configuration();
-
- setupFileSystems(cfg);
-
- Job job = Job.getInstance(cfg);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- job.setMapperClass(CancellingTestMapper.class);
-
- job.setNumReduceTasks(0);
-
- job.setInputFormatClass(InFormat.class);
-
- FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
- FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output/"));
-
- job.setJarByClass(getClass());
-
- return job.getConfiguration();
- }
-
- /**
- * Test input format.
- */
- private static class InFormat extends TextInputFormat {
- @Override public List<InputSplit> getSplits(JobContext ctx) throws IOException {
- List<InputSplit> res = super.getSplits(ctx);
-
- splitsCount.set(res.size());
-
- X.println("___ split of input: " + splitsCount.get());
-
- return res;
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testTaskCancelling() throws Exception {
- Configuration cfg = prepareJobForCancelling();
-
- GridHadoopJobId jobId = new GridHadoopJobId(UUID.randomUUID(), 1);
-
- final IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(cfg));
-
- if (!GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- return splitsCount.get() > 0;
- }
- }, 20000)) {
- U.dumpThreads(log);
-
- assertTrue(false);
- }
-
- if (!GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- return executedTasks.get() == splitsCount.get();
- }
- }, 20000)) {
- U.dumpThreads(log);
-
- assertTrue(false);
- }
-
- // Fail mapper with id "1", cancels others
- failMapperId.set(1);
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fut.get();
-
- return null;
- }
- }, IgniteCheckedException.class, null);
-
- assertEquals(executedTasks.get(), cancelledTasks.get() + 1);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testJobKill() throws Exception {
- Configuration cfg = prepareJobForCancelling();
-
- GridHadoop hadoop = grid(0).hadoop();
-
- GridHadoopJobId jobId = new GridHadoopJobId(UUID.randomUUID(), 1);
-
- //Kill unknown job.
- boolean killRes = hadoop.kill(jobId);
-
- assertFalse(killRes);
-
- final IgniteInternalFuture<?> fut = hadoop.submit(jobId, createJobInfo(cfg));
-
- if (!GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- return splitsCount.get() > 0;
- }
- }, 20000)) {
- U.dumpThreads(log);
-
- assertTrue(false);
- }
-
- if (!GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- X.println("___ executed tasks: " + executedTasks.get());
-
- return executedTasks.get() == splitsCount.get();
- }
- }, 20000)) {
- U.dumpThreads(log);
-
- fail();
- }
-
- //Kill really ran job.
- killRes = hadoop.kill(jobId);
-
- assertTrue(killRes);
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- fut.get();
-
- return null;
- }
- }, IgniteCheckedException.class, null);
-
- assertEquals(executedTasks.get(), cancelledTasks.get());
-
- //Kill the same job again.
- killRes = hadoop.kill(jobId);
-
- assertTrue(killRes);
- }
-
- private static class CancellingTestMapper extends Mapper<Object, Text, Text, IntWritable> {
- private int mapperId;
-
- /** {@inheritDoc} */
- @Override protected void setup(Context ctx) throws IOException, InterruptedException {
- mapperId = executedTasks.incrementAndGet();
- }
-
- /** {@inheritDoc} */
- @Override public void run(Context ctx) throws IOException, InterruptedException {
- try {
- super.run(ctx);
- }
- catch (GridHadoopTaskCancelledException e) {
- cancelledTasks.incrementAndGet();
-
- throw e;
- }
- }
-
- /** {@inheritDoc} */
- @Override protected void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
- if (mapperId == failMapperId.get())
- throw new IOException();
-
- Thread.sleep(1000);
- }
- }
-
- /**
- * Test failing mapper.
- */
- private static class FailMapper extends Mapper<Object, Text, Text, IntWritable> {
- /** {@inheritDoc} */
- @Override protected void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
- throw new IOException("Expected");
- }
- }
-
- /**
- * Mapper calculates number of lines.
- */
- private static class TestMapper extends Mapper<Object, Text, Text, IntWritable> {
- /** Writable integer constant of '1'. */
- private static final IntWritable ONE = new IntWritable(1);
-
- /** Line count constant. */
- public static final Text LINE_COUNT = new Text("lineCount");
-
- /** {@inheritDoc} */
- @Override protected void setup(Context ctx) throws IOException, InterruptedException {
- X.println("___ Mapper: " + ctx.getTaskAttemptID());
-
- String taskId = ctx.getTaskAttemptID().toString();
-
- LocalFileSystem locFs = FileSystem.getLocal(ctx.getConfiguration());
-
- String workDir = locFs.getWorkingDirectory().toString();
-
- assertNull(taskWorkDirs.put(workDir, taskId));
- }
-
- /** {@inheritDoc} */
- @Override protected void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
- if (ctx.getConfiguration().getBoolean(MAP_WRITE, false))
- ctx.write(LINE_COUNT, ONE);
- else
- totalLineCnt.incrementAndGet();
- }
- }
-
- /**
- * Combiner calculates number of lines.
- */
- private static class TestCombiner extends Reducer<Text, IntWritable, Text, IntWritable> {
- /** */
- IntWritable sum = new IntWritable();
-
- /** {@inheritDoc} */
- @Override protected void setup(Context ctx) throws IOException, InterruptedException {
- X.println("___ Combiner: ");
- }
-
- /** {@inheritDoc} */
- @Override protected void reduce(Text key, Iterable<IntWritable> values, Context ctx) throws IOException,
- InterruptedException {
- int lineCnt = 0;
-
- for (IntWritable value : values)
- lineCnt += value.get();
-
- sum.set(lineCnt);
-
- X.println("___ combo: " + lineCnt);
-
- ctx.write(key, sum);
- }
- }
-
- /**
- * Combiner calculates number of lines.
- */
- private static class TestReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
- /** */
- IntWritable sum = new IntWritable();
-
- /** {@inheritDoc} */
- @Override protected void setup(Context ctx) throws IOException, InterruptedException {
- X.println("___ Reducer: " + ctx.getTaskAttemptID());
-
- String taskId = ctx.getTaskAttemptID().toString();
- String workDir = FileSystem.getLocal(ctx.getConfiguration()).getWorkingDirectory().toString();
-
- assertNull(taskWorkDirs.put(workDir, taskId));
- }
-
- /** {@inheritDoc} */
- @Override protected void reduce(Text key, Iterable<IntWritable> values, Context ctx) throws IOException,
- InterruptedException {
- int lineCnt = 0;
-
- for (IntWritable value : values) {
- lineCnt += value.get();
-
- X.println("___ rdcr: " + value.get());
- }
-
- sum.set(lineCnt);
-
- ctx.write(key, sum);
-
- X.println("___ RDCR SUM: " + lineCnt);
-
- totalLineCnt.addAndGet(lineCnt);
- }
- }
-}
[02/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopValidationSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopValidationSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopValidationSelfTest.java
new file mode 100644
index 0000000..558dec5
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopValidationSelfTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.configuration.*;
+
+/**
+ * Configuration validation tests.
+ */
+public class HadoopValidationSelfTest extends HadoopAbstractSelfTest {
+ /** Peer class loading enabled flag. */
+ public boolean peerClassLoading;
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids(true);
+
+ peerClassLoading = false;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setPeerClassLoadingEnabled(peerClassLoading);
+
+ return cfg;
+ }
+
+ /**
+ * Ensure that Grid starts when all configuration parameters are valid.
+ *
+ * @throws Exception If failed.
+ */
+ public void testValid() throws Exception {
+ startGrids(1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount1.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount1.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount1.java
deleted file mode 100644
index 40cf636..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount1.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.examples;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapred.*;
-
-/**
- * Example job for testing hadoop task execution.
- */
-public class GridHadoopWordCount1 {
- /**
- * Entry point to start job.
- * @param args command line parameters.
- * @throws Exception if fails.
- */
- public static void main(String[] args) throws Exception {
- if (args.length != 2) {
- System.out.println("usage: [input] [output]");
- System.exit(-1);
- }
-
- JobConf job = getJob(args[0], args[1]);
-
- JobClient.runJob(job);
- }
-
- /**
- * Gets fully configured JobConf instance.
- *
- * @param input input file name.
- * @param output output directory name.
- * @return Job configuration
- */
- public static JobConf getJob(String input, String output) {
- JobConf conf = new JobConf(GridHadoopWordCount1.class);
- conf.setJobName("wordcount");
-
- conf.setOutputKeyClass(Text.class);
- conf.setOutputValueClass(IntWritable.class);
-
- setTasksClasses(conf, true, true, true);
-
- FileInputFormat.setInputPaths(conf, new Path(input));
- FileOutputFormat.setOutputPath(conf, new Path(output));
-
- return conf;
- }
-
- /**
- * Sets task classes with related info if needed into configuration object.
- *
- * @param jobConf Configuration to change.
- * @param setMapper Option to set mapper and input format classes.
- * @param setCombiner Option to set combiner class.
- * @param setReducer Option to set reducer and output format classes.
- */
- public static void setTasksClasses(JobConf jobConf, boolean setMapper, boolean setCombiner, boolean setReducer) {
- if (setMapper) {
- jobConf.setMapperClass(GridHadoopWordCount1Map.class);
- jobConf.setInputFormat(TextInputFormat.class);
- }
-
- if (setCombiner)
- jobConf.setCombinerClass(GridHadoopWordCount1Reduce.class);
-
- if (setReducer) {
- jobConf.setReducerClass(GridHadoopWordCount1Reduce.class);
- jobConf.setOutputFormat(TextOutputFormat.class);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount1Map.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount1Map.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount1Map.java
deleted file mode 100644
index 5d8e0cc..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount1Map.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.examples;
-
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapred.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Mapper phase of WordCount job.
- */
-public class GridHadoopWordCount1Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
- /** Writable integer constant of '1' is writing as count of found words. */
- private static final IntWritable one = new IntWritable(1);
-
- /** Writable container for writing word. */
- private Text word = new Text();
-
- /** Flag is to check that mapper was configured before run. */
- private boolean wasConfigured;
-
- /** {@inheritDoc} */
- @Override public void map(LongWritable key, Text val, OutputCollector<Text, IntWritable> output, Reporter reporter)
- throws IOException {
-
- assert wasConfigured : "Mapper should be configured";
-
- String line = val.toString();
-
- StringTokenizer tokenizer = new StringTokenizer(line);
-
- while (tokenizer.hasMoreTokens()) {
- word.set(tokenizer.nextToken());
-
- output.collect(word, one);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void configure(JobConf job) {
- super.configure(job);
-
- wasConfigured = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount1Reduce.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount1Reduce.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount1Reduce.java
deleted file mode 100644
index 1b69a43..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount1Reduce.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.examples;
-
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapred.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Combiner and Reducer phase of WordCount job.
- */
-public class GridHadoopWordCount1Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
- /** Flag is to check that mapper was configured before run. */
- private boolean wasConfigured;
-
- /** {@inheritDoc} */
- @Override public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter)
- throws IOException {
- assert wasConfigured : "Reducer should be configured";
-
- int sum = 0;
-
- while (values.hasNext())
- sum += values.next().get();
-
- output.collect(key, new IntWritable(sum));
- }
-
- @Override public void configure(JobConf job) {
- super.configure(job);
-
- wasConfigured = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount2.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount2.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount2.java
deleted file mode 100644
index 6310363..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount2.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.examples;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.hadoop.mapreduce.lib.output.*;
-
-import java.io.*;
-
-/**
- * Example job for testing hadoop task execution.
- */
-public class GridHadoopWordCount2 {
- /**
- * Entry point to start job.
- *
- * @param args Command line parameters.
- * @throws Exception If fails.
- */
- public static void main(String[] args) throws Exception {
- if (args.length != 2) {
- System.out.println("usage: [input] [output]");
- System.exit(-1);
- }
-
- Job job = getJob(args[0], args[1]);
-
- job.submit();
- }
-
- /**
- * Gets fully configured Job instance.
- *
- * @param input Input file name.
- * @param output Output directory name.
- * @return Job instance.
- * @throws IOException If fails.
- */
- public static Job getJob(String input, String output) throws IOException {
- Job job = Job.getInstance();
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- setTasksClasses(job, true, true, true);
-
- FileInputFormat.setInputPaths(job, new Path(input));
- FileOutputFormat.setOutputPath(job, new Path(output));
-
- job.setJarByClass(GridHadoopWordCount2.class);
-
- return job;
- }
-
- /**
- * Sets task classes with related info if needed into configuration object.
- *
- * @param job Configuration to change.
- * @param setMapper Option to set mapper and input format classes.
- * @param setCombiner Option to set combiner class.
- * @param setReducer Option to set reducer and output format classes.
- */
- public static void setTasksClasses(Job job, boolean setMapper, boolean setCombiner, boolean setReducer) {
- if (setMapper) {
- job.setMapperClass(GridHadoopWordCount2Mapper.class);
- job.setInputFormatClass(TextInputFormat.class);
- }
-
- if (setCombiner)
- job.setCombinerClass(GridHadoopWordCount2Reducer.class);
-
- if (setReducer) {
- job.setReducerClass(GridHadoopWordCount2Reducer.class);
- job.setOutputFormatClass(TextOutputFormat.class);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount2Mapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount2Mapper.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount2Mapper.java
deleted file mode 100644
index 849928a..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount2Mapper.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.examples;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapreduce.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Mapper phase of WordCount job.
- */
-public class GridHadoopWordCount2Mapper extends Mapper<Object, Text, Text, IntWritable> implements Configurable {
- /** Writable container for writing word. */
- private Text word = new Text();
-
- /** Writable integer constant of '1' is writing as count of found words. */
- private static final IntWritable one = new IntWritable(1);
-
- /** Flag is to check that mapper was configured before run. */
- private boolean wasConfigured;
-
- /** Flag is to check that mapper was set up before run. */
- private boolean wasSetUp;
-
- /** {@inheritDoc} */
- @Override public void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
- assert wasConfigured : "Mapper should be configured";
- assert wasSetUp : "Mapper should be set up";
-
- StringTokenizer wordList = new StringTokenizer(val.toString());
-
- while (wordList.hasMoreTokens()) {
- word.set(wordList.nextToken());
-
- ctx.write(word, one);
- }
- }
-
- /** {@inheritDoc} */
- @Override protected void setup(Context context) throws IOException, InterruptedException {
- super.setup(context);
- wasSetUp = true;
- }
-
- /** {@inheritDoc} */
- @Override public void setConf(Configuration conf) {
- wasConfigured = true;
- }
-
- /** {@inheritDoc} */
- @Override public Configuration getConf() {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount2Reducer.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount2Reducer.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount2Reducer.java
deleted file mode 100644
index 922bb2f..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/GridHadoopWordCount2Reducer.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.examples;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapreduce.*;
-
-import java.io.*;
-
-/**
- * Combiner and Reducer phase of WordCount job.
- */
-public class GridHadoopWordCount2Reducer extends Reducer<Text, IntWritable, Text, IntWritable> implements Configurable {
- /** Writable container for writing sum of word counts. */
- private IntWritable totalWordCnt = new IntWritable();
-
- /** Flag is to check that mapper was configured before run. */
- private boolean wasConfigured;
-
- /** Flag is to check that mapper was set up before run. */
- private boolean wasSetUp;
-
- /** {@inheritDoc} */
- @Override public void reduce(Text key, Iterable<IntWritable> values, Context ctx) throws IOException, InterruptedException {
- assert wasConfigured : "Reducer should be configured";
- assert wasSetUp : "Reducer should be set up";
-
- int wordCnt = 0;
-
- for (IntWritable value : values)
- wordCnt += value.get();
-
- totalWordCnt.set(wordCnt);
-
- ctx.write(key, totalWordCnt);
- }
-
- /** {@inheritDoc} */
- @Override protected void setup(Context context) throws IOException, InterruptedException {
- super.setup(context);
- wasSetUp = true;
- }
-
- /** {@inheritDoc} */
- @Override public void setConf(Configuration conf) {
- wasConfigured = true;
- }
-
- /** {@inheritDoc} */
- @Override public Configuration getConf() {
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount1.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount1.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount1.java
new file mode 100644
index 0000000..dd9058d
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount1.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.examples;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapred.*;
+
+/**
+ * Example job for testing hadoop task execution.
+ */
+public class HadoopWordCount1 {
+ /**
+ * Entry point to start job.
+ * @param args command line parameters.
+ * @throws Exception if fails.
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length != 2) {
+ System.out.println("usage: [input] [output]");
+ System.exit(-1);
+ }
+
+ JobConf job = getJob(args[0], args[1]);
+
+ JobClient.runJob(job);
+ }
+
+ /**
+ * Gets fully configured JobConf instance.
+ *
+ * @param input input file name.
+ * @param output output directory name.
+ * @return Job configuration
+ */
+ public static JobConf getJob(String input, String output) {
+ JobConf conf = new JobConf(HadoopWordCount1.class);
+ conf.setJobName("wordcount");
+
+ conf.setOutputKeyClass(Text.class);
+ conf.setOutputValueClass(IntWritable.class);
+
+ setTasksClasses(conf, true, true, true);
+
+ FileInputFormat.setInputPaths(conf, new Path(input));
+ FileOutputFormat.setOutputPath(conf, new Path(output));
+
+ return conf;
+ }
+
+ /**
+ * Sets task classes with related info if needed into configuration object.
+ *
+ * @param jobConf Configuration to change.
+ * @param setMapper Option to set mapper and input format classes.
+ * @param setCombiner Option to set combiner class.
+ * @param setReducer Option to set reducer and output format classes.
+ */
+ public static void setTasksClasses(JobConf jobConf, boolean setMapper, boolean setCombiner, boolean setReducer) {
+ if (setMapper) {
+ jobConf.setMapperClass(HadoopWordCount1Map.class);
+ jobConf.setInputFormat(TextInputFormat.class);
+ }
+
+ if (setCombiner)
+ jobConf.setCombinerClass(HadoopWordCount1Reduce.class);
+
+ if (setReducer) {
+ jobConf.setReducerClass(HadoopWordCount1Reduce.class);
+ jobConf.setOutputFormat(TextOutputFormat.class);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount1Map.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount1Map.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount1Map.java
new file mode 100644
index 0000000..c10a7fb
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount1Map.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.examples;
+
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapred.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Mapper phase of WordCount job.
+ */
+public class HadoopWordCount1Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
+ /** Writable integer constant of '1' is writing as count of found words. */
+ private static final IntWritable one = new IntWritable(1);
+
+ /** Writable container for writing word. */
+ private Text word = new Text();
+
+ /** Flag is to check that mapper was configured before run. */
+ private boolean wasConfigured;
+
+ /** {@inheritDoc} */
+ @Override public void map(LongWritable key, Text val, OutputCollector<Text, IntWritable> output, Reporter reporter)
+ throws IOException {
+
+ assert wasConfigured : "Mapper should be configured";
+
+ String line = val.toString();
+
+ StringTokenizer tokenizer = new StringTokenizer(line);
+
+ while (tokenizer.hasMoreTokens()) {
+ word.set(tokenizer.nextToken());
+
+ output.collect(word, one);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void configure(JobConf job) {
+ super.configure(job);
+
+ wasConfigured = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount1Reduce.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount1Reduce.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount1Reduce.java
new file mode 100644
index 0000000..76cd1c3
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount1Reduce.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.examples;
+
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapred.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Combiner and Reducer phase of WordCount job.
+ */
+public class HadoopWordCount1Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
+ /** Flag is to check that mapper was configured before run. */
+ private boolean wasConfigured;
+
+ /** {@inheritDoc} */
+ @Override public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter)
+ throws IOException {
+ assert wasConfigured : "Reducer should be configured";
+
+ int sum = 0;
+
+ while (values.hasNext())
+ sum += values.next().get();
+
+ output.collect(key, new IntWritable(sum));
+ }
+
+ @Override public void configure(JobConf job) {
+ super.configure(job);
+
+ wasConfigured = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2.java
new file mode 100644
index 0000000..dc68df7
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.examples;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.*;
+import org.apache.hadoop.mapreduce.lib.output.*;
+
+import java.io.*;
+
+/**
+ * Example job for testing hadoop task execution.
+ */
+public class HadoopWordCount2 {
+ /**
+ * Entry point to start job.
+ *
+ * @param args Command line parameters.
+ * @throws Exception If fails.
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length != 2) {
+ System.out.println("usage: [input] [output]");
+ System.exit(-1);
+ }
+
+ Job job = getJob(args[0], args[1]);
+
+ job.submit();
+ }
+
+ /**
+ * Gets fully configured Job instance.
+ *
+ * @param input Input file name.
+ * @param output Output directory name.
+ * @return Job instance.
+ * @throws IOException If fails.
+ */
+ public static Job getJob(String input, String output) throws IOException {
+ Job job = Job.getInstance();
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ setTasksClasses(job, true, true, true);
+
+ FileInputFormat.setInputPaths(job, new Path(input));
+ FileOutputFormat.setOutputPath(job, new Path(output));
+
+ job.setJarByClass(HadoopWordCount2.class);
+
+ return job;
+ }
+
+ /**
+ * Sets task classes with related info if needed into configuration object.
+ *
+ * @param job Configuration to change.
+ * @param setMapper Option to set mapper and input format classes.
+ * @param setCombiner Option to set combiner class.
+ * @param setReducer Option to set reducer and output format classes.
+ */
+ public static void setTasksClasses(Job job, boolean setMapper, boolean setCombiner, boolean setReducer) {
+ if (setMapper) {
+ job.setMapperClass(HadoopWordCount2Mapper.class);
+ job.setInputFormatClass(TextInputFormat.class);
+ }
+
+ if (setCombiner)
+ job.setCombinerClass(HadoopWordCount2Reducer.class);
+
+ if (setReducer) {
+ job.setReducerClass(HadoopWordCount2Reducer.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2Mapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2Mapper.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2Mapper.java
new file mode 100644
index 0000000..6ca7ccd
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2Mapper.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.examples;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Mapper phase of WordCount job.
+ */
+public class HadoopWordCount2Mapper extends Mapper<Object, Text, Text, IntWritable> implements Configurable {
+ /** Writable container for writing word. */
+ private Text word = new Text();
+
+ /** Writable integer constant of '1' is writing as count of found words. */
+ private static final IntWritable one = new IntWritable(1);
+
+ /** Flag is to check that mapper was configured before run. */
+ private boolean wasConfigured;
+
+ /** Flag is to check that mapper was set up before run. */
+ private boolean wasSetUp;
+
+ /** {@inheritDoc} */
+ @Override public void map(Object key, Text val, Context ctx) throws IOException, InterruptedException {
+ assert wasConfigured : "Mapper should be configured";
+ assert wasSetUp : "Mapper should be set up";
+
+ StringTokenizer wordList = new StringTokenizer(val.toString());
+
+ while (wordList.hasMoreTokens()) {
+ word.set(wordList.nextToken());
+
+ ctx.write(word, one);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void setup(Context context) throws IOException, InterruptedException {
+ super.setup(context);
+ wasSetUp = true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setConf(Configuration conf) {
+ wasConfigured = true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Configuration getConf() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2Reducer.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2Reducer.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2Reducer.java
new file mode 100644
index 0000000..fedaaf9
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2Reducer.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.examples;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.*;
+
+import java.io.*;
+
+/**
+ * Combiner and Reducer phase of WordCount job.
+ */
+public class HadoopWordCount2Reducer extends Reducer<Text, IntWritable, Text, IntWritable> implements Configurable {
+ /** Writable container for writing sum of word counts. */
+ private IntWritable totalWordCnt = new IntWritable();
+
+ /** Flag is to check that mapper was configured before run. */
+ private boolean wasConfigured;
+
+ /** Flag is to check that mapper was set up before run. */
+ private boolean wasSetUp;
+
+ /** {@inheritDoc} */
+ @Override public void reduce(Text key, Iterable<IntWritable> values, Context ctx) throws IOException, InterruptedException {
+ assert wasConfigured : "Reducer should be configured";
+ assert wasSetUp : "Reducer should be set up";
+
+ int wordCnt = 0;
+
+ for (IntWritable value : values)
+ wordCnt += value.get();
+
+ totalWordCnt.set(wordCnt);
+
+ ctx.write(key, totalWordCnt);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void setup(Context context) throws IOException, InterruptedException {
+ super.setup(context);
+ wasSetUp = true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setConf(Configuration conf) {
+ wasConfigured = true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Configuration getConf() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopAbstractMapTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopAbstractMapTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopAbstractMapTest.java
deleted file mode 100644
index 716fe19..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopAbstractMapTest.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
-
-import org.apache.commons.collections.comparators.*;
-import org.apache.hadoop.io.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-import org.apache.ignite.testframework.junits.common.*;
-import org.jetbrains.annotations.*;
-
-import java.util.*;
-
-/**
- * Abstract class for maps test.
- */
-public abstract class GridHadoopAbstractMapTest extends GridCommonAbstractTest {
- /**
- * Test task context.
- */
- protected static class TaskContext extends GridHadoopTaskContext {
- /**
- */
- protected TaskContext() {
- super(null, null);
- }
-
- /** {@inheritDoc} */
- @Override public <T extends GridHadoopCounter> T counter(String grp, String name, Class<T> cls) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopCounters counters() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopPartitioner partitioner() throws IgniteCheckedException {
- assert false;
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopSerialization keySerialization() throws IgniteCheckedException {
- return new GridHadoopWritableSerialization(IntWritable.class);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopSerialization valueSerialization() throws IgniteCheckedException {
- return new GridHadoopWritableSerialization(IntWritable.class);
- }
-
- /** {@inheritDoc} */
- @Override public Comparator<Object> sortComparator() {
- return ComparableComparator.getInstance();
- }
-
- /** {@inheritDoc} */
- @Override public Comparator<Object> groupComparator() {
- return ComparableComparator.getInstance();
- }
-
- /** {@inheritDoc} */
- @Override public void run() throws IgniteCheckedException {
- assert false;
- }
-
- /** {@inheritDoc} */
- @Override public void cancel() {
- assert false;
- }
-
- /** {@inheritDoc} */
- @Override public void prepareTaskEnvironment() throws IgniteCheckedException {
- assert false;
- }
-
- @Override public void cleanupTaskEnvironment() throws IgniteCheckedException {
- assert false;
- }
- }
-
- /**
- * Test job info.
- */
- protected static class JobInfo implements GridHadoopJobInfo {
- /** {@inheritDoc} */
- @Nullable @Override public String property(String name) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasCombiner() {
- assert false;
-
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasReducer() {
- assert false;
-
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopJob createJob(GridHadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException {
- assert false;
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public int reducers() {
- assert false;
-
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public String jobName() {
- assert false;
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public String user() {
- assert false;
-
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimapSelftest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimapSelftest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimapSelftest.java
deleted file mode 100644
index 88dfd2b..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimapSelftest.java
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
-
-import com.google.common.collect.*;
-import org.apache.hadoop.io.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.io.*;
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-import org.apache.ignite.internal.util.typedef.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-import static org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory.*;
-
-/**
- *
- */
-public class GridHadoopConcurrentHashMultimapSelftest extends GridHadoopAbstractMapTest {
- /** */
- public void testMapSimple() throws Exception {
- GridUnsafeMemory mem = new GridUnsafeMemory(0);
-
-// mem.listen(new GridOffHeapEventListener() {
-// @Override public void onEvent(GridOffHeapEvent evt) {
-// if (evt == GridOffHeapEvent.ALLOCATE)
-// U.dumpStack();
-// }
-// });
-
- Random rnd = new Random();
-
- int mapSize = 16 << rnd.nextInt(3);
-
- GridHadoopJobInfo job = new JobInfo();
-
- GridHadoopTaskContext taskCtx = new TaskContext();
-
- GridHadoopConcurrentHashMultimap m = new GridHadoopConcurrentHashMultimap(job, mem, mapSize);
-
- GridHadoopConcurrentHashMultimap.Adder a = m.startAdding(taskCtx);
-
- Multimap<Integer, Integer> mm = ArrayListMultimap.create();
- Multimap<Integer, Integer> vis = ArrayListMultimap.create();
-
- for (int i = 0, vals = 4 * mapSize + rnd.nextInt(25); i < vals; i++) {
- int key = rnd.nextInt(mapSize);
- int val = rnd.nextInt();
-
- a.write(new IntWritable(key), new IntWritable(val));
- mm.put(key, val);
-
- X.println("k: " + key + " v: " + val);
-
- a.close();
-
- check(m, mm, vis, taskCtx);
-
- a = m.startAdding(taskCtx);
- }
-
-// a.add(new IntWritable(10), new IntWritable(2));
-// mm.put(10, 2);
-// check(m, mm);
-
- a.close();
-
- X.println("Alloc: " + mem.allocatedSize());
-
- m.close();
-
- assertEquals(0, mem.allocatedSize());
- }
-
- private void check(GridHadoopConcurrentHashMultimap m, Multimap<Integer, Integer> mm,
- final Multimap<Integer, Integer> vis, GridHadoopTaskContext taskCtx) throws Exception {
- final GridHadoopTaskInput in = m.input(taskCtx);
-
- Map<Integer, Collection<Integer>> mmm = mm.asMap();
-
- int keys = 0;
-
- while (in.next()) {
- keys++;
-
- IntWritable k = (IntWritable)in.key();
-
- assertNotNull(k);
-
- Deque<Integer> vs = new LinkedList<>();
-
- Iterator<?> it = in.values();
-
- while (it.hasNext())
- vs.addFirst(((IntWritable) it.next()).get());
-
- Collection<Integer> exp = mmm.get(k.get());
-
- assertEquals(exp, vs);
- }
-
- assertEquals(mmm.size(), keys);
-
- assertEquals(m.keys(), keys);
-
- X.println("keys: " + keys + " cap: " + m.capacity());
-
- // Check visitor.
-
- final byte[] buf = new byte[4];
-
- final GridDataInput dataInput = new GridUnsafeDataInput();
-
- m.visit(false, new GridHadoopConcurrentHashMultimap.Visitor() {
- /** */
- IntWritable key = new IntWritable();
-
- /** */
- IntWritable val = new IntWritable();
-
- @Override public void onKey(long keyPtr, int keySize) {
- read(keyPtr, keySize, key);
- }
-
- @Override public void onValue(long valPtr, int valSize) {
- read(valPtr, valSize, val);
-
- vis.put(key.get(), val.get());
- }
-
- private void read(long ptr, int size, Writable w) {
- assert size == 4 : size;
-
- UNSAFE.copyMemory(null, ptr, buf, BYTE_ARR_OFF, size);
-
- dataInput.bytes(buf, size);
-
- try {
- w.readFields(dataInput);
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- });
-
-// X.println("vis: " + vis);
-
- assertEquals(mm, vis);
-
- in.close();
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testMultiThreaded() throws Exception {
- GridUnsafeMemory mem = new GridUnsafeMemory(0);
-
- X.println("___ Started");
-
- Random rnd = new GridRandom();
-
- for (int i = 0; i < 20; i++) {
- GridHadoopJobInfo job = new JobInfo();
-
- final GridHadoopTaskContext taskCtx = new TaskContext();
-
- final GridHadoopConcurrentHashMultimap m = new GridHadoopConcurrentHashMultimap(job, mem, 16);
-
- final ConcurrentMap<Integer, Collection<Integer>> mm = new ConcurrentHashMap<>();
-
- X.println("___ MT");
-
- multithreaded(new Callable<Object>() {
- @Override public Object call() throws Exception {
- X.println("___ TH in");
-
- Random rnd = new GridRandom();
-
- IntWritable key = new IntWritable();
- IntWritable val = new IntWritable();
-
- GridHadoopMultimap.Adder a = m.startAdding(taskCtx);
-
- for (int i = 0; i < 50000; i++) {
- int k = rnd.nextInt(32000);
- int v = rnd.nextInt();
-
- key.set(k);
- val.set(v);
-
- a.write(key, val);
-
- Collection<Integer> list = mm.get(k);
-
- if (list == null) {
- list = new ConcurrentLinkedQueue<>();
-
- Collection<Integer> old = mm.putIfAbsent(k, list);
-
- if (old != null)
- list = old;
- }
-
- list.add(v);
- }
-
- a.close();
-
- X.println("___ TH out");
-
- return null;
- }
- }, 3 + rnd.nextInt(27));
-
- X.println("___ Check: " + m.capacity());
-
- assertEquals(mm.size(), m.keys());
-
- assertTrue(m.capacity() > 32000);
-
- GridHadoopTaskInput in = m.input(taskCtx);
-
- while (in.next()) {
- IntWritable key = (IntWritable) in.key();
-
- Iterator<?> valsIter = in.values();
-
- Collection<Integer> vals = mm.remove(key.get());
-
- assertNotNull(vals);
-
- while (valsIter.hasNext()) {
- IntWritable val = (IntWritable) valsIter.next();
-
- assertTrue(vals.remove(val.get()));
- }
-
- assertTrue(vals.isEmpty());
- }
-
- in.close();
- m.close();
-
- assertEquals(0, mem.allocatedSize());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMapSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMapSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMapSelfTest.java
deleted file mode 100644
index 92177ad..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMapSelfTest.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
-
-import com.google.common.collect.*;
-import org.apache.hadoop.io.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-import org.apache.ignite.internal.util.typedef.*;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-/**
- *
- */
-public class GridHadoopHashMapSelfTest extends GridHadoopAbstractMapTest {
-
- public void _testAllocation() throws Exception {
- final GridUnsafeMemory mem = new GridUnsafeMemory(0);
-
- long size = 3L * 1024 * 1024 * 1024;
-
- final long chunk = 16;// * 1024;
-
- final int page = 4 * 1024;
-
- final int writes = chunk < page ? 1 : (int)(chunk / page);
-
- final long cnt = size / chunk;
-
- assert cnt < Integer.MAX_VALUE;
-
- final int threads = 4;
-
- long start = System.currentTimeMillis();
-
- multithreaded(new Callable<Object>() {
- @Override public Object call() throws Exception {
- int cnt0 = (int)(cnt / threads);
-
- for (int i = 0; i < cnt0; i++) {
- long ptr = mem.allocate(chunk);
-
- for (int j = 0; j < writes; j++)
- mem.writeInt(ptr + j * page, 100500);
- }
-
- return null;
- }
- }, threads);
-
- X.println("End: " + (System.currentTimeMillis() - start) + " mem: " + mem.allocatedSize() + " cnt: " + cnt);
-
- Thread.sleep(30000);
- }
-
-
- /** */
- public void testMapSimple() throws Exception {
- GridUnsafeMemory mem = new GridUnsafeMemory(0);
-
-// mem.listen(new GridOffHeapEventListener() {
-// @Override public void onEvent(GridOffHeapEvent evt) {
-// if (evt == GridOffHeapEvent.ALLOCATE)
-// U.dumpStack();
-// }
-// });
-
- Random rnd = new Random();
-
- int mapSize = 16 << rnd.nextInt(3);
-
- GridHadoopTaskContext taskCtx = new TaskContext();
-
- final GridHadoopHashMultimap m = new GridHadoopHashMultimap(new JobInfo(), mem, mapSize);
-
- GridHadoopMultimap.Adder a = m.startAdding(taskCtx);
-
- Multimap<Integer, Integer> mm = ArrayListMultimap.create();
-
- for (int i = 0, vals = 4 * mapSize + rnd.nextInt(25); i < vals; i++) {
- int key = rnd.nextInt(mapSize);
- int val = rnd.nextInt();
-
- a.write(new IntWritable(key), new IntWritable(val));
- mm.put(key, val);
-
- X.println("k: " + key + " v: " + val);
-
- a.close();
-
- check(m, mm, taskCtx);
-
- a = m.startAdding(taskCtx);
- }
-
-// a.add(new IntWritable(10), new IntWritable(2));
-// mm.put(10, 2);
-// check(m, mm);
-
- a.close();
-
- X.println("Alloc: " + mem.allocatedSize());
-
- m.close();
-
- assertEquals(0, mem.allocatedSize());
- }
-
- private void check(GridHadoopHashMultimap m, Multimap<Integer, Integer> mm, GridHadoopTaskContext taskCtx) throws Exception {
- final GridHadoopTaskInput in = m.input(taskCtx);
-
- Map<Integer, Collection<Integer>> mmm = mm.asMap();
-
- int keys = 0;
-
- while (in.next()) {
- keys++;
-
- IntWritable k = (IntWritable)in.key();
-
- assertNotNull(k);
-
- ArrayList<Integer> vs = new ArrayList<>();
-
- Iterator<?> it = in.values();
-
- while (it.hasNext())
- vs.add(((IntWritable) it.next()).get());
-
- Collection<Integer> exp = mmm.get(k.get());
-
- assertEquals(sorted(exp), sorted(vs));
- }
-
- X.println("keys: " + keys + " cap: " + m.capacity());
-
- assertEquals(mmm.size(), keys);
-
- assertEquals(m.keys(), keys);
-
- in.close();
- }
-
- private GridLongList sorted(Collection<Integer> col) {
- GridLongList lst = new GridLongList(col.size());
-
- for (Integer i : col)
- lst.add(i);
-
- return lst.sort();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipListSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipListSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipListSelfTest.java
deleted file mode 100644
index 6ba00ad..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipListSelfTest.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
-
-import com.google.common.collect.*;
-import org.apache.hadoop.io.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.io.*;
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-import org.apache.ignite.internal.util.typedef.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-import static java.lang.Math.*;
-import static org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory.*;
-
-/**
- * Skip list tests.
- */
-public class GridHadoopSkipListSelfTest extends GridHadoopAbstractMapTest {
- /**
- *
- */
- public void testLevel() {
- Random rnd = new GridRandom();
-
- int[] levelsCnts = new int[32];
-
- int all = 10000;
-
- for (int i = 0; i < all; i++) {
- int level = GridHadoopSkipList.randomLevel(rnd);
-
- levelsCnts[level]++;
- }
-
- X.println("Distribution: " + Arrays.toString(levelsCnts));
-
- for (int level = 0; level < levelsCnts.length; level++) {
- int exp = (level + 1) == levelsCnts.length ? 0 : all >>> (level + 1);
-
- double precission = 0.72 / Math.max(32 >>> level, 1);
-
- int sigma = max((int)ceil(precission * exp), 5);
-
- X.println("Level: " + level + " exp: " + exp + " act: " + levelsCnts[level] + " precission: " + precission +
- " sigma: " + sigma);
-
- assertTrue(abs(exp - levelsCnts[level]) <= sigma);
- }
- }
-
- public void testMapSimple() throws Exception {
- GridUnsafeMemory mem = new GridUnsafeMemory(0);
-
-// mem.listen(new GridOffHeapEventListener() {
-// @Override public void onEvent(GridOffHeapEvent evt) {
-// if (evt == GridOffHeapEvent.ALLOCATE)
-// U.dumpStack();
-// }
-// });
-
- Random rnd = new Random();
-
- int mapSize = 16 << rnd.nextInt(6);
-
- GridHadoopJobInfo job = new JobInfo();
-
- GridHadoopTaskContext taskCtx = new TaskContext();
-
- GridHadoopMultimap m = new GridHadoopSkipList(job, mem);
-
- GridHadoopConcurrentHashMultimap.Adder a = m.startAdding(taskCtx);
-
- Multimap<Integer, Integer> mm = ArrayListMultimap.create();
- Multimap<Integer, Integer> vis = ArrayListMultimap.create();
-
- for (int i = 0, vals = 4 * mapSize + rnd.nextInt(25); i < vals; i++) {
- int key = rnd.nextInt(mapSize);
- int val = rnd.nextInt();
-
- a.write(new IntWritable(key), new IntWritable(val));
- mm.put(key, val);
-
- X.println("k: " + key + " v: " + val);
-
- a.close();
-
- check(m, mm, vis, taskCtx);
-
- a = m.startAdding(taskCtx);
- }
-
-// a.add(new IntWritable(10), new IntWritable(2));
-// mm.put(10, 2);
-// check(m, mm);
-
- a.close();
-
- X.println("Alloc: " + mem.allocatedSize());
-
- m.close();
-
- assertEquals(0, mem.allocatedSize());
- }
-
- private void check(GridHadoopMultimap m, Multimap<Integer, Integer> mm, final Multimap<Integer, Integer> vis, GridHadoopTaskContext taskCtx)
- throws Exception {
- final GridHadoopTaskInput in = m.input(taskCtx);
-
- Map<Integer, Collection<Integer>> mmm = mm.asMap();
-
- int keys = 0;
-
- int prevKey = Integer.MIN_VALUE;
-
- while (in.next()) {
- keys++;
-
- IntWritable k = (IntWritable)in.key();
-
- assertNotNull(k);
-
- assertTrue(k.get() > prevKey);
-
- prevKey = k.get();
-
- Deque<Integer> vs = new LinkedList<>();
-
- Iterator<?> it = in.values();
-
- while (it.hasNext())
- vs.addFirst(((IntWritable) it.next()).get());
-
- Collection<Integer> exp = mmm.get(k.get());
-
- assertEquals(exp, vs);
- }
-
- assertEquals(mmm.size(), keys);
-
-//! assertEquals(m.keys(), keys);
-
- // Check visitor.
-
- final byte[] buf = new byte[4];
-
- final GridDataInput dataInput = new GridUnsafeDataInput();
-
- m.visit(false, new GridHadoopConcurrentHashMultimap.Visitor() {
- /** */
- IntWritable key = new IntWritable();
-
- /** */
- IntWritable val = new IntWritable();
-
- @Override public void onKey(long keyPtr, int keySize) {
- read(keyPtr, keySize, key);
- }
-
- @Override public void onValue(long valPtr, int valSize) {
- read(valPtr, valSize, val);
-
- vis.put(key.get(), val.get());
- }
-
- private void read(long ptr, int size, Writable w) {
- assert size == 4 : size;
-
- UNSAFE.copyMemory(null, ptr, buf, BYTE_ARR_OFF, size);
-
- dataInput.bytes(buf, size);
-
- try {
- w.readFields(dataInput);
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- });
-
-// X.println("vis: " + vis);
-
- assertEquals(mm, vis);
-
- in.close();
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testMultiThreaded() throws Exception {
- GridUnsafeMemory mem = new GridUnsafeMemory(0);
-
- X.println("___ Started");
-
- Random rnd = new GridRandom();
-
- for (int i = 0; i < 20; i++) {
- GridHadoopJobInfo job = new JobInfo();
-
- final GridHadoopTaskContext taskCtx = new TaskContext();
-
- final GridHadoopMultimap m = new GridHadoopSkipList(job, mem);
-
- final ConcurrentMap<Integer, Collection<Integer>> mm = new ConcurrentHashMap<>();
-
- X.println("___ MT");
-
- multithreaded(new Callable<Object>() {
- @Override public Object call() throws Exception {
- X.println("___ TH in");
-
- Random rnd = new GridRandom();
-
- IntWritable key = new IntWritable();
- IntWritable val = new IntWritable();
-
- GridHadoopMultimap.Adder a = m.startAdding(taskCtx);
-
- for (int i = 0; i < 50000; i++) {
- int k = rnd.nextInt(32000);
- int v = rnd.nextInt();
-
- key.set(k);
- val.set(v);
-
- a.write(key, val);
-
- Collection<Integer> list = mm.get(k);
-
- if (list == null) {
- list = new ConcurrentLinkedQueue<>();
-
- Collection<Integer> old = mm.putIfAbsent(k, list);
-
- if (old != null)
- list = old;
- }
-
- list.add(v);
- }
-
- a.close();
-
- X.println("___ TH out");
-
- return null;
- }
- }, 3 + rnd.nextInt(27));
-
- GridHadoopTaskInput in = m.input(taskCtx);
-
- int prevKey = Integer.MIN_VALUE;
-
- while (in.next()) {
- IntWritable key = (IntWritable)in.key();
-
- assertTrue(key.get() > prevKey);
-
- prevKey = key.get();
-
- Iterator<?> valsIter = in.values();
-
- Collection<Integer> vals = mm.remove(key.get());
-
- assertNotNull(vals);
-
- while (valsIter.hasNext()) {
- IntWritable val = (IntWritable) valsIter.next();
-
- assertTrue(vals.remove(val.get()));
- }
-
- assertTrue(vals.isEmpty());
- }
-
- in.close();
- m.close();
-
- assertEquals(0, mem.allocatedSize());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java
new file mode 100644
index 0000000..b4ed5e1
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
+
+import org.apache.commons.collections.comparators.*;
+import org.apache.hadoop.io.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+import org.apache.ignite.testframework.junits.common.*;
+import org.jetbrains.annotations.*;
+
+import java.util.*;
+
+/**
+ * Abstract class for maps test.
+ */
+public abstract class HadoopAbstractMapTest extends GridCommonAbstractTest {
+ /**
+ * Test task context.
+ */
+ protected static class TaskContext extends HadoopTaskContext {
+ /**
+ */
+ protected TaskContext() {
+ super(null, null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T extends HadoopCounter> T counter(String grp, String name, Class<T> cls) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopCounters counters() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopPartitioner partitioner() throws IgniteCheckedException {
+ assert false;
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopSerialization keySerialization() throws IgniteCheckedException {
+ return new HadoopWritableSerialization(IntWritable.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopSerialization valueSerialization() throws IgniteCheckedException {
+ return new HadoopWritableSerialization(IntWritable.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Comparator<Object> sortComparator() {
+ return ComparableComparator.getInstance();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Comparator<Object> groupComparator() {
+ return ComparableComparator.getInstance();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() throws IgniteCheckedException {
+ assert false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancel() {
+ assert false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void prepareTaskEnvironment() throws IgniteCheckedException {
+ assert false;
+ }
+
+ @Override public void cleanupTaskEnvironment() throws IgniteCheckedException {
+ assert false;
+ }
+ }
+
+ /**
+ * Test job info.
+ */
+ protected static class JobInfo implements HadoopJobInfo {
+ /** {@inheritDoc} */
+ @Nullable @Override public String property(String name) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean hasCombiner() {
+ assert false;
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean hasReducer() {
+ assert false;
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopJob createJob(HadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException {
+ assert false;
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int reducers() {
+ assert false;
+
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String jobName() {
+ assert false;
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String user() {
+ assert false;
+
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java
new file mode 100644
index 0000000..ae6bafa
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
+
+import com.google.common.collect.*;
+import org.apache.hadoop.io.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.io.*;
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+import org.apache.ignite.internal.util.typedef.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory.*;
+
+/**
+ *
+ */
+public class HadoopConcurrentHashMultimapSelftest extends HadoopAbstractMapTest {
+ /** */
+ public void testMapSimple() throws Exception {
+ GridUnsafeMemory mem = new GridUnsafeMemory(0);
+
+// mem.listen(new GridOffHeapEventListener() {
+// @Override public void onEvent(GridOffHeapEvent evt) {
+// if (evt == GridOffHeapEvent.ALLOCATE)
+// U.dumpStack();
+// }
+// });
+
+ Random rnd = new Random();
+
+ int mapSize = 16 << rnd.nextInt(3);
+
+ HadoopJobInfo job = new JobInfo();
+
+ HadoopTaskContext taskCtx = new TaskContext();
+
+ HadoopConcurrentHashMultimap m = new HadoopConcurrentHashMultimap(job, mem, mapSize);
+
+ HadoopConcurrentHashMultimap.Adder a = m.startAdding(taskCtx);
+
+ Multimap<Integer, Integer> mm = ArrayListMultimap.create();
+ Multimap<Integer, Integer> vis = ArrayListMultimap.create();
+
+ for (int i = 0, vals = 4 * mapSize + rnd.nextInt(25); i < vals; i++) {
+ int key = rnd.nextInt(mapSize);
+ int val = rnd.nextInt();
+
+ a.write(new IntWritable(key), new IntWritable(val));
+ mm.put(key, val);
+
+ X.println("k: " + key + " v: " + val);
+
+ a.close();
+
+ check(m, mm, vis, taskCtx);
+
+ a = m.startAdding(taskCtx);
+ }
+
+// a.add(new IntWritable(10), new IntWritable(2));
+// mm.put(10, 2);
+// check(m, mm);
+
+ a.close();
+
+ X.println("Alloc: " + mem.allocatedSize());
+
+ m.close();
+
+ assertEquals(0, mem.allocatedSize());
+ }
+
+ private void check(HadoopConcurrentHashMultimap m, Multimap<Integer, Integer> mm,
+ final Multimap<Integer, Integer> vis, HadoopTaskContext taskCtx) throws Exception {
+ final HadoopTaskInput in = m.input(taskCtx);
+
+ Map<Integer, Collection<Integer>> mmm = mm.asMap();
+
+ int keys = 0;
+
+ while (in.next()) {
+ keys++;
+
+ IntWritable k = (IntWritable)in.key();
+
+ assertNotNull(k);
+
+ Deque<Integer> vs = new LinkedList<>();
+
+ Iterator<?> it = in.values();
+
+ while (it.hasNext())
+ vs.addFirst(((IntWritable) it.next()).get());
+
+ Collection<Integer> exp = mmm.get(k.get());
+
+ assertEquals(exp, vs);
+ }
+
+ assertEquals(mmm.size(), keys);
+
+ assertEquals(m.keys(), keys);
+
+ X.println("keys: " + keys + " cap: " + m.capacity());
+
+ // Check visitor.
+
+ final byte[] buf = new byte[4];
+
+ final GridDataInput dataInput = new GridUnsafeDataInput();
+
+ m.visit(false, new HadoopConcurrentHashMultimap.Visitor() {
+ /** */
+ IntWritable key = new IntWritable();
+
+ /** */
+ IntWritable val = new IntWritable();
+
+ @Override public void onKey(long keyPtr, int keySize) {
+ read(keyPtr, keySize, key);
+ }
+
+ @Override public void onValue(long valPtr, int valSize) {
+ read(valPtr, valSize, val);
+
+ vis.put(key.get(), val.get());
+ }
+
+ private void read(long ptr, int size, Writable w) {
+ assert size == 4 : size;
+
+ UNSAFE.copyMemory(null, ptr, buf, BYTE_ARR_OFF, size);
+
+ dataInput.bytes(buf, size);
+
+ try {
+ w.readFields(dataInput);
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+
+// X.println("vis: " + vis);
+
+ assertEquals(mm, vis);
+
+ in.close();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testMultiThreaded() throws Exception {
+ GridUnsafeMemory mem = new GridUnsafeMemory(0);
+
+ X.println("___ Started");
+
+ Random rnd = new GridRandom();
+
+ for (int i = 0; i < 20; i++) {
+ HadoopJobInfo job = new JobInfo();
+
+ final HadoopTaskContext taskCtx = new TaskContext();
+
+ final HadoopConcurrentHashMultimap m = new HadoopConcurrentHashMultimap(job, mem, 16);
+
+ final ConcurrentMap<Integer, Collection<Integer>> mm = new ConcurrentHashMap<>();
+
+ X.println("___ MT");
+
+ multithreaded(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ X.println("___ TH in");
+
+ Random rnd = new GridRandom();
+
+ IntWritable key = new IntWritable();
+ IntWritable val = new IntWritable();
+
+ HadoopMultimap.Adder a = m.startAdding(taskCtx);
+
+ for (int i = 0; i < 50000; i++) {
+ int k = rnd.nextInt(32000);
+ int v = rnd.nextInt();
+
+ key.set(k);
+ val.set(v);
+
+ a.write(key, val);
+
+ Collection<Integer> list = mm.get(k);
+
+ if (list == null) {
+ list = new ConcurrentLinkedQueue<>();
+
+ Collection<Integer> old = mm.putIfAbsent(k, list);
+
+ if (old != null)
+ list = old;
+ }
+
+ list.add(v);
+ }
+
+ a.close();
+
+ X.println("___ TH out");
+
+ return null;
+ }
+ }, 3 + rnd.nextInt(27));
+
+ X.println("___ Check: " + m.capacity());
+
+ assertEquals(mm.size(), m.keys());
+
+ assertTrue(m.capacity() > 32000);
+
+ HadoopTaskInput in = m.input(taskCtx);
+
+ while (in.next()) {
+ IntWritable key = (IntWritable) in.key();
+
+ Iterator<?> valsIter = in.values();
+
+ Collection<Integer> vals = mm.remove(key.get());
+
+ assertNotNull(vals);
+
+ while (valsIter.hasNext()) {
+ IntWritable val = (IntWritable) valsIter.next();
+
+ assertTrue(vals.remove(val.get()));
+ }
+
+ assertTrue(vals.isEmpty());
+ }
+
+ in.close();
+ m.close();
+
+ assertEquals(0, mem.allocatedSize());
+ }
+ }
+}
[33/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSetup.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSetup.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSetup.java
deleted file mode 100644
index 66b1db4..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSetup.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.net.*;
-import java.nio.file.*;
-import java.text.*;
-import java.util.*;
-
-import static org.apache.ignite.internal.IgniteVersionUtils.*;
-
-/**
- * Setup tool to configure Hadoop client.
- */
-public class GridHadoopSetup {
- /** */
- public static final String WINUTILS_EXE = "winutils.exe";
-
- /** */
- private static final FilenameFilter IGNITE_JARS = new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return name.startsWith("ignite-") && name.endsWith(".jar");
- }
- };
-
- /**
- * The main method.
- * @param ignore Params.
- */
- public static void main(String[] ignore) {
- X.println(
- " __________ ________________ ",
- " / _/ ___/ |/ / _/_ __/ __/ ",
- " _/ // (_ / // / / / / _/ ",
- "/___/\\___/_/|_/___/ /_/ /___/ ",
- " for Apache Hadoop ",
- " ");
-
- println("Version " + ACK_VER_STR);
-
- configureHadoop();
- }
-
- /**
- * This operation prepares the clean unpacked Hadoop distributive to work as client with Ignite-Hadoop.
- * It performs these operations:
- * <ul>
- * <li>Check for setting of HADOOP_HOME environment variable.</li>
- * <li>Try to resolve HADOOP_COMMON_HOME or evaluate it relative to HADOOP_HOME.</li>
- * <li>In Windows check if winutils.exe exists and try to fix issue with some restrictions.</li>
- * <li>In Windows check new line character issues in CMD scripts.</li>
- * <li>Scan Hadoop lib directory to detect Ignite JARs. If these don't exist tries to create ones.</li>
- * </ul>
- */
- private static void configureHadoop() {
- String igniteHome = U.getIgniteHome();
-
- println("IGNITE_HOME is set to '" + igniteHome + "'.");
-
- checkIgniteHome(igniteHome);
-
- String homeVar = "HADOOP_HOME";
- String hadoopHome = System.getenv(homeVar);
-
- if (F.isEmpty(hadoopHome)) {
- homeVar = "HADOOP_PREFIX";
- hadoopHome = System.getenv(homeVar);
- }
-
- if (F.isEmpty(hadoopHome))
- exit("Neither HADOOP_HOME nor HADOOP_PREFIX environment variable is set. Please set one of them to a " +
- "valid Hadoop installation directory and run setup tool again.", null);
-
- hadoopHome = hadoopHome.replaceAll("\"", "");
-
- println(homeVar + " is set to '" + hadoopHome + "'.");
-
- String hiveHome = System.getenv("HIVE_HOME");
-
- if (!F.isEmpty(hiveHome)) {
- hiveHome = hiveHome.replaceAll("\"", "");
-
- println("HIVE_HOME is set to '" + hiveHome + "'.");
- }
-
- File hadoopDir = new File(hadoopHome);
-
- if (!hadoopDir.exists())
- exit("Hadoop installation folder does not exist.", null);
-
- if (!hadoopDir.isDirectory())
- exit("HADOOP_HOME must point to a directory.", null);
-
- if (!hadoopDir.canRead())
- exit("Hadoop installation folder can not be read. Please check permissions.", null);
-
- File hadoopCommonDir;
-
- String hadoopCommonHome = System.getenv("HADOOP_COMMON_HOME");
-
- if (F.isEmpty(hadoopCommonHome)) {
- hadoopCommonDir = new File(hadoopDir, "share/hadoop/common");
-
- println("HADOOP_COMMON_HOME is not set, will use '" + hadoopCommonDir.getPath() + "'.");
- }
- else {
- println("HADOOP_COMMON_HOME is set to '" + hadoopCommonHome + "'.");
-
- hadoopCommonDir = new File(hadoopCommonHome);
- }
-
- if (!hadoopCommonDir.canRead())
- exit("Failed to read Hadoop common dir in '" + hadoopCommonHome + "'.", null);
-
- File hadoopCommonLibDir = new File(hadoopCommonDir, "lib");
-
- if (!hadoopCommonLibDir.canRead())
- exit("Failed to read Hadoop 'lib' folder in '" + hadoopCommonLibDir.getPath() + "'.", null);
-
- if (U.isWindows()) {
- checkJavaPathSpaces();
-
- File hadoopBinDir = new File(hadoopDir, "bin");
-
- if (!hadoopBinDir.canRead())
- exit("Failed to read subdirectory 'bin' in HADOOP_HOME.", null);
-
- File winutilsFile = new File(hadoopBinDir, WINUTILS_EXE);
-
- if (!winutilsFile.exists()) {
- if (ask("File '" + WINUTILS_EXE + "' does not exist. " +
- "It may be replaced by a stub. Create it?")) {
- println("Creating file stub '" + winutilsFile.getAbsolutePath() + "'.");
-
- boolean ok = false;
-
- try {
- ok = winutilsFile.createNewFile();
- }
- catch (IOException ignore) {
- // No-op.
- }
-
- if (!ok)
- exit("Failed to create '" + WINUTILS_EXE + "' file. Please check permissions.", null);
- }
- else
- println("Ok. But Hadoop client probably will not work on Windows this way...");
- }
-
- processCmdFiles(hadoopDir, "bin", "sbin", "libexec");
- }
-
- File igniteLibs = new File(new File(igniteHome), "libs");
-
- if (!igniteLibs.exists())
- exit("Ignite 'libs' folder is not found.", null);
-
- Collection<File> jarFiles = new ArrayList<>();
-
- addJarsInFolder(jarFiles, igniteLibs);
- addJarsInFolder(jarFiles, new File(igniteLibs, "ignite-hadoop"));
-
- boolean jarsLinksCorrect = true;
-
- for (File file : jarFiles) {
- File link = new File(hadoopCommonLibDir, file.getName());
-
- jarsLinksCorrect &= isJarLinkCorrect(link, file);
-
- if (!jarsLinksCorrect)
- break;
- }
-
- if (!jarsLinksCorrect) {
- if (ask("Ignite JAR files are not found in Hadoop 'lib' directory. " +
- "Create appropriate symbolic links?")) {
- File[] oldIgniteJarFiles = hadoopCommonLibDir.listFiles(IGNITE_JARS);
-
- if (oldIgniteJarFiles.length > 0 && ask("The Hadoop 'lib' directory contains JARs from other Ignite " +
- "installation. They must be deleted to continue. Continue?")) {
- for (File file : oldIgniteJarFiles) {
- println("Deleting file '" + file.getAbsolutePath() + "'.");
-
- if (!file.delete())
- exit("Failed to delete file '" + file.getPath() + "'.", null);
- }
- }
-
- for (File file : jarFiles) {
- File targetFile = new File(hadoopCommonLibDir, file.getName());
-
- try {
- println("Creating symbolic link '" + targetFile.getAbsolutePath() + "'.");
-
- Files.createSymbolicLink(targetFile.toPath(), file.toPath());
- }
- catch (IOException e) {
- if (U.isWindows()) {
- warn("Ability to create symbolic links is required!");
- warn("On Windows platform you have to grant permission 'Create symbolic links'");
- warn("to your user or run the Accelerator as Administrator.");
- }
-
- exit("Creating symbolic link failed! Check permissions.", e);
- }
- }
- }
- else
- println("Ok. But Hadoop client will not be able to talk to Ignite cluster without those JARs in classpath...");
- }
-
- File hadoopEtc = new File(hadoopDir, "etc" + File.separator + "hadoop");
-
- File igniteDocs = new File(igniteHome, "docs");
-
- if (!igniteDocs.canRead())
- exit("Failed to read Ignite 'docs' folder at '" + igniteDocs.getAbsolutePath() + "'.", null);
-
- if (hadoopEtc.canWrite()) { // TODO Bigtop
- if (ask("Replace 'core-site.xml' and 'mapred-site.xml' files with preconfigured templates " +
- "(existing files will be backed up)?")) {
- replaceWithBackup(new File(igniteDocs, "core-site.ignite.xml"), new File(hadoopEtc, "core-site.xml"));
-
- replaceWithBackup(new File(igniteDocs, "mapred-site.ignite.xml"), new File(hadoopEtc, "mapred-site.xml"));
- }
- else
- println("Ok. You can configure them later, the templates are available at Ignite's 'docs' directory...");
- }
-
- if (!F.isEmpty(hiveHome)) {
- File hiveConfDir = new File(hiveHome + File.separator + "conf");
-
- if (!hiveConfDir.canWrite())
- warn("Can not write to '" + hiveConfDir.getAbsolutePath() + "'. To run Hive queries you have to " +
- "configure 'hive-site.xml' manually. The template is available at Ignite's 'docs' directory.");
- else if (ask("Replace 'hive-site.xml' with preconfigured template (existing file will be backed up)?"))
- replaceWithBackup(new File(igniteDocs, "hive-site.ignite.xml"), new File(hiveConfDir, "hive-site.xml"));
- else
- println("Ok. You can configure it later, the template is available at Ignite's 'docs' directory...");
- }
-
- println("Apache Hadoop setup is complete.");
- }
-
- /**
- * @param jarFiles Jars.
- * @param folder Folder.
- */
- private static void addJarsInFolder(Collection<File> jarFiles, File folder) {
- if (!folder.exists())
- exit("Folder '" + folder.getAbsolutePath() + "' is not found.", null);
-
- jarFiles.addAll(Arrays.asList(folder.listFiles(IGNITE_JARS)));
- }
-
- /**
- * Checks that JAVA_HOME does not contain space characters.
- */
- private static void checkJavaPathSpaces() {
- String javaHome = System.getProperty("java.home");
-
- if (javaHome.contains(" ")) {
- warn("Java installation path contains space characters!");
- warn("Hadoop client will not be able to start using '" + javaHome + "'.");
- warn("Please install JRE to path which does not contain spaces and point JAVA_HOME to that installation.");
- }
- }
-
- /**
- * Checks Ignite home.
- *
- * @param igniteHome Ignite home.
- */
- private static void checkIgniteHome(String igniteHome) {
- URL jarUrl = U.class.getProtectionDomain().getCodeSource().getLocation();
-
- try {
- Path jar = Paths.get(jarUrl.toURI());
- Path igHome = Paths.get(igniteHome);
-
- if (!jar.startsWith(igHome))
- exit("Ignite JAR files are not under IGNITE_HOME.", null);
- }
- catch (Exception e) {
- exit(e.getMessage(), e);
- }
- }
-
- /**
- * Replaces target file with source file.
- *
- * @param from From.
- * @param to To.
- */
- private static void replaceWithBackup(File from, File to) {
- if (!from.canRead())
- exit("Failed to read source file '" + from.getAbsolutePath() + "'.", null);
-
- println("Replacing file '" + to.getAbsolutePath() + "'.");
-
- try {
- U.copy(from, renameToBak(to), true);
- }
- catch (IOException e) {
- exit("Failed to replace file '" + to.getAbsolutePath() + "'.", e);
- }
- }
-
- /**
- * Renames file for backup.
- *
- * @param file File.
- * @return File.
- */
- private static File renameToBak(File file) {
- DateFormat fmt = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
-
- if (file.exists() && !file.renameTo(new File(file.getAbsolutePath() + "." + fmt.format(new Date()) + ".bak")))
- exit("Failed to rename file '" + file.getPath() + "'.", null);
-
- return file;
- }
-
- /**
- * Checks if link is correct.
- *
- * @param link Symbolic link.
- * @param correctTarget Correct link target.
- * @return {@code true} If link target is correct.
- */
- private static boolean isJarLinkCorrect(File link, File correctTarget) {
- if (!Files.isSymbolicLink(link.toPath()))
- return false; // It is a real file or it does not exist.
-
- Path target = null;
-
- try {
- target = Files.readSymbolicLink(link.toPath());
- }
- catch (IOException e) {
- exit("Failed to read symbolic link: " + link.getAbsolutePath(), e);
- }
-
- return Files.exists(target) && target.toFile().equals(correctTarget);
- }
-
- /**
- * Writes the question end read the boolean answer from the console.
- *
- * @param question Question to write.
- * @return {@code true} if user inputs 'Y' or 'y', {@code false} otherwise.
- */
- private static boolean ask(String question) {
- X.println();
- X.print(" < " + question + " (Y/N): ");
-
- String answer = null;
-
- if (!F.isEmpty(System.getenv("IGNITE_HADOOP_SETUP_YES")))
- answer = "Y";
- else {
- BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
-
- try {
- answer = br.readLine();
- }
- catch (IOException e) {
- exit("Failed to read answer: " + e.getMessage(), e);
- }
- }
-
- if (answer != null && "Y".equals(answer.toUpperCase().trim())) {
- X.println(" > Yes.");
-
- return true;
- }
- else {
- X.println(" > No.");
-
- return false;
- }
- }
-
- /**
- * Exit with message.
- *
- * @param msg Exit message.
- */
- private static void exit(String msg, Exception e) {
- X.println(" ");
- X.println(" # " + msg);
- X.println(" # Setup failed, exiting... ");
-
- if (e != null && !F.isEmpty(System.getenv("IGNITE_HADOOP_SETUP_DEBUG")))
- e.printStackTrace();
-
- System.exit(1);
- }
-
- /**
- * Prints message.
- *
- * @param msg Message.
- */
- private static void println(String msg) {
- X.println(" > " + msg);
- }
-
- /**
- * Prints warning.
- *
- * @param msg Message.
- */
- private static void warn(String msg) {
- X.println(" ! " + msg);
- }
-
- /**
- * Checks that CMD files have valid MS Windows new line characters. If not, writes question to console and reads the
- * answer. If it's 'Y' then backups original files and corrects invalid new line characters.
- *
- * @param rootDir Root directory to process.
- * @param dirs Directories inside of the root to process.
- */
- private static void processCmdFiles(File rootDir, String... dirs) {
- boolean answer = false;
-
- for (String dir : dirs) {
- File subDir = new File(rootDir, dir);
-
- File[] cmdFiles = subDir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return name.toLowerCase().endsWith(".cmd");
- }
- });
-
- for (File file : cmdFiles) {
- String content = null;
-
- try (Scanner scanner = new Scanner(file)) {
- content = scanner.useDelimiter("\\Z").next();
- }
- catch (FileNotFoundException e) {
- exit("Failed to read file '" + file + "'.", e);
- }
-
- boolean invalid = false;
-
- for (int i = 0; i < content.length(); i++) {
- if (content.charAt(i) == '\n' && (i == 0 || content.charAt(i - 1) != '\r')) {
- invalid = true;
-
- break;
- }
- }
-
- if (invalid) {
- answer = answer || ask("One or more *.CMD files has invalid new line character. Replace them?");
-
- if (!answer) {
- println("Ok. But Windows most probably will fail to execute them...");
-
- return;
- }
-
- println("Fixing newline characters in file '" + file.getAbsolutePath() + "'.");
-
- renameToBak(file);
-
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) {
- for (int i = 0; i < content.length(); i++) {
- if (content.charAt(i) == '\n' && (i == 0 || content.charAt(i - 1) != '\r'))
- writer.write("\r");
-
- writer.write(content.charAt(i));
- }
- }
- catch (IOException e) {
- exit("Failed to write file '" + file.getPath() + "': " + e.getMessage(), e);
- }
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskCancelledException.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskCancelledException.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskCancelledException.java
deleted file mode 100644
index c762181..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskCancelledException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-
-/**
- * Exception that throws when the task is cancelling.
- */
-public class GridHadoopTaskCancelledException extends IgniteException {
- /** */
- private static final long serialVersionUID = 0L;
-
- /**
- * @param msg Exception message.
- */
- public GridHadoopTaskCancelledException(String msg) {
- super(msg);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopUtils.java
deleted file mode 100644
index 763f45a..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopUtils.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.JobPriority;
-import org.apache.hadoop.mapreduce.JobStatus;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Hadoop utility methods.
- */
-public class GridHadoopUtils {
- /** Property to store timestamp of new job id request. */
- public static final String REQ_NEW_JOBID_TS_PROPERTY = "ignite.job.requestNewIdTs";
-
- /** Property to store timestamp of response of new job id request. */
- public static final String RESPONSE_NEW_JOBID_TS_PROPERTY = "ignite.job.responseNewIdTs";
-
- /** Property to store timestamp of job submission. */
- public static final String JOB_SUBMISSION_START_TS_PROPERTY = "ignite.job.submissionStartTs";
-
- /** Property to set custom writer of job statistics. */
- public static final String JOB_COUNTER_WRITER_PROPERTY = "ignite.counters.writer";
-
- /** Staging constant. */
- private static final String STAGING_CONSTANT = ".staging";
-
- /** Old mapper class attribute. */
- private static final String OLD_MAP_CLASS_ATTR = "mapred.mapper.class";
-
- /** Old reducer class attribute. */
- private static final String OLD_REDUCE_CLASS_ATTR = "mapred.reducer.class";
-
- /**
- * Wraps native split.
- *
- * @param id Split ID.
- * @param split Split.
- * @param hosts Hosts.
- * @throws IOException If failed.
- */
- public static GridHadoopSplitWrapper wrapSplit(int id, Object split, String[] hosts) throws IOException {
- ByteArrayOutputStream arr = new ByteArrayOutputStream();
- ObjectOutput out = new ObjectOutputStream(arr);
-
- assert split instanceof Writable;
-
- ((Writable)split).write(out);
-
- out.flush();
-
- return new GridHadoopSplitWrapper(id, split.getClass().getName(), arr.toByteArray(), hosts);
- }
-
- /**
- * Unwraps native split.
- *
- * @param o Wrapper.
- * @return Split.
- */
- public static Object unwrapSplit(GridHadoopSplitWrapper o) {
- try {
- Writable w = (Writable)GridHadoopUtils.class.getClassLoader().loadClass(o.className()).newInstance();
-
- w.readFields(new ObjectInputStream(new ByteArrayInputStream(o.bytes())));
-
- return w;
- }
- catch (Exception e) {
- throw new IllegalStateException(e);
- }
- }
-
- /**
- * Convert Ignite job status to Hadoop job status.
- *
- * @param status Ignite job status.
- * @return Hadoop job status.
- */
- public static JobStatus status(GridHadoopJobStatus status, Configuration conf) {
- JobID jobId = new JobID(status.jobId().globalId().toString(), status.jobId().localId());
-
- float setupProgress = 0;
- float mapProgress = 0;
- float reduceProgress = 0;
- float cleanupProgress = 0;
-
- JobStatus.State state = JobStatus.State.RUNNING;
-
- switch (status.jobPhase()) {
- case PHASE_SETUP:
- setupProgress = 0.42f;
-
- break;
-
- case PHASE_MAP:
- setupProgress = 1;
- mapProgress = 1f - status.pendingMapperCnt() / (float)status.totalMapperCnt();
-
- break;
-
- case PHASE_REDUCE:
- assert status.totalReducerCnt() > 0;
-
- setupProgress = 1;
- mapProgress = 1;
- reduceProgress = 1f - status.pendingReducerCnt() / (float)status.totalReducerCnt();
-
- break;
-
- case PHASE_CANCELLING:
- case PHASE_COMPLETE:
- if (!status.isFailed()) {
- setupProgress = 1;
- mapProgress = 1;
- reduceProgress = 1;
- cleanupProgress = 1;
-
- state = JobStatus.State.SUCCEEDED;
- }
- else
- state = JobStatus.State.FAILED;
-
- break;
-
- default:
- assert false;
- }
-
- return new JobStatus(jobId, setupProgress, mapProgress, reduceProgress, cleanupProgress, state,
- JobPriority.NORMAL, status.user(), status.jobName(), jobFile(conf, status.user(), jobId).toString(), "N/A");
- }
-
- /**
- * Gets staging area directory.
- *
- * @param conf Configuration.
- * @param usr User.
- * @return Staging area directory.
- */
- public static Path stagingAreaDir(Configuration conf, String usr) {
- return new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR, MRJobConfig.DEFAULT_MR_AM_STAGING_DIR)
- + Path.SEPARATOR + usr + Path.SEPARATOR + STAGING_CONSTANT);
- }
-
- /**
- * Gets job file.
- *
- * @param conf Configuration.
- * @param usr User.
- * @param jobId Job ID.
- * @return Job file.
- */
- public static Path jobFile(Configuration conf, String usr, JobID jobId) {
- return new Path(stagingAreaDir(conf, usr), jobId.toString() + Path.SEPARATOR + MRJobConfig.JOB_CONF_FILE);
- }
-
- /**
- * Checks the attribute in configuration is not set.
- *
- * @param attr Attribute name.
- * @param msg Message for creation of exception.
- * @throws IgniteCheckedException If attribute is set.
- */
- public static void ensureNotSet(Configuration cfg, String attr, String msg) throws IgniteCheckedException {
- if (cfg.get(attr) != null)
- throw new IgniteCheckedException(attr + " is incompatible with " + msg + " mode.");
- }
-
- /**
- * Creates JobInfo from hadoop configuration.
- *
- * @param cfg Hadoop configuration.
- * @return Job info.
- * @throws IgniteCheckedException If failed.
- */
- public static GridHadoopDefaultJobInfo createJobInfo(Configuration cfg) throws IgniteCheckedException {
- JobConf jobConf = new JobConf(cfg);
-
- boolean hasCombiner = jobConf.get("mapred.combiner.class") != null
- || jobConf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null;
-
- int numReduces = jobConf.getNumReduceTasks();
-
- jobConf.setBooleanIfUnset("mapred.mapper.new-api", jobConf.get(OLD_MAP_CLASS_ATTR) == null);
-
- if (jobConf.getUseNewMapper()) {
- String mode = "new map API";
-
- ensureNotSet(jobConf, "mapred.input.format.class", mode);
- ensureNotSet(jobConf, OLD_MAP_CLASS_ATTR, mode);
-
- if (numReduces != 0)
- ensureNotSet(jobConf, "mapred.partitioner.class", mode);
- else
- ensureNotSet(jobConf, "mapred.output.format.class", mode);
- }
- else {
- String mode = "map compatibility";
-
- ensureNotSet(jobConf, MRJobConfig.INPUT_FORMAT_CLASS_ATTR, mode);
- ensureNotSet(jobConf, MRJobConfig.MAP_CLASS_ATTR, mode);
-
- if (numReduces != 0)
- ensureNotSet(jobConf, MRJobConfig.PARTITIONER_CLASS_ATTR, mode);
- else
- ensureNotSet(jobConf, MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, mode);
- }
-
- if (numReduces != 0) {
- jobConf.setBooleanIfUnset("mapred.reducer.new-api", jobConf.get(OLD_REDUCE_CLASS_ATTR) == null);
-
- if (jobConf.getUseNewReducer()) {
- String mode = "new reduce API";
-
- ensureNotSet(jobConf, "mapred.output.format.class", mode);
- ensureNotSet(jobConf, OLD_REDUCE_CLASS_ATTR, mode);
- }
- else {
- String mode = "reduce compatibility";
-
- ensureNotSet(jobConf, MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, mode);
- ensureNotSet(jobConf, MRJobConfig.REDUCE_CLASS_ATTR, mode);
- }
- }
-
- Map<String, String> props = new HashMap<>();
-
- for (Map.Entry<String, String> entry : jobConf)
- props.put(entry.getKey(), entry.getValue());
-
- return new GridHadoopDefaultJobInfo(jobConf.getJobName(), jobConf.getUser(), hasCombiner, numReduces, props);
- }
-
- /**
- * Throws new {@link IgniteCheckedException} with original exception is serialized into string.
- * This is needed to transfer error outside the current class loader.
- *
- * @param e Original exception.
- * @return IgniteCheckedException New exception.
- */
- public static IgniteCheckedException transformException(Throwable e) {
- ByteArrayOutputStream os = new ByteArrayOutputStream();
-
- e.printStackTrace(new PrintStream(os, true));
-
- return new IgniteCheckedException(os.toString());
- }
-
- /**
- * Returns work directory for job execution.
- *
- * @param locNodeId Local node ID.
- * @param jobId Job ID.
- * @return Working directory for job.
- * @throws IgniteCheckedException If Failed.
- */
- public static File jobLocalDir(UUID locNodeId, GridHadoopJobId jobId) throws IgniteCheckedException {
- return new File(new File(U.resolveWorkDirectory("hadoop", false), "node-" + locNodeId), "job_" + jobId);
- }
-
- /**
- * Returns subdirectory of job working directory for task execution.
- *
- * @param locNodeId Local node ID.
- * @param info Task info.
- * @return Working directory for task.
- * @throws IgniteCheckedException If Failed.
- */
- public static File taskLocalDir(UUID locNodeId, GridHadoopTaskInfo info) throws IgniteCheckedException {
- File jobLocDir = jobLocalDir(locNodeId, info.jobId());
-
- return new File(jobLocDir, info.type() + "_" + info.taskNumber() + "_" + info.attempt());
- }
-
- /**
- * Constructor.
- */
- private GridHadoopUtils() {
- // No-op.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
new file mode 100644
index 0000000..1856e41
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -0,0 +1,552 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.jdk8.backport.*;
+import org.jetbrains.annotations.*;
+import org.objectweb.asm.*;
+import org.objectweb.asm.commons.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.concurrent.atomic.*;
+
+/**
+ * Class loader allowing explicitly load classes without delegation to parent class loader.
+ * Also supports class parsing for finding dependencies which contain transitive dependencies
+ * unavailable for parent.
+ */
+public class HadoopClassLoader extends URLClassLoader {
+ /**
+ * We are very parallel capable.
+ */
+ static {
+ registerAsParallelCapable();
+ }
+
+ /** */
+ private static final URLClassLoader APP_CLS_LDR = (URLClassLoader)HadoopClassLoader.class.getClassLoader();
+
+ /** */
+ private static final Collection<URL> appJars = F.asList(APP_CLS_LDR.getURLs());
+
+ /** */
+ private static volatile Collection<URL> hadoopJars;
+
+ /** */
+ private static final Map<String, Boolean> cache = new ConcurrentHashMap8<>();
+
+ /** */
+ private static final Map<String, byte[]> bytesCache = new ConcurrentHashMap8<>();
+
+ /**
+ * @param urls Urls.
+ */
+ public HadoopClassLoader(URL[] urls) {
+ super(addHadoopUrls(urls), APP_CLS_LDR);
+
+ assert !(getParent() instanceof HadoopClassLoader);
+ }
+
+ /**
+ * Need to parse only Ignite Hadoop and IGFS classes.
+ *
+ * @param cls Class name.
+ * @return {@code true} if we need to check this class.
+ */
+ private static boolean isHadoopIgfs(String cls) {
+ String ignitePackagePrefix = "org.apache.ignite";
+ int len = ignitePackagePrefix.length();
+
+ return cls.startsWith(ignitePackagePrefix) && (cls.indexOf("igfs.", len) != -1 || cls.indexOf(".fs.", len) != -1 || cls.indexOf("hadoop.", len) != -1);
+ }
+
+ /**
+ * @param cls Class name.
+ * @return {@code true} If this is Hadoop class.
+ */
+ private static boolean isHadoop(String cls) {
+ return cls.startsWith("org.apache.hadoop.");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+ try {
+ if (isHadoop(name)) { // Always load Hadoop classes explicitly, since Hadoop can be available in App classpath.
+ if (name.endsWith(".util.ShutdownHookManager")) // Dirty hack to get rid of Hadoop shutdown hooks.
+ return loadFromBytes(name, HadoopShutdownHookManager.class.getName());
+ else if (name.endsWith(".util.NativeCodeLoader"))
+ return loadFromBytes(name, HadoopNativeCodeLoader.class.getName());
+
+ return loadClassExplicitly(name, resolve);
+ }
+
+ if (isHadoopIgfs(name)) { // For Ignite Hadoop and IGFS classes we have to check if they depend on Hadoop.
+ Boolean hasDeps = cache.get(name);
+
+ if (hasDeps == null) {
+ hasDeps = hasExternalDependencies(name, new HashSet<String>());
+
+ cache.put(name, hasDeps);
+ }
+
+ if (hasDeps)
+ return loadClassExplicitly(name, resolve);
+ }
+
+ return super.loadClass(name, resolve);
+ }
+ catch (NoClassDefFoundError | ClassNotFoundException e) {
+ throw new ClassNotFoundException("Failed to load class: " + name, e);
+ }
+ }
+
+ /**
+ * @param name Name.
+ * @param replace Replacement.
+ * @return Class.
+ */
+ private Class<?> loadFromBytes(final String name, final String replace) {
+ synchronized (getClassLoadingLock(name)) {
+ // First, check if the class has already been loaded
+ Class c = findLoadedClass(name);
+
+ if (c != null)
+ return c;
+
+ byte[] bytes = bytesCache.get(name);
+
+ if (bytes == null) {
+ InputStream in = loadClassBytes(getParent(), replace);
+
+ ClassReader rdr;
+
+ try {
+ rdr = new ClassReader(in);
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ ClassWriter w = new ClassWriter(Opcodes.ASM4);
+
+ rdr.accept(new RemappingClassAdapter(w, new Remapper() {
+ /** */
+ String replaceType = replace.replace('.', '/');
+
+ /** */
+ String nameType = name.replace('.', '/');
+
+ @Override public String map(String type) {
+ if (type.equals(replaceType))
+ return nameType;
+
+ return type;
+ }
+ }), ClassReader.EXPAND_FRAMES);
+
+ bytes = w.toByteArray();
+
+ bytesCache.put(name, bytes);
+ }
+
+ return defineClass(name, bytes, 0, bytes.length);
+ }
+ }
+
+ /**
+ * @param name Class name.
+ * @param resolve Resolve class.
+ * @return Class.
+ * @throws ClassNotFoundException If failed.
+ */
+ private Class<?> loadClassExplicitly(String name, boolean resolve) throws ClassNotFoundException {
+ synchronized (getClassLoadingLock(name)) {
+ // First, check if the class has already been loaded
+ Class c = findLoadedClass(name);
+
+ if (c == null) {
+ long t1 = System.nanoTime();
+
+ c = findClass(name);
+
+ // this is the defining class loader; record the stats
+ sun.misc.PerfCounter.getFindClassTime().addElapsedTimeFrom(t1);
+ sun.misc.PerfCounter.getFindClasses().increment();
+ }
+
+ if (resolve)
+ resolveClass(c);
+
+ return c;
+ }
+ }
+
+ /**
+ * @param ldr Loader.
+ * @param clsName Class.
+ * @return Input stream.
+ */
+ @Nullable private InputStream loadClassBytes(ClassLoader ldr, String clsName) {
+ return ldr.getResourceAsStream(clsName.replace('.', '/') + ".class");
+ }
+
+ /**
+ * @param clsName Class name.
+ * @return {@code true} If the class has external dependencies.
+ */
+ boolean hasExternalDependencies(final String clsName, final Set<String> visited) {
+ if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea sucks, so filtering explicitly as external.
+ return true;
+
+ // Try to get from parent to check if the type accessible.
+ InputStream in = loadClassBytes(getParent(), clsName);
+
+ if (in == null) // The class is external itself, it must be loaded from this class loader.
+ return true;
+
+ if (!isHadoopIgfs(clsName)) // Other classes should not have external dependencies.
+ return false;
+
+ final ClassReader rdr;
+
+ try {
+ rdr = new ClassReader(in);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Failed to read class: " + clsName, e);
+ }
+
+ visited.add(clsName);
+
+ final AtomicBoolean hasDeps = new AtomicBoolean();
+
+ rdr.accept(new ClassVisitor(Opcodes.ASM4) {
+ AnnotationVisitor av = new AnnotationVisitor(Opcodes.ASM4) {
+ // TODO
+ };
+
+ FieldVisitor fv = new FieldVisitor(Opcodes.ASM4) {
+ @Override public AnnotationVisitor visitAnnotation(String desc, boolean b) {
+ onType(desc);
+
+ return av;
+ }
+ };
+
+ MethodVisitor mv = new MethodVisitor(Opcodes.ASM4) {
+ @Override public AnnotationVisitor visitAnnotation(String desc, boolean b) {
+ onType(desc);
+
+ return av;
+ }
+
+ @Override public AnnotationVisitor visitParameterAnnotation(int i, String desc, boolean b) {
+ onType(desc);
+
+ return av;
+ }
+
+ @Override public AnnotationVisitor visitAnnotationDefault() {
+ return av;
+ }
+
+ @Override public void visitFieldInsn(int i, String owner, String name, String desc) {
+ onType(owner);
+ onType(desc);
+ }
+
+ @Override public void visitFrame(int i, int i2, Object[] locTypes, int i3, Object[] stackTypes) {
+ for (Object o : locTypes) {
+ if (o instanceof String)
+ onType((String)o);
+ }
+
+ for (Object o : stackTypes) {
+ if (o instanceof String)
+ onType((String)o);
+ }
+ }
+
+ @Override public void visitLocalVariable(String name, String desc, String signature, Label lb,
+ Label lb2, int i) {
+ onType(desc);
+ }
+
+ @Override public void visitMethodInsn(int i, String owner, String name, String desc) {
+ onType(owner);
+ }
+
+ @Override public void visitMultiANewArrayInsn(String desc, int dim) {
+ onType(desc);
+ }
+
+ @Override public void visitTryCatchBlock(Label lb, Label lb2, Label lb3, String e) {
+ onType(e);
+ }
+ };
+
+ void onClass(String depCls) {
+ assert validateClassName(depCls) : depCls;
+
+ if (depCls.startsWith("java.")) // Filter out platform classes.
+ return;
+
+ if (visited.contains(depCls))
+ return;
+
+ Boolean res = cache.get(depCls);
+
+ if (res == Boolean.TRUE || (res == null && hasExternalDependencies(depCls, visited)))
+ hasDeps.set(true);
+ }
+
+ void onType(String type) {
+ if (type == null)
+ return;
+
+ int off = 0;
+
+ while (type.charAt(off) == '[')
+ off++; // Handle arrays.
+
+ if (off != 0)
+ type = type.substring(off);
+
+ if (type.length() == 1)
+ return; // Get rid of primitives.
+
+ if (type.charAt(type.length() - 1) == ';') {
+ assert type.charAt(0) == 'L' : type;
+
+ type = type.substring(1, type.length() - 1);
+ }
+
+ type = type.replace('/', '.');
+
+ onClass(type);
+ }
+
+ @Override public void visit(int i, int i2, String name, String signature, String superName,
+ String[] ifaces) {
+ onType(superName);
+
+ if (ifaces != null) {
+ for (String iface : ifaces)
+ onType(iface);
+ }
+ }
+
+ @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
+ onType(desc);
+
+ return av;
+ }
+
+ @Override public void visitInnerClass(String name, String outerName, String innerName, int i) {
+ onType(name);
+ }
+
+ @Override public FieldVisitor visitField(int i, String name, String desc, String signature, Object val) {
+ onType(desc);
+
+ return fv;
+ }
+
+ @Override public MethodVisitor visitMethod(int i, String name, String desc, String signature,
+ String[] exceptions) {
+ if (exceptions != null) {
+ for (String e : exceptions)
+ onType(e);
+ }
+
+ return mv;
+ }
+ }, 0);
+
+ if (hasDeps.get()) // We already know that we have dependencies, no need to check parent.
+ return true;
+
+ // Here we are known to not have any dependencies but possibly we have a parent which have them.
+ int idx = clsName.lastIndexOf('$');
+
+ if (idx == -1) // No parent class.
+ return false;
+
+ String parentCls = clsName.substring(0, idx);
+
+ if (visited.contains(parentCls))
+ return false;
+
+ Boolean res = cache.get(parentCls);
+
+ if (res == null)
+ res = hasExternalDependencies(parentCls, visited);
+
+ return res;
+ }
+
+ /**
+ * @param name Class name.
+ * @return {@code true} If this is a valid class name.
+ */
+ private static boolean validateClassName(String name) {
+ int len = name.length();
+
+ if (len <= 1)
+ return false;
+
+ if (!Character.isJavaIdentifierStart(name.charAt(0)))
+ return false;
+
+ boolean hasDot = false;
+
+ for (int i = 1; i < len; i++) {
+ char c = name.charAt(i);
+
+ if (c == '.')
+ hasDot = true;
+ else if (!Character.isJavaIdentifierPart(c))
+ return false;
+ }
+
+ return hasDot;
+ }
+
+ /**
+ * @param name Variable name.
+ * @param dflt Default.
+ * @return Value.
+ */
+ private static String getEnv(String name, String dflt) {
+ String res = System.getProperty(name);
+
+ if (F.isEmpty(res))
+ res = System.getenv(name);
+
+ return F.isEmpty(res) ? dflt : res;
+ }
+
+ /**
+ * @param res Result.
+ * @param dir Directory.
+ * @param startsWith Starts with prefix.
+ * @throws MalformedURLException If failed.
+ */
+ private static void addUrls(Collection<URL> res, File dir, final String startsWith) throws Exception {
+ File[] files = dir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return startsWith == null || name.startsWith(startsWith);
+ }
+ });
+
+ if (files == null)
+ throw new IOException("Path is not a directory: " + dir);
+
+ for (File file : files)
+ res.add(file.toURI().toURL());
+ }
+
+ /**
+ * @param urls URLs.
+ * @return URLs.
+ */
+ private static URL[] addHadoopUrls(URL[] urls) {
+ Collection<URL> hadoopJars;
+
+ try {
+ hadoopJars = hadoopUrls();
+ }
+ catch (IgniteCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ ArrayList<URL> list = new ArrayList<>(hadoopJars.size() + appJars.size() + (urls == null ? 0 : urls.length));
+
+ list.addAll(appJars);
+ list.addAll(hadoopJars);
+
+ if (!F.isEmpty(urls))
+ list.addAll(F.asList(urls));
+
+ return list.toArray(new URL[list.size()]);
+ }
+
+ /**
+ * @return HADOOP_HOME Variable.
+ */
+ @Nullable public static String hadoopHome() {
+ return getEnv("HADOOP_PREFIX", getEnv("HADOOP_HOME", null));
+ }
+
+ /**
+ * @return Collection of jar URLs.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static Collection<URL> hadoopUrls() throws IgniteCheckedException {
+ Collection<URL> hadoopUrls = hadoopJars;
+
+ if (hadoopUrls != null)
+ return hadoopUrls;
+
+ synchronized (HadoopClassLoader.class) {
+ hadoopUrls = hadoopJars;
+
+ if (hadoopUrls != null)
+ return hadoopUrls;
+
+ hadoopUrls = new ArrayList<>();
+
+ String hadoopPrefix = hadoopHome();
+
+ if (F.isEmpty(hadoopPrefix))
+ throw new IgniteCheckedException("Failed resolve Hadoop installation location. Either HADOOP_PREFIX or " +
+ "HADOOP_HOME environment variables must be set.");
+
+ String commonHome = getEnv("HADOOP_COMMON_HOME", hadoopPrefix + "/share/hadoop/common");
+ String hdfsHome = getEnv("HADOOP_HDFS_HOME", hadoopPrefix + "/share/hadoop/hdfs");
+ String mapredHome = getEnv("HADOOP_MAPRED_HOME", hadoopPrefix + "/share/hadoop/mapreduce");
+
+ try {
+ addUrls(hadoopUrls, new File(commonHome + "/lib"), null);
+ addUrls(hadoopUrls, new File(hdfsHome + "/lib"), null);
+ addUrls(hadoopUrls, new File(mapredHome + "/lib"), null);
+
+ addUrls(hadoopUrls, new File(hdfsHome), "hadoop-hdfs-");
+
+ addUrls(hadoopUrls, new File(commonHome), "hadoop-common-");
+ addUrls(hadoopUrls, new File(commonHome), "hadoop-auth-");
+ addUrls(hadoopUrls, new File(commonHome + "/lib"), "hadoop-auth-");
+
+ addUrls(hadoopUrls, new File(mapredHome), "hadoop-mapreduce-client-common");
+ addUrls(hadoopUrls, new File(mapredHome), "hadoop-mapreduce-client-core");
+ }
+ catch (Exception e) {
+ throw new IgniteCheckedException(e);
+ }
+
+ hadoopJars = hadoopUrls;
+
+ return hadoopUrls;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopComponent.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopComponent.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopComponent.java
new file mode 100644
index 0000000..cea11eb
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopComponent.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+
+/**
+ * Abstract class for all hadoop components.
+ */
+public abstract class HadoopComponent {
+ /** Hadoop context. */
+ protected HadoopContext ctx;
+
+ /** Logger. */
+ protected IgniteLogger log;
+
+ /**
+ * @param ctx Hadoop context.
+ */
+ public void start(HadoopContext ctx) throws IgniteCheckedException {
+ this.ctx = ctx;
+
+ log = ctx.kernalContext().log(getClass());
+ }
+
+ /**
+ * Stops manager.
+ */
+ public void stop(boolean cancel) {
+ // No-op.
+ }
+
+ /**
+ * Callback invoked when all grid components are started.
+ */
+ public void onKernalStart() throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /**
+ * Callback invoked before all grid components are stopped.
+ */
+ public void onKernalStop(boolean cancel) {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopContext.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopContext.java
new file mode 100644
index 0000000..68f0baf
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopContext.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.cluster.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
+import org.apache.ignite.internal.processors.hadoop.shuffle.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.util.*;
+
+/**
+ * Hadoop accelerator context.
+ */
+public class HadoopContext {
+ /** Kernal context. */
+ private GridKernalContext ctx;
+
+ /** Hadoop configuration. */
+ private HadoopConfiguration cfg;
+
+ /** Job tracker. */
+ private HadoopJobTracker jobTracker;
+
+ /** External task executor. */
+ private HadoopTaskExecutorAdapter taskExecutor;
+
+ /** */
+ private HadoopShuffle shuffle;
+
+ /** Managers list. */
+ private List<HadoopComponent> components = new ArrayList<>();
+
+ /**
+ * @param ctx Kernal context.
+ */
+ public HadoopContext(
+ GridKernalContext ctx,
+ HadoopConfiguration cfg,
+ HadoopJobTracker jobTracker,
+ HadoopTaskExecutorAdapter taskExecutor,
+ HadoopShuffle shuffle
+ ) {
+ this.ctx = ctx;
+ this.cfg = cfg;
+
+ this.jobTracker = add(jobTracker);
+ this.taskExecutor = add(taskExecutor);
+ this.shuffle = add(shuffle);
+ }
+
+ /**
+ * Gets list of managers.
+ *
+ * @return List of managers.
+ */
+ public List<HadoopComponent> components() {
+ return components;
+ }
+
+ /**
+ * Gets kernal context.
+ *
+ * @return Grid kernal context instance.
+ */
+ public GridKernalContext kernalContext() {
+ return ctx;
+ }
+
+ /**
+ * Gets Hadoop configuration.
+ *
+ * @return Hadoop configuration.
+ */
+ public HadoopConfiguration configuration() {
+ return cfg;
+ }
+
+ /**
+ * Gets local node ID. Shortcut for {@code kernalContext().localNodeId()}.
+ *
+ * @return Local node ID.
+ */
+ public UUID localNodeId() {
+ return ctx.localNodeId();
+ }
+
+ /**
+ * Gets local node order.
+ *
+ * @return Local node order.
+ */
+ public long localNodeOrder() {
+ assert ctx.discovery() != null;
+
+ return ctx.discovery().localNode().order();
+ }
+
+ /**
+ * @return Hadoop-enabled nodes.
+ */
+ public Collection<ClusterNode> nodes() {
+ return ctx.discovery().cacheNodes(CU.SYS_CACHE_HADOOP_MR, ctx.discovery().topologyVersion());
+ }
+
+ /**
+ * @return {@code True} if
+ */
+ public boolean jobUpdateLeader() {
+ long minOrder = Long.MAX_VALUE;
+ ClusterNode minOrderNode = null;
+
+ for (ClusterNode node : nodes()) {
+ if (node.order() < minOrder) {
+ minOrder = node.order();
+ minOrderNode = node;
+ }
+ }
+
+ assert minOrderNode != null;
+
+ return localNodeId().equals(minOrderNode.id());
+ }
+
+ /**
+ * @param meta Job metadata.
+ * @return {@code true} If local node is participating in job execution.
+ */
+ public boolean isParticipating(HadoopJobMetadata meta) {
+ UUID locNodeId = localNodeId();
+
+ if (locNodeId.equals(meta.submitNodeId()))
+ return true;
+
+ HadoopMapReducePlan plan = meta.mapReducePlan();
+
+ return plan.mapperNodeIds().contains(locNodeId) || plan.reducerNodeIds().contains(locNodeId) || jobUpdateLeader();
+ }
+
+ /**
+ * @return Jon tracker instance.
+ */
+ public HadoopJobTracker jobTracker() {
+ return jobTracker;
+ }
+
+ /**
+ * @return Task executor.
+ */
+ public HadoopTaskExecutorAdapter taskExecutor() {
+ return taskExecutor;
+ }
+
+ /**
+ * @return Shuffle.
+ */
+ public HadoopShuffle shuffle() {
+ return shuffle;
+ }
+
+ /**
+ * @return Map-reduce planner.
+ */
+ public HadoopMapReducePlanner planner() {
+ return cfg.getMapReducePlanner();
+ }
+
+ /**
+ * Adds component.
+ *
+ * @param c Component to add.
+ * @return Added manager.
+ */
+ private <C extends HadoopComponent> C add(C c) {
+ components.add(c);
+
+ return c;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
new file mode 100644
index 0000000..77eb6d2
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.lang.reflect.*;
+import java.util.*;
+
+/**
+ * Hadoop job info based on default Hadoop configuration.
+ */
+public class HadoopDefaultJobInfo implements HadoopJobInfo, Externalizable {
+ /** */
+ private static final long serialVersionUID = 5489900236464999951L;
+
+ /** {@code true} If job has combiner. */
+ private boolean hasCombiner;
+
+ /** Number of reducers configured for job. */
+ private int numReduces;
+
+ /** Configuration. */
+ private Map<String,String> props = new HashMap<>();
+
+ /** Job name. */
+ private String jobName;
+
+ /** User name. */
+ private String user;
+
+ /** */
+ private static volatile Class<?> jobCls;
+
+ /**
+ * Default constructor required by {@link Externalizable}.
+ */
+ public HadoopDefaultJobInfo() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param jobName Job name.
+ * @param user User name.
+ * @param hasCombiner {@code true} If job has combiner.
+ * @param numReduces Number of reducers configured for job.
+ * @param props All other properties of the job.
+ */
+ public HadoopDefaultJobInfo(String jobName, String user, boolean hasCombiner, int numReduces,
+ Map<String, String> props) {
+ this.jobName = jobName;
+ this.user = user;
+ this.hasCombiner = hasCombiner;
+ this.numReduces = numReduces;
+ this.props = props;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public String property(String name) {
+ return props.get(name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopJob createJob(HadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException {
+ try {
+ Class<?> jobCls0 = jobCls;
+
+ if (jobCls0 == null) { // It is enough to have only one class loader with only Hadoop classes.
+ synchronized (HadoopDefaultJobInfo.class) {
+ if ((jobCls0 = jobCls) == null) {
+ HadoopClassLoader ldr = new HadoopClassLoader(null);
+
+ jobCls = jobCls0 = ldr.loadClass(HadoopV2Job.class.getName());
+ }
+ }
+ }
+
+ Constructor<?> constructor = jobCls0.getConstructor(HadoopJobId.class, HadoopDefaultJobInfo.class,
+ IgniteLogger.class);
+
+ return (HadoopJob)constructor.newInstance(jobId, this, log);
+ }
+ // NB: java.lang.NoClassDefFoundError may be thrown from Class#getConstructor() call.
+ catch (Throwable t) {
+ throw new IgniteCheckedException(t);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean hasCombiner() {
+ return hasCombiner;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean hasReducer() {
+ return reducers() > 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int reducers() {
+ return numReduces;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String jobName() {
+ return jobName;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String user() {
+ return user;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ U.writeString(out, jobName);
+ U.writeString(out, user);
+
+ out.writeBoolean(hasCombiner);
+ out.writeInt(numReduces);
+
+ U.writeStringMap(out, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobName = U.readString(in);
+ user = U.readString(in);
+
+ hasCombiner = in.readBoolean();
+ numReduces = in.readInt();
+
+ props = U.readStringMap(in);
+ }
+
+ /**
+ * @return Properties of the job.
+ */
+ public Map<String, String> properties() {
+ return props;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopImpl.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopImpl.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopImpl.java
new file mode 100644
index 0000000..27542a1
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopImpl.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
+import org.apache.ignite.internal.util.*;
+import org.jetbrains.annotations.*;
+
+/**
+ * Hadoop facade implementation.
+ */
+public class HadoopImpl implements Hadoop {
+ /** Hadoop processor. */
+ private final HadoopProcessor proc;
+
+ /** Busy lock. */
+ private final GridSpinBusyLock busyLock = new GridSpinBusyLock();
+
+ /**
+ * Constructor.
+ *
+ * @param proc Hadoop processor.
+ */
+ HadoopImpl(HadoopProcessor proc) {
+ this.proc = proc;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopConfiguration configuration() {
+ return proc.config();
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopJobId nextJobId() {
+ if (busyLock.enterBusy()) {
+ try {
+ return proc.nextJobId();
+ }
+ finally {
+ busyLock.leaveBusy();
+ }
+ }
+ else
+ throw new IllegalStateException("Failed to get next job ID (grid is stopping).");
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteInternalFuture<?> submit(HadoopJobId jobId, HadoopJobInfo jobInfo) {
+ if (busyLock.enterBusy()) {
+ try {
+ return proc.submit(jobId, jobInfo);
+ }
+ finally {
+ busyLock.leaveBusy();
+ }
+ }
+ else
+ throw new IllegalStateException("Failed to submit job (grid is stopping).");
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public HadoopJobStatus status(HadoopJobId jobId) throws IgniteCheckedException {
+ if (busyLock.enterBusy()) {
+ try {
+ return proc.status(jobId);
+ }
+ finally {
+ busyLock.leaveBusy();
+ }
+ }
+ else
+ throw new IllegalStateException("Failed to get job status (grid is stopping).");
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public HadoopCounters counters(HadoopJobId jobId) throws IgniteCheckedException {
+ if (busyLock.enterBusy()) {
+ try {
+ return proc.counters(jobId);
+ }
+ finally {
+ busyLock.leaveBusy();
+ }
+ }
+ else
+ throw new IllegalStateException("Failed to get job counters (grid is stopping).");
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public IgniteInternalFuture<?> finishFuture(HadoopJobId jobId) throws IgniteCheckedException {
+ if (busyLock.enterBusy()) {
+ try {
+ return proc.finishFuture(jobId);
+ }
+ finally {
+ busyLock.leaveBusy();
+ }
+ }
+ else
+ throw new IllegalStateException("Failed to get job finish future (grid is stopping).");
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean kill(HadoopJobId jobId) throws IgniteCheckedException {
+ if (busyLock.enterBusy()) {
+ try {
+ return proc.kill(jobId);
+ }
+ finally {
+ busyLock.leaveBusy();
+ }
+ }
+ else
+ throw new IllegalStateException("Failed to kill job (grid is stopping).");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceCounterGroup.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceCounterGroup.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceCounterGroup.java
new file mode 100644
index 0000000..b0c2d3e
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceCounterGroup.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.counters.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Hadoop +counter group adapter.
+ */
+class HadoopMapReduceCounterGroup implements CounterGroup {
+ /** Counters. */
+ private final HadoopMapReduceCounters cntrs;
+
+ /** Group name. */
+ private final String name;
+
+ /**
+ * Creates new instance.
+ *
+ * @param cntrs Client counters instance.
+ * @param name Group name.
+ */
+ HadoopMapReduceCounterGroup(HadoopMapReduceCounters cntrs, String name) {
+ this.cntrs = cntrs;
+ this.name = name;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getName() {
+ return name;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getDisplayName() {
+ return name;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setDisplayName(String displayName) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void addCounter(Counter counter) {
+ addCounter(counter.getName(), counter.getDisplayName(), 0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Counter addCounter(String name, String displayName, long value) {
+ final Counter counter = cntrs.findCounter(this.name, name);
+
+ counter.setValue(value);
+
+ return counter;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Counter findCounter(String counterName, String displayName) {
+ return cntrs.findCounter(name, counterName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Counter findCounter(String counterName, boolean create) {
+ return cntrs.findCounter(name, counterName, create);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Counter findCounter(String counterName) {
+ return cntrs.findCounter(name, counterName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int size() {
+ return cntrs.groupSize(name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void incrAllCounters(CounterGroupBase<Counter> rightGroup) {
+ for (final Counter counter : rightGroup)
+ cntrs.findCounter(name, counter.getName()).increment(counter.getValue());
+ }
+
+ /** {@inheritDoc} */
+ @Override public CounterGroupBase<Counter> getUnderlyingGroup() {
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Iterator<Counter> iterator() {
+ return cntrs.iterateGroup(name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(DataOutput out) throws IOException {
+ throw new UnsupportedOperationException("not implemented");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFields(DataInput in) throws IOException {
+ throw new UnsupportedOperationException("not implemented");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceCounters.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceCounters.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceCounters.java
new file mode 100644
index 0000000..c2c9e2a
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceCounters.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.counters.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+import org.apache.ignite.internal.util.typedef.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Hadoop counters adapter.
+ */
+public class HadoopMapReduceCounters extends Counters {
+ /** */
+ private final Map<T2<String,String>,HadoopLongCounter> cntrs = new HashMap<>();
+
+ /**
+ * Creates new instance based on given counters.
+ *
+ * @param cntrs Counters to adapt.
+ */
+ public HadoopMapReduceCounters(org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters cntrs) {
+ for (HadoopCounter cntr : cntrs.all())
+ if (cntr instanceof HadoopLongCounter)
+ this.cntrs.put(new T2<>(cntr.group(), cntr.name()), (HadoopLongCounter) cntr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized CounterGroup addGroup(CounterGroup grp) {
+ return addGroup(grp.getName(), grp.getDisplayName());
+ }
+
+ /** {@inheritDoc} */
+ @Override public CounterGroup addGroup(String name, String displayName) {
+ return new HadoopMapReduceCounterGroup(this, name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Counter findCounter(String grpName, String cntrName) {
+ return findCounter(grpName, cntrName, true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized Counter findCounter(Enum<?> key) {
+ return findCounter(key.getDeclaringClass().getName(), key.name(), true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized Counter findCounter(String scheme, FileSystemCounter key) {
+ return findCounter(String.format("FileSystem Counter (%s)", scheme), key.name());
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized Iterable<String> getGroupNames() {
+ Collection<String> res = new HashSet<>();
+
+ for (HadoopCounter counter : cntrs.values())
+ res.add(counter.group());
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Iterator<CounterGroup> iterator() {
+ final Iterator<String> iter = getGroupNames().iterator();
+
+ return new Iterator<CounterGroup>() {
+ @Override public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override public CounterGroup next() {
+ if (!hasNext())
+ throw new NoSuchElementException();
+
+ return new HadoopMapReduceCounterGroup(HadoopMapReduceCounters.this, iter.next());
+ }
+
+ @Override public void remove() {
+ throw new UnsupportedOperationException("not implemented");
+ }
+ };
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized CounterGroup getGroup(String grpName) {
+ return new HadoopMapReduceCounterGroup(this, grpName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized int countCounters() {
+ return cntrs.size();
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void write(DataOutput out) throws IOException {
+ throw new UnsupportedOperationException("not implemented");
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void readFields(DataInput in) throws IOException {
+ throw new UnsupportedOperationException("not implemented");
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void incrAllCounters(AbstractCounters<Counter, CounterGroup> other) {
+ for (CounterGroup group : other) {
+ for (Counter counter : group) {
+ findCounter(group.getName(), counter.getName()).increment(counter.getValue());
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object genericRight) {
+ if (!(genericRight instanceof HadoopMapReduceCounters))
+ return false;
+
+ return cntrs.equals(((HadoopMapReduceCounters) genericRight).cntrs);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return cntrs.hashCode();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setWriteAllCounters(boolean snd) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean getWriteAllCounters() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Limits limits() {
+ return null;
+ }
+
+ /**
+ * Returns size of a group.
+ *
+ * @param grpName Name of the group.
+ * @return amount of counters in the given group.
+ */
+ public int groupSize(String grpName) {
+ int res = 0;
+
+ for (HadoopCounter counter : cntrs.values()) {
+ if (grpName.equals(counter.group()))
+ res++;
+ }
+
+ return res;
+ }
+
+ /**
+ * Returns counters iterator for specified group.
+ *
+ * @param grpName Name of the group to iterate.
+ * @return Counters iterator.
+ */
+ public Iterator<Counter> iterateGroup(String grpName) {
+ Collection<Counter> grpCounters = new ArrayList<>();
+
+ for (HadoopLongCounter counter : cntrs.values()) {
+ if (grpName.equals(counter.group()))
+ grpCounters.add(new HadoopV2Counter(counter));
+ }
+
+ return grpCounters.iterator();
+ }
+
+ /**
+ * Find a counter in the group.
+ *
+ * @param grpName The name of the counter group.
+ * @param cntrName The name of the counter.
+ * @param create Create the counter if not found if true.
+ * @return The counter that was found or added or {@code null} if create is false.
+ */
+ public Counter findCounter(String grpName, String cntrName, boolean create) {
+ T2<String, String> key = new T2<>(grpName, cntrName);
+
+ HadoopLongCounter internalCntr = cntrs.get(key);
+
+ if (internalCntr == null & create) {
+ internalCntr = new HadoopLongCounter(grpName,cntrName);
+
+ cntrs.put(key, new HadoopLongCounter(grpName,cntrName));
+ }
+
+ return internalCntr == null ? null : new HadoopV2Counter(internalCntr);
+ }
+}
[41/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInput.java
new file mode 100644
index 0000000..ad6446f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInput.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+
+import java.util.*;
+
+/**
+ * Task input.
+ */
+public interface HadoopTaskInput extends AutoCloseable {
+ /**
+ * Moves cursor to the next element.
+ *
+ * @return {@code false} If input is exceeded.
+ */
+ boolean next();
+
+ /**
+ * Gets current key.
+ *
+ * @return Key.
+ */
+ Object key();
+
+ /**
+ * Gets values for current key.
+ *
+ * @return Values.
+ */
+ Iterator<?> values();
+
+ /**
+ * Closes input.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ @Override public void close() throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskOutput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskOutput.java
new file mode 100644
index 0000000..41d9847
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskOutput.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+
+/**
+ * Task output.
+ */
+public interface HadoopTaskOutput extends AutoCloseable {
+ /**
+ * Writes key and value to the output.
+ *
+ * @param key Key.
+ * @param val Value.
+ */
+ public void write(Object key, Object val) throws IgniteCheckedException;
+
+ /**
+ * Closes output.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ @Override public void close() throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskType.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskType.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskType.java
new file mode 100644
index 0000000..a88e189
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskType.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.jetbrains.annotations.*;
+
+/**
+* Task type.
+*/
+public enum HadoopTaskType {
+ /** Setup task. */
+ SETUP,
+
+ /** Map task. */
+ MAP,
+
+ /** Reduce task. */
+ REDUCE,
+
+ /** Combine task. */
+ COMBINE,
+
+ /** Commit task. */
+ COMMIT,
+
+ /** Abort task. */
+ ABORT;
+
+ /** Enumerated values. */
+ private static final HadoopTaskType[] VALS = values();
+
+ /**
+ * Efficiently gets enumerated value from its ordinal.
+ *
+ * @param ord Ordinal value.
+ * @return Enumerated value.
+ */
+ @Nullable public static HadoopTaskType fromOrdinal(byte ord) {
+ return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopNoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopNoopProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopNoopProcessor.java
deleted file mode 100644
index d0ef4ce..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopNoopProcessor.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.util.future.*;
-
-/**
- * Hadoop processor.
- */
-public class IgniteHadoopNoopProcessor extends IgniteHadoopProcessorAdapter {
- /**
- * @param ctx Kernal context.
- */
- public IgniteHadoopNoopProcessor(GridKernalContext ctx) {
- super(ctx);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoop hadoop() {
- throw new IllegalStateException("Hadoop module is not found in class path.");
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopConfiguration config() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopJobId nextJobId() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<?> submit(GridHadoopJobId jobId, GridHadoopJobInfo jobInfo) {
- return new GridFinishedFutureEx<>(new IgniteCheckedException("Hadoop is not available."));
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopJobStatus status(GridHadoopJobId jobId) throws IgniteCheckedException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopCounters counters(GridHadoopJobId jobId) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<?> finishFuture(GridHadoopJobId jobId) throws IgniteCheckedException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public boolean kill(GridHadoopJobId jobId) throws IgniteCheckedException {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessorAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessorAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessorAdapter.java
deleted file mode 100644
index c2cf542..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessorAdapter.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.*;
-
-/**
- * Hadoop processor.
- */
-public abstract class IgniteHadoopProcessorAdapter extends GridProcessorAdapter {
- /**
- * @param ctx Kernal context.
- */
- protected IgniteHadoopProcessorAdapter(GridKernalContext ctx) {
- super(ctx);
- }
-
- /**
- * @return Hadoop facade.
- */
- public abstract GridHadoop hadoop();
-
- /**
- * @return Hadoop configuration.
- */
- public abstract GridHadoopConfiguration config();
-
- /**
- * @return Collection of generated IDs.
- */
- public abstract GridHadoopJobId nextJobId();
-
- /**
- * Submits job to job tracker.
- *
- * @param jobId Job ID to submit.
- * @param jobInfo Job info to submit.
- * @return Execution future.
- */
- public abstract IgniteInternalFuture<?> submit(GridHadoopJobId jobId, GridHadoopJobInfo jobInfo);
-
- /**
- * Gets Hadoop job execution status.
- *
- * @param jobId Job ID to get status for.
- * @return Job execution status.
- * @throws IgniteCheckedException If failed.
- */
- public abstract GridHadoopJobStatus status(GridHadoopJobId jobId) throws IgniteCheckedException;
-
- /**
- * Returns Hadoop job counters.
- *
- * @param jobId Job ID to get counters for.
- * @return Job counters.
- * @throws IgniteCheckedException If failed.
- */
- public abstract GridHadoopCounters counters(GridHadoopJobId jobId) throws IgniteCheckedException;
-
- /**
- * Gets Hadoop job finish future.
- *
- * @param jobId Job ID.
- * @return Job finish future or {@code null}.
- * @throws IgniteCheckedException If failed.
- */
- public abstract IgniteInternalFuture<?> finishFuture(GridHadoopJobId jobId) throws IgniteCheckedException;
-
- /**
- * Kills job.
- *
- * @param jobId Job ID.
- * @return {@code True} if job was killed.
- * @throws IgniteCheckedException If failed.
- */
- public abstract boolean kill(GridHadoopJobId jobId) throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounter.java
new file mode 100644
index 0000000..918c3bc
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounter.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.counter;
+
+/**
+ * Hadoop counter.
+ */
+public interface HadoopCounter {
+ /**
+ * Gets name.
+ *
+ * @return Name of the counter.
+ */
+ public String name();
+
+ /**
+ * Gets counter group.
+ *
+ * @return Counter group's name.
+ */
+ public String group();
+
+ /**
+ * Merge the given counter to this counter.
+ *
+ * @param cntr Counter to merge into this counter.
+ */
+ public void merge(HadoopCounter cntr);
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounterWriter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounterWriter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounterWriter.java
new file mode 100644
index 0000000..ce67c57
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounterWriter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.counter;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+
+/**
+ * The object that writes some system counters to some storage for each running job. This operation is a part of
+ * whole statistics collection process.
+ */
+public interface HadoopCounterWriter {
+ /**
+ * Writes counters of given job to some statistics storage.
+ *
+ * @param jobInfo Job info.
+ * @param jobId Job id.
+ * @param cntrs Counters.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void write(HadoopJobInfo jobInfo, HadoopJobId jobId, HadoopCounters cntrs) throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounters.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounters.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounters.java
new file mode 100644
index 0000000..706ba77
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounters.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.counter;
+
+import java.util.*;
+
+/**
+ * Counters store.
+ */
+public interface HadoopCounters {
+ /**
+ * Returns counter for the specified group and counter name. Creates new if it does not exist.
+ *
+ * @param grp Counter group name.
+ * @param name Counter name.
+ * @param cls Class for new instance creation if it's needed.
+ * @return The counter that was found or added or {@code null} if create is false.
+ */
+ <T extends HadoopCounter> T counter(String grp, String name, Class<T> cls);
+
+ /**
+ * Returns all existing counters.
+ *
+ * @return Collection of counters.
+ */
+ Collection<HadoopCounter> all();
+
+ /**
+ * Merges all counters from another store with existing counters.
+ *
+ * @param other Counters to merge with.
+ */
+ void merge(HadoopCounters other);
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java
index 1479a88..48a32f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java
@@ -21,6 +21,7 @@ import org.apache.ignite.*;
import org.apache.ignite.configuration.*;
import org.apache.ignite.igfs.*;
import org.apache.ignite.igfs.mapreduce.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.apache.ignite.lang.*;
@@ -32,7 +33,7 @@ import java.util.*;
/**
* Igfs supporting asynchronous operations.
*/
-public class IgfsAsyncImpl extends AsyncSupportAdapter<IgniteFs> implements IgfsEx {
+public class IgfsAsyncImpl extends AsyncSupportAdapter<IgniteFileSystem> implements IgfsEx {
/** */
private final IgfsImpl igfs;
@@ -192,7 +193,7 @@ public class IgfsAsyncImpl extends AsyncSupportAdapter<IgniteFs> implements Igfs
}
/** {@inheritDoc} */
- @Override public IgfsConfiguration configuration() {
+ @Override public FileSystemConfiguration configuration() {
return igfs.configuration();
}
@@ -310,7 +311,7 @@ public class IgfsAsyncImpl extends AsyncSupportAdapter<IgniteFs> implements Igfs
}
/** {@inheritDoc} */
- @Override public Map<String, String> properties() {
- return igfs.properties();
+ @Override public IgfsSecondaryFileSystem asSecondary() {
+ return igfs.asSecondary();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
index 245a1dd..475a7fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
@@ -37,7 +37,7 @@ public class IgfsContext {
private final GridKernalContext ctx;
/** Configuration. */
- private final IgfsConfiguration cfg;
+ private final FileSystemConfiguration cfg;
/** Managers. */
private List<IgfsManager> mgrs = new LinkedList<>();
@@ -68,7 +68,7 @@ public class IgfsContext {
*/
public IgfsContext(
GridKernalContext ctx,
- IgfsConfiguration cfg,
+ FileSystemConfiguration cfg,
IgfsMetaManager metaMgr,
IgfsDataManager dataMgr,
IgfsServerManager srvMgr,
@@ -102,7 +102,7 @@ public class IgfsContext {
/**
* @return IGFS configuration.
*/
- public IgfsConfiguration configuration() {
+ public FileSystemConfiguration configuration() {
return cfg;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
index e960422..72bd60a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
@@ -24,6 +24,7 @@ import org.apache.ignite.cluster.*;
import org.apache.ignite.configuration.*;
import org.apache.ignite.events.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.cluster.*;
import org.apache.ignite.internal.managers.communication.*;
@@ -306,7 +307,7 @@ public class IgfsDataManager extends IgfsManager {
IgniteDataLoader<IgfsBlockKey, byte[]> ldr =
igfsCtx.kernalContext().<IgfsBlockKey, byte[]>dataLoad().dataLoader(dataCachePrj.name());
- IgfsConfiguration cfg = igfsCtx.configuration();
+ FileSystemConfiguration cfg = igfsCtx.configuration();
if (cfg.getPerNodeBatchSize() > 0)
ldr.perNodeBufferSize(cfg.getPerNodeBatchSize());
@@ -382,7 +383,7 @@ public class IgfsDataManager extends IgfsManager {
* @throws IgniteCheckedException If failed.
*/
@Nullable public IgniteInternalFuture<byte[]> dataBlock(final IgfsFileInfo fileInfo, final IgfsPath path,
- final long blockIdx, @Nullable final IgfsReader secReader)
+ final long blockIdx, @Nullable final IgfsSecondaryFileSystemPositionedReadable secReader)
throws IgniteCheckedException {
//assert validTxState(any); // Allow this method call for any transaction state.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
index a380a6d..0c5debd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.igfs;
import org.apache.ignite.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.lang.*;
import org.jetbrains.annotations.*;
@@ -26,14 +27,26 @@ import org.jetbrains.annotations.*;
import java.net.*;
/**
- * Internal API extension for {@link org.apache.ignite.IgniteFs}.
+ * Internal API extension for {@link org.apache.ignite.IgniteFileSystem}.
*/
-public interface IgfsEx extends IgniteFs {
+public interface IgfsEx extends IgniteFileSystem {
+ /** File property: user name. */
+ public static final String PROP_USER_NAME = "usrName";
+
+ /** File property: group name. */
+ public static final String PROP_GROUP_NAME = "grpName";
+
+ /** File property: permission. */
+ public static final String PROP_PERMISSION = "permission";
+
+ /** File property: prefer writes to local node. */
+ public static final String PROP_PREFER_LOCAL_WRITES = "locWrite";
+
/** Property name for path to Hadoop configuration. */
- String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH";
+ public static final String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH";
/** Property name for URI of file system. */
- String SECONDARY_FS_URI = "SECONDARY_FS_URI";
+ public static final String SECONDARY_FS_URI = "SECONDARY_FS_URI";
/**
* Stops IGFS cleaning all used resources.
@@ -146,4 +159,11 @@ public interface IgfsEx extends IgniteFs {
* @return {@code True} if proxy.
*/
public boolean isProxy(URI path);
+
+ /**
+ * Return the given IGFS as a secondary file system.
+ *
+ * @return Secondary file system wrapper.
+ */
+ public IgfsSecondaryFileSystem asSecondary();
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
index 43def03..e88503b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
@@ -99,13 +99,13 @@ public final class IgfsFileInfo implements Externalizable {
}
/**
- * Constructs directory or file info with {@link org.apache.ignite.configuration.IgfsConfiguration#DFLT_BLOCK_SIZE default} block size.
+ * Constructs directory or file info with {@link org.apache.ignite.configuration.FileSystemConfiguration#DFLT_BLOCK_SIZE default} block size.
*
* @param isDir Constructs directory info if {@code true} or file info if {@code false}.
* @param props Meta properties to set.
*/
public IgfsFileInfo(boolean isDir, @Nullable Map<String, String> props) {
- this(isDir, null, isDir ? 0 : IgfsConfiguration.DFLT_BLOCK_SIZE, 0, null, null, props, null, false,
+ this(isDir, null, isDir ? 0 : FileSystemConfiguration.DFLT_BLOCK_SIZE, 0, null, null, props, null, false,
System.currentTimeMillis(), false);
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index eff987e..614815f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -26,6 +26,7 @@ import org.apache.ignite.configuration.*;
import org.apache.ignite.events.*;
import org.apache.ignite.igfs.*;
import org.apache.ignite.igfs.mapreduce.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.managers.communication.*;
import org.apache.ignite.internal.managers.eventstorage.*;
@@ -71,7 +72,7 @@ public final class IgfsImpl implements IgfsEx {
private IgfsDataManager data;
/** FS configuration. */
- private IgfsConfiguration cfg;
+ private FileSystemConfiguration cfg;
/** IGFS context. */
private IgfsContext igfsCtx;
@@ -89,7 +90,7 @@ public final class IgfsImpl implements IgfsEx {
private final IgfsModeResolver modeRslvr;
/** Connection to the secondary file system. */
- private Igfs secondaryFs;
+ private IgfsSecondaryFileSystem secondaryFs;
/** Busy lock. */
private final GridSpinBusyLock busyLock = new GridSpinBusyLock();
@@ -124,6 +125,7 @@ public final class IgfsImpl implements IgfsEx {
* @param igfsCtx Context.
* @throws IgniteCheckedException In case of error.
*/
+ @SuppressWarnings("ConstantConditions")
IgfsImpl(IgfsContext igfsCtx) throws IgniteCheckedException {
assert igfsCtx != null;
@@ -376,7 +378,7 @@ public final class IgfsImpl implements IgfsEx {
}
/** {@inheritDoc} */
- @Override public IgfsConfiguration configuration() {
+ @Override public FileSystemConfiguration configuration() {
return cfg;
}
@@ -938,6 +940,7 @@ public final class IgfsImpl implements IgfsEx {
}
/** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
@Override public Collection<IgfsPath> listPaths(final IgfsPath path) {
if (enterBusy()) {
try {
@@ -1067,11 +1070,6 @@ public final class IgfsImpl implements IgfsEx {
}
/** {@inheritDoc} */
- @Override public Map<String, String> properties() {
- return Collections.emptyMap();
- }
-
- /** {@inheritDoc} */
@Override public IgfsInputStreamAdapter open(IgfsPath path) {
return open(path, cfg.getStreamBufferSize(), cfg.getSequentialReadsBeforePrefetch());
}
@@ -1193,7 +1191,7 @@ public final class IgfsImpl implements IgfsEx {
IgfsMode mode = modeRslvr.resolveMode(path);
- IgfsFileWorkerBatch batch = null;
+ IgfsFileWorkerBatch batch;
if (mode == PROXY)
throw new IgniteException("PROXY mode cannot be used in IGFS directly: " + path);
@@ -1250,6 +1248,8 @@ public final class IgfsImpl implements IgfsEx {
IgfsFileInfo oldInfo = meta.info(oldId);
+ assert oldInfo != null;
+
if (oldInfo.isDirectory())
throw new IgfsPathAlreadyExistsException("Failed to create file (path points to a " +
"directory): " + path);
@@ -1268,7 +1268,7 @@ public final class IgfsImpl implements IgfsEx {
info = meta.lock(info.id());
IgfsEventAwareOutputStream os = new IgfsEventAwareOutputStream(path, info, parentId,
- bufSize == 0 ? cfg.getStreamBufferSize() : bufSize, mode, batch);
+ bufSize == 0 ? cfg.getStreamBufferSize() : bufSize, mode, null);
if (evts.isRecordable(EVT_IGFS_FILE_OPENED_WRITE))
evts.record(new IgfsEvent(path, localNode(), EVT_IGFS_FILE_OPENED_WRITE));
@@ -1339,8 +1339,7 @@ public final class IgfsImpl implements IgfsEx {
if (parentId == null)
throw new IgfsInvalidPathException("Failed to resolve parent directory: " + path);
- info = new IgfsFileInfo(cfg.getBlockSize(), /**affinity key*/null, evictExclude(path,
- mode == PRIMARY), props);
+ info = new IgfsFileInfo(cfg.getBlockSize(), /**affinity key*/null, evictExclude(path, true), props);
IgniteUuid oldId = meta.putIfAbsent(parentId, path.name(), info);
@@ -1351,6 +1350,8 @@ public final class IgfsImpl implements IgfsEx {
evts.record(new IgfsEvent(path, localNode(), EVT_IGFS_FILE_CREATED));
}
+ assert info != null;
+
if (!info.isFile())
throw new IgfsInvalidPathException("Failed to open file (not a file): " + path);
@@ -1788,7 +1789,7 @@ public final class IgfsImpl implements IgfsEx {
/**
* Executes IGFS task with overridden maximum range length (see
- * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information).
+ * {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()} for more information).
*
* @param task Task to execute.
* @param rslvr Optional resolver to control split boundaries.
@@ -1822,7 +1823,7 @@ public final class IgfsImpl implements IgfsEx {
/**
* Executes IGFS task asynchronously with overridden maximum range length (see
- * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information).
+ * {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()} for more information).
*
* @param taskCls Task class to execute.
* @param rslvr Optional resolver to control split boundaries.
@@ -1833,6 +1834,7 @@ public final class IgfsImpl implements IgfsEx {
* @param arg Optional task argument.
* @return Execution future.
*/
+ @SuppressWarnings("unchecked")
<T, R> IgniteInternalFuture<R> executeAsync(Class<? extends IgfsTask<T, R>> taskCls,
@Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles,
long maxRangeLen, @Nullable T arg) {
@@ -1898,7 +1900,7 @@ public final class IgfsImpl implements IgfsEx {
}
/** {@inheritDoc} */
- @Override public IgniteFs withAsync() {
+ @Override public IgniteFileSystem withAsync() {
return new IgfsAsyncImpl(this);
}
@@ -2033,7 +2035,7 @@ public final class IgfsImpl implements IgfsEx {
* @param metrics Metrics.
*/
IgfsEventAwareInputStream(IgfsContext igfsCtx, IgfsPath path, IgfsFileInfo fileInfo,
- int prefetchBlocks, int seqReadsBeforePrefetch, @Nullable IgfsReader secReader,
+ int prefetchBlocks, int seqReadsBeforePrefetch, @Nullable IgfsSecondaryFileSystemPositionedReadable secReader,
IgfsLocalMetrics metrics) {
super(igfsCtx, path, fileInfo, prefetchBlocks, seqReadsBeforePrefetch, secReader, metrics);
@@ -2083,7 +2085,7 @@ public final class IgfsImpl implements IgfsEx {
private Ignite g;
@Nullable @Override public IgniteBiTuple<Long, Long> execute() {
- IgniteFs igfs = ((IgniteKernal)g).context().igfs().igfs(igfsName);
+ IgniteFileSystem igfs = ((IgniteKernal)g).context().igfs().igfs(igfsName);
if (igfs == null)
return F.t(0L, 0L);
@@ -2129,6 +2131,7 @@ public final class IgfsImpl implements IgfsEx {
*/
private class FormatMessageListener implements GridMessageListener {
/** {@inheritDoc} */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
@Override public void onMessage(UUID nodeId, Object msg) {
if (msg instanceof IgfsDeleteMessage) {
ClusterNode node = igfsCtx.kernalContext().discovery().node(nodeId);
@@ -2219,4 +2222,9 @@ public final class IgfsImpl implements IgfsEx {
return mode == PROXY;
}
+
+ /** {@inheritDoc} */
+ @Override public IgfsSecondaryFileSystem asSecondary() {
+ return new IgfsSecondaryFileSystemImpl(this);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamAdapter.java
index 51e57db..fc333b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamAdapter.java
@@ -18,13 +18,15 @@
package org.apache.ignite.internal.processors.igfs;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import java.io.*;
/**
* Implementation adapter providing necessary methods.
*/
-public abstract class IgfsInputStreamAdapter extends IgfsInputStream {
+public abstract class IgfsInputStreamAdapter extends IgfsInputStream
+ implements IgfsSecondaryFileSystemPositionedReadable {
/** {@inheritDoc} */
@Override public long length() {
return fileInfo().length();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
index 92b4383..5afa523 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.igfs;
import org.apache.ignite.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.util.typedef.internal.*;
@@ -45,7 +46,7 @@ public class IgfsInputStreamImpl extends IgfsInputStreamAdapter {
/** Secondary file system reader. */
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private final IgfsReader secReader;
+ private final IgfsSecondaryFileSystemPositionedReadable secReader;
/** Logger. */
private IgniteLogger log;
@@ -110,7 +111,7 @@ public class IgfsInputStreamImpl extends IgfsInputStreamAdapter {
* @param metrics Local IGFS metrics.
*/
IgfsInputStreamImpl(IgfsContext igfsCtx, IgfsPath path, IgfsFileInfo fileInfo, int prefetchBlocks,
- int seqReadsBeforePrefetch, @Nullable IgfsReader secReader, IgfsLocalMetrics metrics) {
+ int seqReadsBeforePrefetch, @Nullable IgfsSecondaryFileSystemPositionedReadable secReader, IgfsLocalMetrics metrics) {
assert igfsCtx != null;
assert path != null;
assert fileInfo != null;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsJobImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsJobImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsJobImpl.java
index f6b17bf..fa90e21 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsJobImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsJobImpl.java
@@ -79,7 +79,7 @@ public class IgfsJobImpl implements ComputeJob, GridInternalWrapper<IgfsJob> {
/** {@inheritDoc} */
@Override public Object execute() {
- IgniteFs fs = ignite.fileSystem(igfsName);
+ IgniteFileSystem fs = ignite.fileSystem(igfsName);
try (IgfsInputStream in = fs.open(path)) {
IgfsFileRange split = new IgfsFileRange(path, start, len);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index 9d1795a..adc0254 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -23,6 +23,7 @@ import org.apache.ignite.cluster.*;
import org.apache.ignite.configuration.*;
import org.apache.ignite.events.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.managers.eventstorage.*;
import org.apache.ignite.internal.processors.cache.*;
@@ -51,7 +52,7 @@ import static org.apache.ignite.transactions.TransactionIsolation.*;
@SuppressWarnings("all")
public class IgfsMetaManager extends IgfsManager {
/** IGFS configuration. */
- private IgfsConfiguration cfg;
+ private FileSystemConfiguration cfg;
/** Metadata cache. */
private GridCache<Object, Object> metaCache;
@@ -1588,7 +1589,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return Output stream descriptor.
* @throws IgniteCheckedException If file creation failed.
*/
- public IgfsSecondaryOutputStreamDescriptor createDual(final Igfs fs,
+ public IgfsSecondaryOutputStreamDescriptor createDual(final IgfsSecondaryFileSystem fs,
final IgfsPath path,
final boolean simpleCreate,
@Nullable final Map<String, String> props,
@@ -1752,7 +1753,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return Output stream descriptor.
* @throws IgniteCheckedException If output stream open for append has failed.
*/
- public IgfsSecondaryOutputStreamDescriptor appendDual(final Igfs fs, final IgfsPath path,
+ public IgfsSecondaryOutputStreamDescriptor appendDual(final IgfsSecondaryFileSystem fs, final IgfsPath path,
final int bufSize) throws IgniteCheckedException {
if (busyLock.enterBusy()) {
try {
@@ -1783,7 +1784,7 @@ public class IgfsMetaManager extends IgfsManager {
if (remainder > 0) {
int blockIdx = (int)(len / blockSize);
- IgfsReader reader = fs.open(path, bufSize);
+ IgfsSecondaryFileSystemPositionedReadable reader = fs.open(path, bufSize);
try {
igfsCtx.data().dataBlock(info, path, blockIdx, reader).get();
@@ -1832,7 +1833,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return Input stream descriptor.
* @throws IgniteCheckedException If input stream open has failed.
*/
- public IgfsSecondaryInputStreamDescriptor openDual(final Igfs fs, final IgfsPath path,
+ public IgfsSecondaryInputStreamDescriptor openDual(final IgfsSecondaryFileSystem fs, final IgfsPath path,
final int bufSize)
throws IgniteCheckedException {
if (busyLock.enterBusy()) {
@@ -1893,7 +1894,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return File info or {@code null} if file not found.
* @throws IgniteCheckedException If sync task failed.
*/
- @Nullable public IgfsFileInfo synchronizeFileDual(final Igfs fs, final IgfsPath path)
+ @Nullable public IgfsFileInfo synchronizeFileDual(final IgfsSecondaryFileSystem fs, final IgfsPath path)
throws IgniteCheckedException {
assert fs != null;
assert path != null;
@@ -1941,7 +1942,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return {@code True} in case rename was successful.
* @throws IgniteCheckedException If directory creation failed.
*/
- public boolean mkdirsDual(final Igfs fs, final IgfsPath path, final Map<String, String> props)
+ public boolean mkdirsDual(final IgfsSecondaryFileSystem fs, final IgfsPath path, final Map<String, String> props)
throws IgniteCheckedException {
if (busyLock.enterBusy()) {
try {
@@ -2025,7 +2026,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return Operation result.
* @throws IgniteCheckedException If failed.
*/
- public boolean renameDual(final Igfs fs, final IgfsPath src, final IgfsPath dest) throws
+ public boolean renameDual(final IgfsSecondaryFileSystem fs, final IgfsPath src, final IgfsPath dest) throws
IgniteCheckedException {
if (busyLock.enterBusy()) {
try {
@@ -2124,7 +2125,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return Operation result.
* @throws IgniteCheckedException If delete failed.
*/
- public boolean deleteDual(final Igfs fs, final IgfsPath path, final boolean recursive)
+ public boolean deleteDual(final IgfsSecondaryFileSystem fs, final IgfsPath path, final boolean recursive)
throws IgniteCheckedException {
if (busyLock.enterBusy()) {
try {
@@ -2190,7 +2191,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return Update file info.
* @throws IgniteCheckedException If update failed.
*/
- public IgfsFileInfo updateDual(final Igfs fs, final IgfsPath path, final Map<String, String> props)
+ public IgfsFileInfo updateDual(final IgfsSecondaryFileSystem fs, final IgfsPath path, final Map<String, String> props)
throws IgniteCheckedException {
assert fs != null;
assert path != null;
@@ -2243,7 +2244,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return File info of the end path.
* @throws IgniteCheckedException If failed.
*/
- private IgfsFileInfo synchronize(Igfs fs,
+ private IgfsFileInfo synchronize(IgfsSecondaryFileSystem fs,
IgfsPath startPath,
IgfsFileInfo startPathInfo,
IgfsPath endPath,
@@ -2328,7 +2329,7 @@ public class IgfsMetaManager extends IgfsManager {
* @throws IgniteCheckedException If failed.
*/
private <T> T synchronizeAndExecute(SynchronizationTask<T> task,
- Igfs fs,
+ IgfsSecondaryFileSystem fs,
boolean strict,
IgfsPath... paths)
throws IgniteCheckedException
@@ -2349,7 +2350,7 @@ public class IgfsMetaManager extends IgfsManager {
* @throws IgniteCheckedException If failed.
*/
private <T> T synchronizeAndExecute(SynchronizationTask<T> task,
- Igfs fs,
+ IgfsSecondaryFileSystem fs,
boolean strict,
@Nullable Collection<IgniteUuid> extraLockIds,
IgfsPath... paths)
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java
index a299fc4..41dcc31 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java
@@ -49,12 +49,12 @@ public class IgfsNoopProcessor extends IgfsProcessorAdapter {
}
/** {@inheritDoc} */
- @Override public Collection<IgniteFs> igfss() {
+ @Override public Collection<IgniteFileSystem> igfss() {
return Collections.emptyList();
}
/** {@inheritDoc} */
- @Nullable @Override public IgniteFs igfs(@Nullable String name) {
+ @Nullable @Override public IgniteFileSystem igfs(@Nullable String name) {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java
index 5941579..6c6dd9d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java
@@ -405,7 +405,7 @@ class IgfsOutputStreamImpl extends IgfsOutputStreamAdapter {
if (!igfsCtx.configuration().isFragmentizerEnabled())
return null;
- if (!Boolean.parseBoolean(fileInfo.properties().get(IgniteFs.PROP_PREFER_LOCAL_WRITES)))
+ if (!Boolean.parseBoolean(fileInfo.properties().get(IgfsEx.PROP_PREFER_LOCAL_WRITES)))
return null;
int blockSize = fileInfo.blockSize();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java
index 847cd50..e1b5114 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java
@@ -51,8 +51,8 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
private static final String NULL_NAME = UUID.randomUUID().toString();
/** Converts context to IGFS. */
- private static final IgniteClosure<IgfsContext,IgniteFs> CTX_TO_IGFS = new C1<IgfsContext, IgniteFs>() {
- @Override public IgniteFs apply(IgfsContext igfsCtx) {
+ private static final IgniteClosure<IgfsContext,IgniteFileSystem> CTX_TO_IGFS = new C1<IgfsContext, IgniteFileSystem>() {
+ @Override public IgniteFileSystem apply(IgfsContext igfsCtx) {
return igfsCtx.igfs();
}
};
@@ -73,17 +73,17 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
if (ctx.config().isDaemon())
return;
- IgfsConfiguration[] cfgs = ctx.config().getIgfsConfiguration();
+ FileSystemConfiguration[] cfgs = ctx.config().getFileSystemConfiguration();
assert cfgs != null && cfgs.length > 0;
validateLocalIgfsConfigurations(cfgs);
// Start IGFS instances.
- for (IgfsConfiguration cfg : cfgs) {
+ for (FileSystemConfiguration cfg : cfgs) {
IgfsContext igfsCtx = new IgfsContext(
ctx,
- new IgfsConfiguration(cfg),
+ new FileSystemConfiguration(cfg),
new IgfsMetaManager(),
new IgfsDataManager(),
new IgfsServerManager(),
@@ -167,12 +167,12 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
- @Override public Collection<IgniteFs> igfss() {
+ @Override public Collection<IgniteFileSystem> igfss() {
return F.viewReadOnly(igfsCache.values(), CTX_TO_IGFS);
}
/** {@inheritDoc} */
- @Override @Nullable public IgniteFs igfs(@Nullable String name) {
+ @Override @Nullable public IgniteFileSystem igfs(@Nullable String name) {
IgfsContext igfsCtx = igfsCache.get(maskName(name));
return igfsCtx == null ? null : igfsCtx.igfs();
@@ -202,7 +202,7 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
// is daemon;
// doesn't have configured IGFS;
// doesn't have configured caches.
- if (gridCfg.isDaemon() || F.isEmpty(gridCfg.getIgfsConfiguration()) ||
+ if (gridCfg.isDaemon() || F.isEmpty(gridCfg.getFileSystemConfiguration()) ||
F.isEmpty(gridCfg.getCacheConfiguration()))
return;
@@ -216,9 +216,9 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
Collection<IgfsAttributes> attrVals = new ArrayList<>();
- assert gridCfg.getIgfsConfiguration() != null;
+ assert gridCfg.getFileSystemConfiguration() != null;
- for (IgfsConfiguration igfsCfg : gridCfg.getIgfsConfiguration()) {
+ for (FileSystemConfiguration igfsCfg : gridCfg.getFileSystemConfiguration()) {
CacheConfiguration cacheCfg = cacheCfgs.get(igfsCfg.getDataCacheName());
if (cacheCfg == null)
@@ -258,10 +258,10 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
* @param cfgs IGFS configurations
* @throws IgniteCheckedException If any of IGFS configurations is invalid.
*/
- private void validateLocalIgfsConfigurations(IgfsConfiguration[] cfgs) throws IgniteCheckedException {
+ private void validateLocalIgfsConfigurations(FileSystemConfiguration[] cfgs) throws IgniteCheckedException {
Collection<String> cfgNames = new HashSet<>();
- for (IgfsConfiguration cfg : cfgs) {
+ for (FileSystemConfiguration cfg : cfgs) {
String name = cfg.getName();
if (cfgNames.contains(name))
@@ -333,7 +333,7 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
if (secondary) {
// When working in any mode except of primary, secondary FS config must be provided.
assertParameter(cfg.getSecondaryFileSystem() != null,
- "secondaryFileSystem cannot be null when mode is SECONDARY");
+ "secondaryFileSystem cannot be null when mode is not " + IgfsMode.PRIMARY);
}
cfgNames.add(name);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java
index b695104..ee38ab3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java
@@ -46,7 +46,7 @@ public abstract class IgfsProcessorAdapter extends GridProcessorAdapter {
*
* @return Collection of IGFS instances.
*/
- public abstract Collection<IgniteFs> igfss();
+ public abstract Collection<IgniteFileSystem> igfss();
/**
* Gets IGFS instance.
@@ -54,7 +54,7 @@ public abstract class IgfsProcessorAdapter extends GridProcessorAdapter {
* @param name (Nullable) IGFS name.
* @return IGFS instance.
*/
- @Nullable public abstract IgniteFs igfs(@Nullable String name);
+ @Nullable public abstract IgniteFileSystem igfs(@Nullable String name);
/**
* Gets server endpoints for particular IGFS.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
new file mode 100644
index 0000000..683b317
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs;
+
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Secondary file system over native IGFS.
+ */
+class IgfsSecondaryFileSystemImpl implements IgfsSecondaryFileSystem {
+ /** Delegate. */
+ private final IgfsImpl igfs;
+
+ /**
+ * Constructor.
+ *
+ * @param igfs Delegate.
+ */
+ IgfsSecondaryFileSystemImpl(IgfsImpl igfs) {
+ this.igfs = igfs;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean exists(IgfsPath path) {
+ return igfs.exists(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteException {
+ return igfs.update(path, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void rename(IgfsPath src, IgfsPath dest) throws IgniteException {
+ igfs.rename(src, dest);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean delete(IgfsPath path, boolean recursive) throws IgniteException {
+ return igfs.delete(path, recursive);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdirs(IgfsPath path) throws IgniteException {
+ igfs.mkdirs(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) throws IgniteException {
+ igfs.mkdirs(path, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteException {
+ return igfs.listPaths(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteException {
+ return igfs.listFiles(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsSecondaryFileSystemPositionedReadable open(IgfsPath path, int bufSize)
+ throws IgniteException {
+ return igfs.open(path, bufSize);
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputStream create(IgfsPath path, boolean overwrite) throws IgniteException {
+ return igfs.create(path, overwrite);
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
+ long blockSize, @Nullable Map<String, String> props) throws IgniteException {
+ return igfs.create(path, bufSize, overwrite, replication, blockSize, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputStream append(IgfsPath path, int bufSize, boolean create,
+ @Nullable Map<String, String> props) throws IgniteException {
+ return igfs.append(path, bufSize, create, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsFile info(IgfsPath path) throws IgniteException {
+ return igfs.info(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long usedSpaceSize() throws IgniteException {
+ return igfs.usedSpaceSize();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, String> properties() {
+ return Collections.emptyMap();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java
index 6e48103..e9ba6f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java
@@ -17,7 +17,7 @@
package org.apache.ignite.internal.processors.igfs;
-import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
/**
* Descriptor of an input stream opened to the secondary file system.
@@ -27,7 +27,7 @@ public class IgfsSecondaryInputStreamDescriptor {
private final IgfsFileInfo info;
/** Secondary file system input stream wrapper. */
- private final IgfsReader secReader;
+ private final IgfsSecondaryFileSystemPositionedReadable secReader;
/**
* Constructor.
@@ -35,7 +35,7 @@ public class IgfsSecondaryInputStreamDescriptor {
* @param info File info in the primary file system.
* @param secReader Secondary file system reader.
*/
- IgfsSecondaryInputStreamDescriptor(IgfsFileInfo info, IgfsReader secReader) {
+ IgfsSecondaryInputStreamDescriptor(IgfsFileInfo info, IgfsSecondaryFileSystemPositionedReadable secReader) {
assert info != null;
assert secReader != null;
@@ -53,7 +53,7 @@ public class IgfsSecondaryInputStreamDescriptor {
/**
* @return Secondary file system reader.
*/
- IgfsReader reader() {
+ IgfsSecondaryFileSystemPositionedReadable reader() {
return secReader;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsServerManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsServerManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsServerManager.java
index e1f4a0f..643eeff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsServerManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsServerManager.java
@@ -29,7 +29,7 @@ import org.apache.ignite.thread.*;
import java.util.*;
import java.util.concurrent.*;
-import static org.apache.ignite.configuration.IgfsConfiguration.*;
+import static org.apache.ignite.configuration.FileSystemConfiguration.*;
/**
* IGFS server manager.
@@ -49,7 +49,7 @@ public class IgfsServerManager extends IgfsManager {
/** {@inheritDoc} */
@Override protected void start0() throws IgniteCheckedException {
- IgfsConfiguration igfsCfg = igfsCtx.configuration();
+ FileSystemConfiguration igfsCfg = igfsCtx.configuration();
Map<String,String> cfg = igfsCfg.getIpcEndpointConfiguration();
if (F.isEmpty(cfg)) {
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfs.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfs.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfs.java
index 2258d4d..edcde6a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfs.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfs.java
@@ -24,7 +24,7 @@ import org.apache.ignite.internal.util.typedef.internal.*;
import java.io.*;
/**
- * Data transfer object for {@link org.apache.ignite.IgniteFs}.
+ * Data transfer object for {@link org.apache.ignite.IgniteFileSystem}.
*/
public class VisorIgfs implements Serializable {
/** */
@@ -66,7 +66,7 @@ public class VisorIgfs implements Serializable {
* @param igfs Source IGFS.
* @return Data transfer object for given IGFS.
*/
- public static VisorIgfs from(IgniteFs igfs) {
+ public static VisorIgfs from(IgniteFileSystem igfs) {
assert igfs != null;
return new VisorIgfs(
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java
index f63e669..814fc50 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java
@@ -59,7 +59,7 @@ public class VisorIgfsProfilerClearTask extends VisorOneNodeTask<String, IgniteB
int notDeleted = 0;
try {
- IgniteFs igfs = ignite.fileSystem(arg);
+ IgniteFileSystem igfs = ignite.fileSystem(arg);
Path logsDir = resolveIgfsProfilerLogsDir(igfs);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java
index 96c69d9..cc6ae63 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java
@@ -114,7 +114,7 @@ public class VisorGridConfiguration implements Serializable {
rest = VisorRestConfiguration.from(c);
userAttrs = c.getUserAttributes();
caches = VisorCacheConfiguration.list(ignite, c.getCacheConfiguration());
- igfss = VisorIgfsConfiguration.list(c.getIgfsConfiguration());
+ igfss = VisorIgfsConfiguration.list(c.getFileSystemConfiguration());
streamers = VisorStreamerConfiguration.list(c.getStreamerConfiguration());
env = new HashMap<>(getenv());
sysProps = getProperties();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
index 78943a2..9064ea6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.visor.node;
import org.apache.ignite.configuration.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.jetbrains.annotations.*;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
@@ -120,7 +121,7 @@ public class VisorIgfsConfiguration implements Serializable {
* @param igfs IGFS configuration.
* @return Data transfer object for IGFS configuration properties.
*/
- public static VisorIgfsConfiguration from(IgfsConfiguration igfs) {
+ public static VisorIgfsConfiguration from(FileSystemConfiguration igfs) {
VisorIgfsConfiguration cfg = new VisorIgfsConfiguration();
cfg.name = igfs.getName();
@@ -132,7 +133,7 @@ public class VisorIgfsConfiguration implements Serializable {
cfg.perNodeBatchSize = igfs.getPerNodeBatchSize();
cfg.perNodeParallelBatchCnt = igfs.getPerNodeParallelBatchCount();
- Igfs secFs = igfs.getSecondaryFileSystem();
+ IgfsSecondaryFileSystem secFs = igfs.getSecondaryFileSystem();
if (secFs != null) {
Map<String, String> props = secFs.properties();
@@ -171,13 +172,13 @@ public class VisorIgfsConfiguration implements Serializable {
* @param igfss Igfs configurations.
* @return igfs configurations properties.
*/
- public static Iterable<VisorIgfsConfiguration> list(IgfsConfiguration[] igfss) {
+ public static Iterable<VisorIgfsConfiguration> list(FileSystemConfiguration[] igfss) {
if (igfss == null)
return Collections.emptyList();
final Collection<VisorIgfsConfiguration> cfgs = new ArrayList<>(igfss.length);
- for (IgfsConfiguration igfs : igfss)
+ for (FileSystemConfiguration igfs : igfss)
cfgs.add(from(igfs));
return cfgs;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
index 43a987c..b86ea5a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
@@ -151,7 +151,7 @@ public class VisorNodeDataCollectorJob extends VisorJob<VisorNodeDataCollectorTa
try {
IgfsProcessorAdapter igfsProc = ((IgniteKernal)ignite).context().igfs();
- for (IgniteFs igfs : igfsProc.igfss()) {
+ for (IgniteFileSystem igfs : igfsProc.igfss()) {
long start0 = U.currentTimeMillis();
try {
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
index b03674f..51e3c6e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
@@ -46,7 +46,7 @@ import java.util.concurrent.atomic.*;
import java.util.zip.*;
import static java.lang.System.*;
-import static org.apache.ignite.configuration.IgfsConfiguration.*;
+import static org.apache.ignite.configuration.FileSystemConfiguration.*;
import static org.apache.ignite.events.EventType.*;
/**
@@ -592,7 +592,7 @@ public class VisorTaskUtils {
* @return {@link Path} to log dir or {@code null} if not found.
* @throws IgniteCheckedException if failed to resolve.
*/
- public static Path resolveIgfsProfilerLogsDir(IgniteFs igfs) throws IgniteCheckedException {
+ public static Path resolveIgfsProfilerLogsDir(IgniteFileSystem igfs) throws IgniteCheckedException {
String logsDir;
if (igfs instanceof IgfsEx)
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml
----------------------------------------------------------------------
diff --git a/modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml b/modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml
index 2dc952a..43f9395 100644
--- a/modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml
+++ b/modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml
@@ -29,12 +29,12 @@
<property>
<name>fs.igfs.impl</name>
- <value>org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem</value>
+ <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value>
</property>
<property>
<name>fs.AbstractFileSystem.igfs.impl</name>
- <value>org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem</value>
+ <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/config/hadoop/core-site-loopback.xml
----------------------------------------------------------------------
diff --git a/modules/core/src/test/config/hadoop/core-site-loopback.xml b/modules/core/src/test/config/hadoop/core-site-loopback.xml
index 63902fa..e1d1320 100644
--- a/modules/core/src/test/config/hadoop/core-site-loopback.xml
+++ b/modules/core/src/test/config/hadoop/core-site-loopback.xml
@@ -29,12 +29,12 @@
<property>
<name>fs.igfs.impl</name>
- <value>org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem</value>
+ <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value>
</property>
<property>
<name>fs.AbstractFileSystem.igfs.impl</name>
- <value>org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem</value>
+ <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/config/hadoop/core-site-secondary.xml
----------------------------------------------------------------------
diff --git a/modules/core/src/test/config/hadoop/core-site-secondary.xml b/modules/core/src/test/config/hadoop/core-site-secondary.xml
index 648df36..fa301be 100644
--- a/modules/core/src/test/config/hadoop/core-site-secondary.xml
+++ b/modules/core/src/test/config/hadoop/core-site-secondary.xml
@@ -29,12 +29,12 @@
<property>
<name>fs.igfs.impl</name>
- <value>org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem</value>
+ <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value>
</property>
<property>
<name>fs.AbstractFileSystem.igfs.impl</name>
- <value>org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem</value>
+ <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/config/hadoop/core-site.xml
----------------------------------------------------------------------
diff --git a/modules/core/src/test/config/hadoop/core-site.xml b/modules/core/src/test/config/hadoop/core-site.xml
index eb16ce7..0a9eee7 100644
--- a/modules/core/src/test/config/hadoop/core-site.xml
+++ b/modules/core/src/test/config/hadoop/core-site.xml
@@ -29,11 +29,11 @@
<property>
<name>fs.igfs.impl</name>
- <value>org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem</value>
+ <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value>
</property>
<property>
<name>fs.AbstractFileSystem.igfs.impl</name>
- <value>org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem</value>
+ <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/config/igfs-loopback.xml
----------------------------------------------------------------------
diff --git a/modules/core/src/test/config/igfs-loopback.xml b/modules/core/src/test/config/igfs-loopback.xml
index 81ce501..46377d9 100644
--- a/modules/core/src/test/config/igfs-loopback.xml
+++ b/modules/core/src/test/config/igfs-loopback.xml
@@ -78,9 +78,9 @@
</bean>
</property>
- <property name="igfsConfiguration">
+ <property name="fileSystemConfiguration">
<list>
- <bean class="org.apache.ignite.configuration.IgfsConfiguration">
+ <bean class="org.apache.ignite.configuration.FileSystemConfiguration">
<property name="name" value="igfs"/>
<property name="metaCacheName" value="igfs-meta"/>
<property name="dataCacheName" value="igfs-data"/>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/config/igfs-shmem.xml
----------------------------------------------------------------------
diff --git a/modules/core/src/test/config/igfs-shmem.xml b/modules/core/src/test/config/igfs-shmem.xml
index d30051b..c08f78d 100644
--- a/modules/core/src/test/config/igfs-shmem.xml
+++ b/modules/core/src/test/config/igfs-shmem.xml
@@ -78,9 +78,9 @@
</bean>
</property>
- <property name="igfsConfiguration">
+ <property name="fileSystemConfiguration">
<list>
- <bean class="org.apache.ignite.configuration.IgfsConfiguration">
+ <bean class="org.apache.ignite.configuration.FileSystemConfiguration">
<property name="name" value="igfs"/>
<property name="metaCacheName" value="igfs-meta"/>
<property name="dataCacheName" value="igfs-data"/>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java
index 364eaa1..2d9d269 100644
--- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java
@@ -40,7 +40,7 @@ import static org.apache.ignite.events.EventType.*;
import static org.apache.ignite.testframework.GridTestUtils.*;
/**
- * Tests events, generated by {@link org.apache.ignite.IgniteFs} implementation.
+ * Tests events, generated by {@link org.apache.ignite.IgniteFileSystem} implementation.
*/
public abstract class IgfsEventsAbstractSelfTest extends GridCommonAbstractTest {
/** IGFS. */
@@ -84,8 +84,8 @@ public abstract class IgfsEventsAbstractSelfTest extends GridCommonAbstractTest
/**
* @return IGFS configuration for this test.
*/
- protected IgfsConfiguration getIgfsConfiguration() throws IgniteCheckedException {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException {
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName("dataCache");
igfsCfg.setMetaCacheName("metaCache");
@@ -109,7 +109,7 @@ public abstract class IgfsEventsAbstractSelfTest extends GridCommonAbstractTest
* @return Grid configuration.
* @throws Exception If failed.
*/
- protected IgniteConfiguration getConfiguration(String gridName, IgfsConfiguration igfsCfg) throws Exception {
+ protected IgniteConfiguration getConfiguration(String gridName, FileSystemConfiguration igfsCfg) throws Exception {
IgniteConfiguration cfg = IgnitionEx.loadConfiguration("config/hadoop/default-config.xml").get1();
assert cfg != null;
@@ -118,7 +118,7 @@ public abstract class IgfsEventsAbstractSelfTest extends GridCommonAbstractTest
cfg.setIncludeEventTypes(concat(EVTS_IGFS, EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED));
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
cfg.setCacheConfiguration(getCacheConfiguration(gridName));
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java
index 6ae9c86..3fa0804 100644
--- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java
@@ -63,7 +63,7 @@ public class IgfsFragmentizerAbstractSelfTest extends IgfsCommonAbstractTest {
cfg.setCacheConfiguration(metaConfiguration(), dataConfiguration());
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setName("igfs");
igfsCfg.setMetaCacheName(META_CACHE_NAME);
@@ -76,7 +76,7 @@ public class IgfsFragmentizerAbstractSelfTest extends IgfsCommonAbstractTest {
igfsCfg.setFragmentizerThrottlingBlockLength(16 * IGFS_BLOCK_SIZE);
igfsCfg.setFragmentizerThrottlingDelay(10);
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
return cfg;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
index 99a40c4..b51db0c 100644
--- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
@@ -36,7 +36,7 @@ public class IgfsFragmentizerSelfTest extends IgfsFragmentizerAbstractSelfTest {
* @throws Exception If failed.
*/
public void testReadFragmentizing() throws Exception {
- IgniteFs igfs = grid(0).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(0).fileSystem("igfs");
IgfsPath path = new IgfsPath("/someFile");
@@ -108,7 +108,7 @@ public class IgfsFragmentizerSelfTest extends IgfsFragmentizerAbstractSelfTest {
int fileSize = 30 * IGFS_GROUP_SIZE * IGFS_BLOCK_SIZE;
while (written < fileSize) {
- IgniteFs igfs = grid(igfsIdx).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(igfsIdx).fileSystem("igfs");
try (IgfsOutputStream out = igfs.append(path, true)) {
byte[] data = new byte[chunkSize];
@@ -131,7 +131,7 @@ public class IgfsFragmentizerSelfTest extends IgfsFragmentizerAbstractSelfTest {
}
}
- IgniteFs igfs = grid(0).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(0).fileSystem("igfs");
try (IgfsInputStream in = igfs.open(path)) {
i = 0;
@@ -180,7 +180,7 @@ public class IgfsFragmentizerSelfTest extends IgfsFragmentizerAbstractSelfTest {
int fileSize = 50 * IGFS_GROUP_SIZE * IGFS_BLOCK_SIZE;
- IgniteFs igfs = grid(0).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(0).fileSystem("igfs");
byte[] chunk = new byte[chunkSize];
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerTopologySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerTopologySelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerTopologySelfTest.java
index 59faa42..cb7b63a 100644
--- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerTopologySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerTopologySelfTest.java
@@ -33,7 +33,7 @@ public class IgfsFragmentizerTopologySelfTest extends IgfsFragmentizerAbstractSe
try {
IgfsPath path = new IgfsPath("/someFile");
- IgniteFs igfs = grid(1).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(1).fileSystem("igfs");
try (IgfsOutputStream out = igfs.create(path, true)) {
for (int i = 0; i < 10 * IGFS_GROUP_SIZE; i++)
[40/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java
deleted file mode 100644
index f0e2c70..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java
+++ /dev/null
@@ -1,485 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.igfs;
-
-import org.apache.ignite.*;
-import org.apache.ignite.cache.*;
-import org.apache.ignite.cache.eviction.igfs.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.cache.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.spi.discovery.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
-import org.apache.ignite.testframework.*;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.cache.CacheMode.*;
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * Tests for IGFS per-block LR eviction policy.
- */
-@SuppressWarnings({"ConstantConditions", "ThrowableResultOfMethodCallIgnored"})
-public class GridCacheIgfsPerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstractTest {
- /** Primary IGFS name. */
- private static final String IGFS_PRIMARY = "igfs-primary";
-
- /** Primary IGFS name. */
- private static final String IGFS_SECONDARY = "igfs-secondary";
-
- /** Secondary file system REST endpoint configuration map. */
- private static final Map<String, String> SECONDARY_REST_CFG = new HashMap<String, String>() {{
- put("type", "tcp");
- put("port", "11500");
- }};
-
- /** File working in PRIMARY mode. */
- public static final IgfsPath FILE = new IgfsPath("/file");
-
- /** File working in DUAL mode. */
- public static final IgfsPath FILE_RMT = new IgfsPath("/fileRemote");
-
- /** Primary IGFS instances. */
- private static IgfsImpl igfsPrimary;
-
- /** Secondary IGFS instance. */
- private static IgniteFs secondaryFs;
-
- /** Primary file system data cache. */
- private static GridCacheAdapter<IgfsBlockKey, byte[]> dataCache;
-
- /** Eviction policy */
- private static CacheIgfsPerBlockLruEvictionPolicy evictPlc;
-
- /**
- * Start a grid with the primary file system.
- *
- * @throws Exception If failed.
- */
- private void startPrimary() throws Exception {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("dataCache");
- igfsCfg.setMetaCacheName("metaCache");
- igfsCfg.setName(IGFS_PRIMARY);
- igfsCfg.setBlockSize(512);
- igfsCfg.setDefaultMode(PRIMARY);
- igfsCfg.setPrefetchBlocks(1);
- igfsCfg.setSequentialReadsBeforePrefetch(Integer.MAX_VALUE);
- igfsCfg.setSecondaryFileSystem(secondaryFs);
-
- Map<String, IgfsMode> pathModes = new HashMap<>();
-
- pathModes.put(FILE_RMT.toString(), DUAL_SYNC);
-
- igfsCfg.setPathModes(pathModes);
-
- CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
-
- dataCacheCfg.setName("dataCache");
- dataCacheCfg.setCacheMode(PARTITIONED);
- dataCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- evictPlc = new CacheIgfsPerBlockLruEvictionPolicy();
-
- dataCacheCfg.setEvictionPolicy(evictPlc);
- dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
- dataCacheCfg.setBackups(0);
- dataCacheCfg.setQueryIndexEnabled(false);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("metaCache");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- IgniteConfiguration cfg = new IgniteConfiguration();
-
- cfg.setGridName("grid-primary");
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
-
- cfg.setLocalHost("127.0.0.1");
- cfg.setConnectorConfiguration(null);
-
- Ignite g = G.start(cfg);
-
- igfsPrimary = (IgfsImpl)g.fileSystem(IGFS_PRIMARY);
-
- dataCache = igfsPrimary.context().kernalContext().cache().internalCache(
- igfsPrimary.context().configuration().getDataCacheName());
- }
-
- /**
- * Start a grid with the secondary file system.
- *
- * @throws Exception If failed.
- */
- private void startSecondary() throws Exception {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("dataCache");
- igfsCfg.setMetaCacheName("metaCache");
- igfsCfg.setName(IGFS_SECONDARY);
- igfsCfg.setBlockSize(512);
- igfsCfg.setDefaultMode(PRIMARY);
- igfsCfg.setIpcEndpointConfiguration(SECONDARY_REST_CFG);
-
- CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
-
- dataCacheCfg.setName("dataCache");
- dataCacheCfg.setCacheMode(PARTITIONED);
- dataCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
- dataCacheCfg.setBackups(0);
- dataCacheCfg.setQueryIndexEnabled(false);
- dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("metaCache");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- IgniteConfiguration cfg = new IgniteConfiguration();
-
- cfg.setGridName("grid-secondary");
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
-
- cfg.setLocalHost("127.0.0.1");
- cfg.setConnectorConfiguration(null);
-
- Ignite g = G.start(cfg);
-
- secondaryFs = g.fileSystem(IGFS_SECONDARY);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- try {
- // Cleanup.
- igfsPrimary.format();
-
- while (!dataCache.isEmpty())
- U.sleep(100);
-
- checkEvictionPolicy(0, 0);
- }
- finally {
- stopAllGrids(false);
- }
- }
-
- /**
- * Startup primary and secondary file systems.
- *
- * @throws Exception If failed.
- */
- private void start() throws Exception {
- startSecondary();
- startPrimary();
-
- evictPlc.setMaxBlocks(0);
- evictPlc.setMaxSize(0);
- evictPlc.setExcludePaths(null);
- }
-
- /**
- * Test how evictions are handled for a file working in PRIMARY mode.
- *
- * @throws Exception If failed.
- */
- public void testFilePrimary() throws Exception {
- start();
-
- // Create file in primary mode. It must not be propagated to eviction policy.
- igfsPrimary.create(FILE, true).close();
-
- checkEvictionPolicy(0, 0);
-
- int blockSize = igfsPrimary.info(FILE).blockSize();
-
- append(FILE, blockSize);
-
- checkEvictionPolicy(0, 0);
-
- read(FILE, 0, blockSize);
-
- checkEvictionPolicy(0, 0);
- }
-
- /**
- * Test how evictions are handled for a file working in PRIMARY mode.
- *
- * @throws Exception If failed.
- */
- public void testFileDual() throws Exception {
- start();
-
- igfsPrimary.create(FILE_RMT, true).close();
-
- checkEvictionPolicy(0, 0);
-
- int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
-
- // File write.
- append(FILE_RMT, blockSize);
-
- checkEvictionPolicy(1, blockSize);
-
- // One more write.
- append(FILE_RMT, blockSize);
-
- checkEvictionPolicy(2, blockSize * 2);
-
- // Read.
- read(FILE_RMT, 0, blockSize);
-
- checkEvictionPolicy(2, blockSize * 2);
- }
-
- /**
- * Ensure that a DUAL mode file is not propagated to eviction policy
- *
- * @throws Exception If failed.
- */
- public void testFileDualExclusion() throws Exception {
- start();
-
- evictPlc.setExcludePaths(Collections.singleton(FILE_RMT.toString()));
-
- // Create file in primary mode. It must not be propagated to eviction policy.
- igfsPrimary.create(FILE_RMT, true).close();
-
- checkEvictionPolicy(0, 0);
-
- int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
-
- append(FILE_RMT, blockSize);
-
- checkEvictionPolicy(0, 0);
-
- read(FILE_RMT, 0, blockSize);
-
- checkEvictionPolicy(0, 0);
- }
-
- /**
- * Ensure that exception is thrown in case we are trying to rename file with one exclude setting to the file with
- * another.
- *
- * @throws Exception If failed.
- */
- public void testRenameDifferentExcludeSettings() throws Exception {
- start();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- igfsPrimary.rename(FILE, FILE_RMT);
-
- return null;
- }
- }, IgfsInvalidPathException.class, "Cannot move file to a path with different eviction exclude setting " +
- "(need to copy and remove)");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- igfsPrimary.rename(FILE_RMT, FILE);
-
- return null;
- }
- }, IgfsInvalidPathException.class, "Cannot move file to a path with different eviction exclude setting " +
- "(need to copy and remove)");
- }
-
- /**
- * Test eviction caused by too much blocks.
- *
- * @throws Exception If failed.
- */
- public void testBlockCountEviction() throws Exception {
- start();
-
- int blockCnt = 3;
-
- evictPlc.setMaxBlocks(blockCnt);
-
- igfsPrimary.create(FILE_RMT, true).close();
-
- checkEvictionPolicy(0, 0);
-
- int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
-
- // Write blocks up to the limit.
- append(FILE_RMT, blockSize * blockCnt);
-
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
-
- // Write one more block what should cause eviction.
- append(FILE_RMT, blockSize);
-
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
-
- // Read the first block.
- read(FILE_RMT, 0, blockSize);
-
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
- checkMetrics(1, 1);
- }
-
- /**
- * Test eviction caused by too big data size.
- *
- * @throws Exception If failed.
- */
- public void testDataSizeEviction() throws Exception {
- start();
-
- igfsPrimary.create(FILE_RMT, true).close();
-
- int blockCnt = 3;
- int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
-
- evictPlc.setMaxSize(blockSize * blockCnt);
-
- // Write blocks up to the limit.
- append(FILE_RMT, blockSize * blockCnt);
-
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
-
- // Reset metrics.
- igfsPrimary.resetMetrics();
-
- // Read the first block what should cause reordering.
- read(FILE_RMT, 0, blockSize);
-
- checkMetrics(1, 0);
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
-
- // Write one more block what should cause eviction of the block 2.
- append(FILE_RMT, blockSize);
-
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
-
- // Read the first block.
- read(FILE_RMT, 0, blockSize);
-
- checkMetrics(2, 0);
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
-
- // Read the second block (which was evicted).
- read(FILE_RMT, blockSize, blockSize);
-
- checkMetrics(3, 1);
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
- }
-
- /**
- * Read some data from the given file with the given offset.
- *
- * @param path File path.
- * @param off Offset.
- * @param len Length.
- * @throws Exception If failed.
- */
- private void read(IgfsPath path, int off, int len) throws Exception {
- IgfsInputStream is = igfsPrimary.open(path);
-
- is.readFully(off, new byte[len]);
-
- is.close();
- }
-
- /**
- * Append some data to the given file.
- *
- * @param path File path.
- * @param len Data length.
- * @throws Exception If failed.
- */
- private void append(IgfsPath path, int len) throws Exception {
- IgfsOutputStream os = igfsPrimary.append(path, false);
-
- os.write(new byte[len]);
-
- os.close();
- }
-
- /**
- * Check metrics counters.
- *
- * @param blocksRead Expected blocks read.
- * @param blocksReadRmt Expected blocks read remote.
- * @throws Exception If failed.
- */
- public void checkMetrics(final long blocksRead, final long blocksReadRmt) throws Exception {
- assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- IgfsMetrics metrics = igfsPrimary.metrics();
-
- return metrics.blocksReadTotal() == blocksRead && metrics.blocksReadRemote() == blocksReadRmt;
- }
- }, 5000) : "Unexpected metrics [expectedBlocksReadTotal=" + blocksRead + ", actualBlocksReadTotal=" +
- igfsPrimary.metrics().blocksReadTotal() + ", expectedBlocksReadRemote=" + blocksReadRmt +
- ", actualBlocksReadRemote=" + igfsPrimary.metrics().blocksReadRemote() + ']';
- }
-
- /**
- * Check eviction policy state.
- *
- * @param curBlocks Current blocks.
- * @param curBytes Current bytes.
- */
- private void checkEvictionPolicy(final int curBlocks, final long curBytes) throws IgniteInterruptedCheckedException {
- assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- return evictPlc.getCurrentBlocks() == curBlocks && evictPlc.getCurrentSize() == curBytes;
- }
- }, 5000) : "Unexpected counts [expectedBlocks=" + curBlocks + ", actualBlocks=" + evictPlc.getCurrentBlocks() +
- ", expectedBytes=" + curBytes + ", currentBytes=" + curBytes + ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
index a7e6780..9a99611 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.*;
import org.apache.ignite.cache.*;
import org.apache.ignite.configuration.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.util.lang.*;
import org.apache.ignite.internal.util.typedef.*;
@@ -37,11 +38,11 @@ import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
-import static org.apache.ignite.IgniteFs.*;
import static org.apache.ignite.cache.CacheAtomicityMode.*;
import static org.apache.ignite.cache.CacheMemoryMode.*;
import static org.apache.ignite.cache.CacheMode.*;
import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
/**
* Test fo regular igfs operations.
@@ -154,7 +155,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
igfsSecondary = (IgfsImpl) igniteSecondary.fileSystem("igfs-secondary");
- Ignite ignite = startGridWithIgfs("ignite", "igfs", mode, igfsSecondary, PRIMARY_REST_CFG);
+ Ignite ignite = startGridWithIgfs("ignite", "igfs", mode, igfsSecondary.asSecondary(), PRIMARY_REST_CFG);
igfs = (IgfsImpl) ignite.fileSystem("igfs");
}
@@ -181,8 +182,8 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
protected Ignite startGridWithIgfs(String gridName, String igfsName, IgfsMode mode,
- @Nullable Igfs secondaryFs, @Nullable Map<String, String> restCfg) throws Exception {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable Map<String, String> restCfg) throws Exception {
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName("dataCache");
igfsCfg.setMetaCacheName("metaCache");
@@ -225,7 +226,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
cfg.setLocalHost("127.0.0.1");
cfg.setConnectorConfiguration(null);
@@ -262,7 +263,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testExists() throws Exception {
- create(igfs, paths(DIR), null);
+ create(igfs.asSecondary(), paths(DIR), null);
checkExist(igfs, igfsSecondary, DIR);
}
@@ -373,7 +374,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
IgfsPath file1 = new IgfsPath("/file1");
IgfsPath file2 = new IgfsPath("/file2");
- create(igfs, null, paths(file1));
+ create(igfs.asSecondary(), null, paths(file1));
igfs.rename(file1, file2);
@@ -405,7 +406,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
IgfsPath dir1 = new IgfsPath("/dir1");
IgfsPath dir2 = new IgfsPath("/dir2");
- create(igfs, paths(dir1), null);
+ create(igfs.asSecondary(), paths(dir1), null);
igfs.rename(dir1, dir2);
@@ -434,7 +435,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testMoveFileDestinationRoot() throws Exception {
- create(igfs, paths(DIR, SUBDIR), paths(FILE));
+ create(igfs.asSecondary(), paths(DIR, SUBDIR), paths(FILE));
igfs.rename(FILE, new IgfsPath());
@@ -526,7 +527,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testMoveDirectoryDestinationRoot() throws Exception {
- create(igfs, paths(DIR, SUBDIR, SUBSUBDIR), null);
+ create(igfs.asSecondary(), paths(DIR, SUBDIR, SUBSUBDIR), null);
igfs.rename(SUBSUBDIR, new IgfsPath());
@@ -542,7 +543,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testMoveDirectorySourceParentRoot() throws Exception {
IgfsPath dir = new IgfsPath("/" + SUBSUBDIR.name());
- create(igfs, paths(DIR_NEW, SUBDIR_NEW, dir), null);
+ create(igfs.asSecondary(), paths(DIR_NEW, SUBDIR_NEW, dir), null);
igfs.rename(dir, SUBDIR_NEW);
@@ -851,7 +852,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testOpen() throws Exception {
create(igfs, paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, true, chunk);
+ createFile(igfs.asSecondary(), FILE, true, chunk);
checkFileContent(igfs, FILE, chunk);
}
@@ -886,9 +887,9 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testCreate() throws Exception {
- create(igfs, paths(DIR, SUBDIR), null);
+ create(igfs.asSecondary(), paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, true, chunk);
+ createFile(igfs.asSecondary(), FILE, true, chunk);
checkFile(igfs, igfsSecondary, FILE, chunk);
}
@@ -901,7 +902,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testCreateParentRoot() throws Exception {
IgfsPath file = new IgfsPath("/" + FILE.name());
- createFile(igfs, file, true, chunk);
+ createFile(igfs.asSecondary(), file, true, chunk);
checkFile(igfs, igfsSecondary, file, chunk);
}
@@ -1092,7 +1093,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
assert igfs.exists(path);
}
- awaitFileClose(igfs, path);
+ awaitFileClose(igfs.asSecondary(), path);
checkFileContent(igfs, path, chunk);
}
@@ -1166,7 +1167,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
fut.get();
- awaitFileClose(igfs, FILE);
+ awaitFileClose(igfs.asSecondary(), FILE);
if (err.get() != null)
throw err.get();
@@ -1212,7 +1213,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testAppendNoClose() throws Exception {
create(igfs, paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, false);
+ createFile(igfs.asSecondary(), FILE, false);
GridTestUtils.assertThrowsInherited(log(), new Callable<Object>() {
@Override public Object call() throws Exception {
@@ -1241,7 +1242,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testAppendRenameNoClose() throws Exception {
create(igfs, paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, false);
+ createFile(igfs.asSecondary(), FILE, false);
IgfsOutputStream os = null;
@@ -1263,9 +1264,9 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testAppendRenameParentNoClose() throws Exception {
- create(igfs, paths(DIR, SUBDIR), null);
+ create(igfs.asSecondary(), paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, false);
+ createFile(igfs.asSecondary(), FILE, false);
IgfsOutputStream os = null;
@@ -1289,7 +1290,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testAppendDeleteNoClose() throws Exception {
create(igfs, paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, false);
+ createFile(igfs.asSecondary(), FILE, false);
GridTestUtils.assertThrows(log, new Callable<Object>() {
@Override public Object call() throws Exception {
@@ -1321,7 +1322,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testAppendDeleteParentNoClose() throws Exception {
create(igfs, paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, false);
+ createFile(igfs.asSecondary(), FILE, false);
GridTestUtils.assertThrows(log, new Callable<Object>() {
@Override public Object call() throws Exception {
@@ -1360,7 +1361,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
create(igfs, paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, false);
+ createFile(igfs.asSecondary(), FILE, false);
IgfsOutputStream os = null;
@@ -1388,7 +1389,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
int threadCnt = 10;
for (int i = 0; i < threadCnt; i++)
- createFile(igfs, new IgfsPath("/file" + i), false);
+ createFile(igfs.asSecondary(), new IgfsPath("/file" + i), false);
multithreaded(new Runnable() {
@Override public void run() {
@@ -1411,7 +1412,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
assert igfs.exists(path);
}
- awaitFileClose(igfs, path);
+ awaitFileClose(igfs.asSecondary(), path);
checkFileContent(igfs, path, chunks);
}
@@ -1483,7 +1484,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
fut.get();
- awaitFileClose(igfs, FILE);
+ awaitFileClose(igfs.asSecondary(), FILE);
if (err.get() != null)
throw err.get();
@@ -2137,7 +2138,19 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param files Files.
* @throws Exception If failed.
*/
- public static void create(Igfs igfs, @Nullable IgfsPath[] dirs, @Nullable IgfsPath[] files)
+ public static void create(IgfsImpl igfs, @Nullable IgfsPath[] dirs, @Nullable IgfsPath[] files) throws Exception {
+ create(igfs.asSecondary(), dirs, files);
+ }
+
+ /**
+ * Create the given directories and files in the given IGFS.
+ *
+ * @param igfs IGFS.
+ * @param dirs Directories.
+ * @param files Files.
+ * @throws Exception If failed.
+ */
+ public static void create(IgfsSecondaryFileSystem igfs, @Nullable IgfsPath[] dirs, @Nullable IgfsPath[] files)
throws Exception {
if (dirs != null) {
for (IgfsPath dir : dirs)
@@ -2162,7 +2175,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param chunks Data chunks.
* @throws IOException In case of IO exception.
*/
- protected static void createFile(Igfs igfs, IgfsPath file, boolean overwrite, @Nullable byte[]... chunks)
+ protected static void createFile(IgfsSecondaryFileSystem igfs, IgfsPath file, boolean overwrite, @Nullable byte[]... chunks)
throws IOException {
OutputStream os = null;
@@ -2188,7 +2201,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param chunks Data chunks.
* @throws Exception If failed.
*/
- protected void createFile(IgniteFs igfs, IgfsPath file, boolean overwrite, long blockSize,
+ protected void createFile(IgfsImpl igfs, IgfsPath file, boolean overwrite, long blockSize,
@Nullable byte[]... chunks) throws Exception {
IgfsOutputStream os = null;
@@ -2200,7 +2213,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
finally {
U.closeQuiet(os);
- awaitFileClose(igfs, file);
+ awaitFileClose(igfs.asSecondary(), file);
}
}
@@ -2212,7 +2225,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param chunks Data chunks.
* @throws Exception If failed.
*/
- protected void appendFile(IgniteFs igfs, IgfsPath file, @Nullable byte[]... chunks)
+ protected void appendFile(IgfsImpl igfs, IgfsPath file, @Nullable byte[]... chunks)
throws Exception {
IgfsOutputStream os = null;
@@ -2224,7 +2237,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
finally {
U.closeQuiet(os);
- awaitFileClose(igfs, file);
+ awaitFileClose(igfs.asSecondary(), file);
}
}
@@ -2248,7 +2261,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param igfs IGFS.
* @param file File.
*/
- public static void awaitFileClose(Igfs igfs, IgfsPath file) {
+ public static void awaitFileClose(IgfsSecondaryFileSystem igfs, IgfsPath file) {
try {
igfs.update(file, Collections.singletonMap("prop", "val"));
}
@@ -2418,7 +2431,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param igfsSecondary Second IGFS.
* @throws Exception If failed.
*/
- protected void clear(IgniteFs igfs, IgniteFs igfsSecondary) throws Exception {
+ protected void clear(IgniteFileSystem igfs, IgniteFileSystem igfsSecondary) throws Exception {
clear(igfs);
if (dual)
@@ -2431,7 +2444,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param igfs IGFS.
* @throws Exception If failed.
*/
- public static void clear(IgniteFs igfs) throws Exception {
+ public static void clear(IgniteFileSystem igfs) throws Exception {
Field workerMapFld = IgfsImpl.class.getDeclaredField("workerMap");
workerMapFld.setAccessible(true);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java
new file mode 100644
index 0000000..fd590f5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs;
+
+import org.apache.ignite.*;
+import org.apache.ignite.cache.*;
+import org.apache.ignite.cache.eviction.igfs.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.cache.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+import org.apache.ignite.testframework.*;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * Tests for IGFS per-block LR eviction policy.
+ */
+@SuppressWarnings({"ConstantConditions", "ThrowableResultOfMethodCallIgnored"})
+public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstractTest {
+ /** Primary IGFS name. */
+ private static final String IGFS_PRIMARY = "igfs-primary";
+
+ /** Primary IGFS name. */
+ private static final String IGFS_SECONDARY = "igfs-secondary";
+
+ /** Secondary file system REST endpoint configuration map. */
+ private static final Map<String, String> SECONDARY_REST_CFG = new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", "11500");
+ }};
+
+ /** File working in PRIMARY mode. */
+ public static final IgfsPath FILE = new IgfsPath("/file");
+
+ /** File working in DUAL mode. */
+ public static final IgfsPath FILE_RMT = new IgfsPath("/fileRemote");
+
+ /** Primary IGFS instances. */
+ private static IgfsImpl igfsPrimary;
+
+ /** Secondary IGFS instance. */
+ private static IgfsImpl secondaryFs;
+
+ /** Primary file system data cache. */
+ private static GridCacheAdapter<IgfsBlockKey, byte[]> dataCache;
+
+ /** Eviction policy */
+ private static CacheIgfsPerBlockLruEvictionPolicy evictPlc;
+
+ /**
+ * Start a grid with the primary file system.
+ *
+ * @throws Exception If failed.
+ */
+ private void startPrimary() throws Exception {
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("dataCache");
+ igfsCfg.setMetaCacheName("metaCache");
+ igfsCfg.setName(IGFS_PRIMARY);
+ igfsCfg.setBlockSize(512);
+ igfsCfg.setDefaultMode(PRIMARY);
+ igfsCfg.setPrefetchBlocks(1);
+ igfsCfg.setSequentialReadsBeforePrefetch(Integer.MAX_VALUE);
+ igfsCfg.setSecondaryFileSystem(secondaryFs.asSecondary());
+
+ Map<String, IgfsMode> pathModes = new HashMap<>();
+
+ pathModes.put(FILE_RMT.toString(), DUAL_SYNC);
+
+ igfsCfg.setPathModes(pathModes);
+
+ CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
+
+ dataCacheCfg.setName("dataCache");
+ dataCacheCfg.setCacheMode(PARTITIONED);
+ dataCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ evictPlc = new CacheIgfsPerBlockLruEvictionPolicy();
+
+ dataCacheCfg.setEvictionPolicy(evictPlc);
+ dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ dataCacheCfg.setBackups(0);
+ dataCacheCfg.setQueryIndexEnabled(false);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("metaCache");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("grid-primary");
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ cfg.setLocalHost("127.0.0.1");
+ cfg.setConnectorConfiguration(null);
+
+ Ignite g = G.start(cfg);
+
+ igfsPrimary = (IgfsImpl)g.fileSystem(IGFS_PRIMARY);
+
+ dataCache = igfsPrimary.context().kernalContext().cache().internalCache(
+ igfsPrimary.context().configuration().getDataCacheName());
+ }
+
+ /**
+ * Start a grid with the secondary file system.
+ *
+ * @throws Exception If failed.
+ */
+ private void startSecondary() throws Exception {
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
+
+ igfsCfg.setDataCacheName("dataCache");
+ igfsCfg.setMetaCacheName("metaCache");
+ igfsCfg.setName(IGFS_SECONDARY);
+ igfsCfg.setBlockSize(512);
+ igfsCfg.setDefaultMode(PRIMARY);
+ igfsCfg.setIpcEndpointConfiguration(SECONDARY_REST_CFG);
+
+ CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
+
+ dataCacheCfg.setName("dataCache");
+ dataCacheCfg.setCacheMode(PARTITIONED);
+ dataCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ dataCacheCfg.setBackups(0);
+ dataCacheCfg.setQueryIndexEnabled(false);
+ dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("metaCache");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("grid-secondary");
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
+
+ cfg.setLocalHost("127.0.0.1");
+ cfg.setConnectorConfiguration(null);
+
+ Ignite g = G.start(cfg);
+
+ secondaryFs = (IgfsImpl)g.fileSystem(IGFS_SECONDARY);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ try {
+ // Cleanup.
+ igfsPrimary.format();
+
+ while (!dataCache.isEmpty())
+ U.sleep(100);
+
+ checkEvictionPolicy(0, 0);
+ }
+ finally {
+ stopAllGrids(false);
+ }
+ }
+
+ /**
+ * Startup primary and secondary file systems.
+ *
+ * @throws Exception If failed.
+ */
+ private void start() throws Exception {
+ startSecondary();
+ startPrimary();
+
+ evictPlc.setMaxBlocks(0);
+ evictPlc.setMaxSize(0);
+ evictPlc.setExcludePaths(null);
+ }
+
+ /**
+ * Test how evictions are handled for a file working in PRIMARY mode.
+ *
+ * @throws Exception If failed.
+ */
+ public void testFilePrimary() throws Exception {
+ start();
+
+ // Create file in primary mode. It must not be propagated to eviction policy.
+ igfsPrimary.create(FILE, true).close();
+
+ checkEvictionPolicy(0, 0);
+
+ int blockSize = igfsPrimary.info(FILE).blockSize();
+
+ append(FILE, blockSize);
+
+ checkEvictionPolicy(0, 0);
+
+ read(FILE, 0, blockSize);
+
+ checkEvictionPolicy(0, 0);
+ }
+
+ /**
+ * Test how evictions are handled for a file working in PRIMARY mode.
+ *
+ * @throws Exception If failed.
+ */
+ public void testFileDual() throws Exception {
+ start();
+
+ igfsPrimary.create(FILE_RMT, true).close();
+
+ checkEvictionPolicy(0, 0);
+
+ int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
+
+ // File write.
+ append(FILE_RMT, blockSize);
+
+ checkEvictionPolicy(1, blockSize);
+
+ // One more write.
+ append(FILE_RMT, blockSize);
+
+ checkEvictionPolicy(2, blockSize * 2);
+
+ // Read.
+ read(FILE_RMT, 0, blockSize);
+
+ checkEvictionPolicy(2, blockSize * 2);
+ }
+
+ /**
+ * Ensure that a DUAL mode file is not propagated to eviction policy
+ *
+ * @throws Exception If failed.
+ */
+ public void testFileDualExclusion() throws Exception {
+ start();
+
+ evictPlc.setExcludePaths(Collections.singleton(FILE_RMT.toString()));
+
+ // Create file in primary mode. It must not be propagated to eviction policy.
+ igfsPrimary.create(FILE_RMT, true).close();
+
+ checkEvictionPolicy(0, 0);
+
+ int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
+
+ append(FILE_RMT, blockSize);
+
+ checkEvictionPolicy(0, 0);
+
+ read(FILE_RMT, 0, blockSize);
+
+ checkEvictionPolicy(0, 0);
+ }
+
+ /**
+ * Ensure that exception is thrown in case we are trying to rename file with one exclude setting to the file with
+ * another.
+ *
+ * @throws Exception If failed.
+ */
+ public void testRenameDifferentExcludeSettings() throws Exception {
+ start();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ igfsPrimary.rename(FILE, FILE_RMT);
+
+ return null;
+ }
+ }, IgfsInvalidPathException.class, "Cannot move file to a path with different eviction exclude setting " +
+ "(need to copy and remove)");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ igfsPrimary.rename(FILE_RMT, FILE);
+
+ return null;
+ }
+ }, IgfsInvalidPathException.class, "Cannot move file to a path with different eviction exclude setting " +
+ "(need to copy and remove)");
+ }
+
+ /**
+ * Test eviction caused by too much blocks.
+ *
+ * @throws Exception If failed.
+ */
+ public void testBlockCountEviction() throws Exception {
+ start();
+
+ int blockCnt = 3;
+
+ evictPlc.setMaxBlocks(blockCnt);
+
+ igfsPrimary.create(FILE_RMT, true).close();
+
+ checkEvictionPolicy(0, 0);
+
+ int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
+
+ // Write blocks up to the limit.
+ append(FILE_RMT, blockSize * blockCnt);
+
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+
+ // Write one more block what should cause eviction.
+ append(FILE_RMT, blockSize);
+
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+
+ // Read the first block.
+ read(FILE_RMT, 0, blockSize);
+
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+ checkMetrics(1, 1);
+ }
+
+ /**
+ * Test eviction caused by too big data size.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDataSizeEviction() throws Exception {
+ start();
+
+ igfsPrimary.create(FILE_RMT, true).close();
+
+ int blockCnt = 3;
+ int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
+
+ evictPlc.setMaxSize(blockSize * blockCnt);
+
+ // Write blocks up to the limit.
+ append(FILE_RMT, blockSize * blockCnt);
+
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+
+ // Reset metrics.
+ igfsPrimary.resetMetrics();
+
+ // Read the first block what should cause reordering.
+ read(FILE_RMT, 0, blockSize);
+
+ checkMetrics(1, 0);
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+
+ // Write one more block what should cause eviction of the block 2.
+ append(FILE_RMT, blockSize);
+
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+
+ // Read the first block.
+ read(FILE_RMT, 0, blockSize);
+
+ checkMetrics(2, 0);
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+
+ // Read the second block (which was evicted).
+ read(FILE_RMT, blockSize, blockSize);
+
+ checkMetrics(3, 1);
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+ }
+
+ /**
+ * Read some data from the given file with the given offset.
+ *
+ * @param path File path.
+ * @param off Offset.
+ * @param len Length.
+ * @throws Exception If failed.
+ */
+ private void read(IgfsPath path, int off, int len) throws Exception {
+ IgfsInputStream is = igfsPrimary.open(path);
+
+ is.readFully(off, new byte[len]);
+
+ is.close();
+ }
+
+ /**
+ * Append some data to the given file.
+ *
+ * @param path File path.
+ * @param len Data length.
+ * @throws Exception If failed.
+ */
+ private void append(IgfsPath path, int len) throws Exception {
+ IgfsOutputStream os = igfsPrimary.append(path, false);
+
+ os.write(new byte[len]);
+
+ os.close();
+ }
+
+ /**
+ * Check metrics counters.
+ *
+ * @param blocksRead Expected blocks read.
+ * @param blocksReadRmt Expected blocks read remote.
+ * @throws Exception If failed.
+ */
+ public void checkMetrics(final long blocksRead, final long blocksReadRmt) throws Exception {
+ assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ IgfsMetrics metrics = igfsPrimary.metrics();
+
+ return metrics.blocksReadTotal() == blocksRead && metrics.blocksReadRemote() == blocksReadRmt;
+ }
+ }, 5000) : "Unexpected metrics [expectedBlocksReadTotal=" + blocksRead + ", actualBlocksReadTotal=" +
+ igfsPrimary.metrics().blocksReadTotal() + ", expectedBlocksReadRemote=" + blocksReadRmt +
+ ", actualBlocksReadRemote=" + igfsPrimary.metrics().blocksReadRemote() + ']';
+ }
+
+ /**
+ * Check eviction policy state.
+ *
+ * @param curBlocks Current blocks.
+ * @param curBytes Current bytes.
+ */
+ private void checkEvictionPolicy(final int curBlocks, final long curBytes) throws IgniteInterruptedCheckedException {
+ assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return evictPlc.getCurrentBlocks() == curBlocks && evictPlc.getCurrentSize() == curBytes;
+ }
+ }, 5000) : "Unexpected counts [expectedBlocks=" + curBlocks + ", actualBlocks=" + evictPlc.getCurrentBlocks() +
+ ", expectedBytes=" + curBytes + ", currentBytes=" + curBytes + ']';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java
index bde395b..255e319 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java
@@ -57,13 +57,13 @@ public class IgfsCacheSelfTest extends IgfsCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setMetaCacheName(META_CACHE_NAME);
igfsCfg.setDataCacheName(DATA_CACHE_NAME);
igfsCfg.setName("igfs");
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
return cfg;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java
index 2e48648..297cc7b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java
@@ -24,7 +24,6 @@ import org.apache.ignite.igfs.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.processors.cache.*;
import org.apache.ignite.internal.processors.cache.transactions.*;
-import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.apache.ignite.lang.*;
@@ -96,7 +95,7 @@ public class IgfsDataManagerSelfTest extends IgfsCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setMetaCacheName(META_CACHE_NAME);
igfsCfg.setDataCacheName(DATA_CACHE_NAME);
@@ -104,7 +103,7 @@ public class IgfsDataManagerSelfTest extends IgfsCommonAbstractTest {
igfsCfg.setName("igfs");
igfsCfg.setBlockSize(BLOCK_SIZE);
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
return cfg;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
index 327d7fa..f44a988 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
@@ -28,8 +28,8 @@ import java.io.*;
import java.util.*;
import java.util.concurrent.*;
-import static org.apache.ignite.IgniteFs.*;
import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
/**
* Tests for IGFS working in mode when remote file system exists: DUAL_SYNC, DUAL_ASYNC.
@@ -1120,7 +1120,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest {
create(igfsSecondary, paths(DIR, SUBDIR), null);
create(igfs, null, null);
- createFile(igfsSecondary, FILE, true, chunk);
+ createFile(igfsSecondary.asSecondary(), FILE, true, chunk);
checkFileContent(igfs, FILE, chunk);
}
@@ -1148,7 +1148,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest {
out.close();
- awaitFileClose(igfsSecondary, FILE);
+ awaitFileClose(igfsSecondary.asSecondary(), FILE);
// Read the first block.
int totalRead = 0;
@@ -1227,7 +1227,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest {
out.close();
- awaitFileClose(igfsSecondary, FILE);
+ awaitFileClose(igfsSecondary.asSecondary(), FILE);
// Read the first two blocks.
int totalRead = 0;
@@ -1292,7 +1292,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest {
igfsSecondary.update(SUBDIR, props);
- createFile(igfs, FILE, true, chunk);
+ createFile(igfs.asSecondary(), FILE, true, chunk);
// Ensure that directory structure was created.
checkExist(igfs, igfsSecondary, SUBDIR);
@@ -1317,7 +1317,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest {
igfsSecondary.update(DIR, propsDir);
igfsSecondary.update(SUBDIR, propsSubDir);
- createFile(igfs, FILE, true, chunk);
+ createFile(igfs.asSecondary(), FILE, true, chunk);
checkExist(igfs, igfsSecondary, SUBDIR);
checkFile(igfs, igfsSecondary, FILE, chunk);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
index 1a52b7c..11d6cc5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
@@ -75,13 +75,13 @@ public class IgfsMetaManagerSelfTest extends IgfsCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setMetaCacheName(META_CACHE_NAME);
igfsCfg.setDataCacheName(DATA_CACHE_NAME);
igfsCfg.setName("igfs");
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
return cfg;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
index be5afe7..0af1dea 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
@@ -56,10 +56,10 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
/** Primary IGFS instances. */
- private static IgniteFs[] igfsPrimary;
+ private static IgniteFileSystem[] igfsPrimary;
/** Secondary IGFS instance. */
- private static IgniteFs igfsSecondary;
+ private static IgfsImpl igfsSecondary;
/** Primary file system block size. */
public static final int PRIMARY_BLOCK_SIZE = 512;
@@ -84,7 +84,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
private void startPrimary() throws Exception {
- igfsPrimary = new IgniteFs[NODES_CNT];
+ igfsPrimary = new IgniteFileSystem[NODES_CNT];
for (int i = 0; i < NODES_CNT; i++) {
Ignite g = G.start(primaryConfiguration(i));
@@ -101,14 +101,14 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
private IgniteConfiguration primaryConfiguration(int idx) throws Exception {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName("dataCache");
igfsCfg.setMetaCacheName("metaCache");
igfsCfg.setName(IGFS_PRIMARY);
igfsCfg.setBlockSize(PRIMARY_BLOCK_SIZE);
igfsCfg.setDefaultMode(PRIMARY);
- igfsCfg.setSecondaryFileSystem(igfsSecondary);
+ igfsCfg.setSecondaryFileSystem(igfsSecondary.asSecondary());
Map<String, IgfsMode> pathModes = new HashMap<>();
@@ -146,7 +146,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
cfg.setLocalHost("127.0.0.1");
@@ -159,7 +159,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
private void startSecondary() throws Exception {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName("dataCache");
igfsCfg.setMetaCacheName("metaCache");
@@ -198,18 +198,18 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
cfg.setLocalHost("127.0.0.1");
Ignite g = G.start(cfg);
- igfsSecondary = g.fileSystem(IGFS_SECONDARY);
+ igfsSecondary = (IgfsImpl)g.fileSystem(IGFS_SECONDARY);
}
/** @throws Exception If failed. */
public void testMetrics() throws Exception {
- IgniteFs fs = igfsPrimary[0];
+ IgniteFileSystem fs = igfsPrimary[0];
assertNotNull(fs);
@@ -349,7 +349,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
/** @throws Exception If failed. */
public void testMultipleClose() throws Exception {
- IgniteFs fs = igfsPrimary[0];
+ IgniteFileSystem fs = igfsPrimary[0];
IgfsOutputStream out = fs.create(new IgfsPath("/file"), false);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java
index 2b501f7..4a58285 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java
@@ -82,7 +82,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest {
private void startUp() throws Exception {
startUpSecondary();
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName("partitioned");
igfsCfg.setMetaCacheName("replicated");
@@ -97,7 +97,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest {
igfsCfg.setPathModes(pathModes);
if (setSecondaryFs)
- igfsCfg.setSecondaryFileSystem(igfsSecondary);
+ igfsCfg.setSecondaryFileSystem(igfsSecondary.asSecondary());
CacheConfiguration cacheCfg = defaultCacheConfiguration();
@@ -128,7 +128,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
cfg.setLocalHost("127.0.0.1");
cfg.setConnectorConfiguration(null);
@@ -144,7 +144,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
private void startUpSecondary() throws Exception {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName("partitioned");
igfsCfg.setMetaCacheName("replicated");
@@ -185,7 +185,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
cfg.setLocalHost("127.0.0.1");
cfg.setConnectorConfiguration(null);
@@ -378,7 +378,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest {
}
assertTrue(errMsg.startsWith(
- "Grid configuration parameter invalid: secondaryFileSystem cannot be null when mode is SECONDARY"));
+ "Grid configuration parameter invalid: secondaryFileSystem cannot be null when mode is not PRIMARY"));
}
/**
@@ -444,7 +444,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest {
}
assertTrue(errMsg.startsWith(
- "Grid configuration parameter invalid: secondaryFileSystem cannot be null when mode is SECONDARY"));
+ "Grid configuration parameter invalid: secondaryFileSystem cannot be null when mode is not PRIMARY"));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
index ead7511..0e03aa8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
@@ -59,7 +59,7 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest {
protected final SecureRandom rnd = new SecureRandom();
/** File system. */
- protected IgniteFs igfs;
+ protected IgniteFileSystem igfs;
/** Meta cache. */
private GridCache<Object, Object> metaCache;
@@ -73,7 +73,7 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest {
igfs = grid.fileSystem(igfsName());
- IgfsConfiguration[] cfgs = grid.configuration().getIgfsConfiguration();
+ FileSystemConfiguration[] cfgs = grid.configuration().getFileSystemConfiguration();
assert cfgs.length == 1;
@@ -111,13 +111,13 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setMetaCacheName(META_CACHE_NAME);
igfsCfg.setDataCacheName(DATA_CACHE_NAME);
igfsCfg.setName("igfs");
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
return cfg;
}
@@ -157,7 +157,7 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest {
/** @throws Exception If failed. */
public void testigfsEnabled() throws Exception {
- IgniteFs igfs = grid(0).fileSystem(igfsName());
+ IgniteFileSystem igfs = grid(0).fileSystem(igfsName());
assertNotNull(igfs);
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorValidationSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorValidationSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorValidationSelfTest.java
index 6f444c0..dcd0413 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorValidationSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorValidationSelfTest.java
@@ -37,7 +37,7 @@ import static org.apache.ignite.igfs.IgfsMode.*;
* Tests for node validation logic in {@link IgfsProcessor}.
* <p>
* Tests starting with "testLocal" are checking
- * {@link IgfsProcessor#validateLocalIgfsConfigurations(org.apache.ignite.configuration.IgfsConfiguration[])}.
+ * {@link IgfsProcessor#validateLocalIgfsConfigurations(org.apache.ignite.configuration.FileSystemConfiguration[])}.
* <p>
* Tests starting with "testRemote" are checking {@link IgfsProcessor#checkIgfsOnRemoteNode(org.apache.ignite.cluster.ClusterNode)}.
*/
@@ -61,10 +61,10 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
private static final String metaCache2Name = "metaCache2";
/** First IGFS config in grid #1. */
- private IgfsConfiguration g1IgfsCfg1 = new IgfsConfiguration();
+ private FileSystemConfiguration g1IgfsCfg1 = new FileSystemConfiguration();
/** Second IGFS config in grid#1. */
- private IgfsConfiguration g1IgfsCfg2 = new IgfsConfiguration();
+ private FileSystemConfiguration g1IgfsCfg2 = new FileSystemConfiguration();
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
@@ -89,7 +89,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
g1IgfsCfg2.setDataCacheName(dataCache2Name);
g1IgfsCfg2.setMetaCacheName(metaCache2Name);
- cfg.setIgfsConfiguration(g1IgfsCfg1, g1IgfsCfg2);
+ cfg.setFileSystemConfiguration(g1IgfsCfg1, g1IgfsCfg2);
cfg.setLocalHost("127.0.0.1");
@@ -266,7 +266,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
g1IgfsCfg2.setDefaultMode(PROXY);
- checkGridStartFails(g1Cfg, "secondaryFileSystem cannot be null when mode is SECONDARY", true);
+ checkGridStartFails(g1Cfg, "secondaryFileSystem cannot be null when mode is not PRIMARY", true);
}
/**
@@ -278,11 +278,11 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
g1Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches(), CacheConfiguration.class));
g2Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches(), CacheConfiguration.class));
- IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1);
+ FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1);
g2IgfsCfg1.setBlockSize(g2IgfsCfg1.getBlockSize() + 100);
- g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g1IgfsCfg2);
+ g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g1IgfsCfg2);
G.start(g1Cfg);
@@ -310,8 +310,8 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
public void testRemoteIfMetaCacheNameDiffers() throws Exception {
IgniteConfiguration g2Cfg = getConfiguration("g2");
- IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1);
- IgfsConfiguration g2IgfsCfg2 = new IgfsConfiguration(g1IgfsCfg2);
+ FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1);
+ FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2);
g2IgfsCfg1.setMetaCacheName("g2MetaCache1");
g2IgfsCfg2.setMetaCacheName("g2MetaCache2");
@@ -320,7 +320,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
g2Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches("g2MetaCache1", "g2MetaCache2"),
CacheConfiguration.class));
- g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g2IgfsCfg2);
+ g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2);
G.start(g1Cfg);
@@ -333,8 +333,8 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
public void testRemoteIfMetaCacheNameEquals() throws Exception {
IgniteConfiguration g2Cfg = getConfiguration("g2");
- IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1);
- IgfsConfiguration g2IgfsCfg2 = new IgfsConfiguration(g1IgfsCfg2);
+ FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1);
+ FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2);
g2IgfsCfg1.setName("g2IgfsCfg1");
g2IgfsCfg2.setName("g2IgfsCfg2");
@@ -346,7 +346,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
g2Cfg.setCacheConfiguration(concat(dataCaches(1024, "g2DataCache1", "g2DataCache2"), metaCaches(),
CacheConfiguration.class));
- g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g2IgfsCfg2);
+ g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2);
G.start(g1Cfg);
@@ -359,8 +359,8 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
public void testRemoteIfDataCacheNameDiffers() throws Exception {
IgniteConfiguration g2Cfg = getConfiguration("g2");
- IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1);
- IgfsConfiguration g2IgfsCfg2 = new IgfsConfiguration(g1IgfsCfg2);
+ FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1);
+ FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2);
g2IgfsCfg1.setDataCacheName("g2DataCache1");
g2IgfsCfg2.setDataCacheName("g2DataCache2");
@@ -369,7 +369,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
g2Cfg.setCacheConfiguration(concat(dataCaches(1024, "g2DataCache1", "g2DataCache2"), metaCaches(),
CacheConfiguration.class));
- g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g2IgfsCfg2);
+ g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2);
G.start(g1Cfg);
@@ -382,8 +382,8 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
public void testRemoteIfDataCacheNameEquals() throws Exception {
IgniteConfiguration g2Cfg = getConfiguration("g2");
- IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1);
- IgfsConfiguration g2IgfsCfg2 = new IgfsConfiguration(g1IgfsCfg2);
+ FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1);
+ FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2);
g2IgfsCfg1.setName("g2IgfsCfg1");
g2IgfsCfg2.setName("g2IgfsCfg2");
@@ -395,7 +395,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
g2Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches("g2MetaCache1", "g2MetaCache2"),
CacheConfiguration.class));
- g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g2IgfsCfg2);
+ g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2);
G.start(g1Cfg);
@@ -408,8 +408,8 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
public void testRemoteIfDefaultModeDiffers() throws Exception {
IgniteConfiguration g2Cfg = getConfiguration("g2");
- IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1);
- IgfsConfiguration g2IgfsCfg2 = new IgfsConfiguration(g1IgfsCfg2);
+ FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1);
+ FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2);
g1IgfsCfg1.setDefaultMode(DUAL_ASYNC);
g1IgfsCfg2.setDefaultMode(DUAL_ASYNC);
@@ -420,7 +420,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
g1Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches(), CacheConfiguration.class));
g2Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches(), CacheConfiguration.class));
- g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g2IgfsCfg2);
+ g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2);
G.start(g1Cfg);
@@ -433,8 +433,8 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
public void testRemoteIfPathModeDiffers() throws Exception {
IgniteConfiguration g2Cfg = getConfiguration("g2");
- IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1);
- IgfsConfiguration g2IgfsCfg2 = new IgfsConfiguration(g1IgfsCfg2);
+ FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1);
+ FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2);
g2IgfsCfg1.setPathModes(Collections.singletonMap("/somePath", DUAL_SYNC));
g2IgfsCfg2.setPathModes(Collections.singletonMap("/somePath", DUAL_SYNC));
@@ -442,7 +442,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest {
g1Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches(), CacheConfiguration.class));
g2Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches(), CacheConfiguration.class));
- g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g2IgfsCfg2);
+ g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2);
G.start(g1Cfg);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest.java
index ee8e7bd..694d5c3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest.java
@@ -36,7 +36,7 @@ import java.util.*;
import java.util.concurrent.atomic.*;
import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.configuration.IgfsConfiguration.*;
+import static org.apache.ignite.configuration.FileSystemConfiguration.*;
/**
* Base test class for {@link IgfsServer} checking IPC endpoint registrations.
@@ -58,7 +58,7 @@ public abstract class IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest e
public void testLoopbackEndpointsRegistration() throws Exception {
IgniteConfiguration cfg = gridConfiguration();
- cfg.setIgfsConfiguration(
+ cfg.setFileSystemConfiguration(
igfsConfiguration("tcp", DFLT_IPC_PORT, null)
);
@@ -77,7 +77,7 @@ public abstract class IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest e
public void testLoopbackEndpointsCustomHostRegistration() throws Exception {
IgniteConfiguration cfg = gridConfiguration();
- cfg.setIgfsConfiguration(
+ cfg.setFileSystemConfiguration(
igfsConfiguration("tcp", DFLT_IPC_PORT, "127.0.0.1"),
igfsConfiguration("tcp", DFLT_IPC_PORT + 1, U.getLocalHost().getHostName()));
@@ -154,7 +154,7 @@ public abstract class IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest e
* @param endPntHost End point host.
* @return test-purposed IgfsConfiguration.
*/
- protected IgfsConfiguration igfsConfiguration(@Nullable String endPntType, @Nullable Integer endPntPort,
+ protected FileSystemConfiguration igfsConfiguration(@Nullable String endPntType, @Nullable Integer endPntPort,
@Nullable String endPntHost) throws IgniteCheckedException {
HashMap<String, String> endPntCfg = null;
@@ -170,7 +170,7 @@ public abstract class IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest e
endPntCfg.put("host", endPntHost);
}
- IgfsConfiguration igfsConfiguration = new IgfsConfiguration();
+ FileSystemConfiguration igfsConfiguration = new FileSystemConfiguration();
igfsConfiguration.setDataCacheName("partitioned");
igfsConfiguration.setMetaCacheName("replicated");
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest.java
index d7319f4..57f10d9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest.java
@@ -20,7 +20,7 @@ package org.apache.ignite.internal.processors.igfs;
import org.apache.ignite.configuration.*;
import org.apache.ignite.internal.util.typedef.*;
-import static org.apache.ignite.configuration.IgfsConfiguration.*;
+import static org.apache.ignite.configuration.FileSystemConfiguration.*;
/**
* Tests for {@link IgfsServer} that checks all IPC endpoint registration types
@@ -34,7 +34,7 @@ public class IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest
public void testLoopbackAndShmemEndpointsRegistration() throws Exception {
IgniteConfiguration cfg = gridConfiguration();
- cfg.setIgfsConfiguration(
+ cfg.setFileSystemConfiguration(
igfsConfiguration(null, null, null), // Check null IPC endpoint config won't bring any hassles.
igfsConfiguration("tcp", DFLT_IPC_PORT + 1, null),
igfsConfiguration("shmem", DFLT_IPC_PORT + 2, null));
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest.java
index 259f31a..4f18aff 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest.java
@@ -40,7 +40,7 @@ public class IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest
@Override public Object call() throws Exception {
IgniteConfiguration cfg = gridConfiguration();
- cfg.setIgfsConfiguration(igfsConfiguration("shmem", IpcSharedMemoryServerEndpoint.DFLT_IPC_PORT,
+ cfg.setFileSystemConfiguration(igfsConfiguration("shmem", IpcSharedMemoryServerEndpoint.DFLT_IPC_PORT,
null));
return G.start(cfg);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java
index b212f02..c62e759 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java
@@ -111,7 +111,7 @@ public class IgfsSizeSelfTest extends IgfsCommonAbstractTest {
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName(DATA_CACHE_NAME);
igfsCfg.setMetaCacheName(META_CACHE_NAME);
@@ -154,7 +154,7 @@ public class IgfsSizeSelfTest extends IgfsCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
cfg.setCacheConfiguration(metaCfg, dataCfg);
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
return cfg;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java
index 054665d..25816c7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java
@@ -74,7 +74,7 @@ public class IgfsStreamsSelfTest extends IgfsCommonAbstractTest {
public static final int ASSERT_RETRY_INTERVAL = 100;
/** File system to test. */
- private IgniteFs fs;
+ private IgniteFileSystem fs;
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
@@ -110,7 +110,7 @@ public class IgfsStreamsSelfTest extends IgfsCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setMetaCacheName(META_CACHE_NAME);
igfsCfg.setDataCacheName(DATA_CACHE_NAME);
@@ -118,7 +118,7 @@ public class IgfsStreamsSelfTest extends IgfsCommonAbstractTest {
igfsCfg.setBlockSize(CFG_BLOCK_SIZE);
igfsCfg.setFragmentizerEnabled(true);
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
return cfg;
}
@@ -229,12 +229,12 @@ public class IgfsStreamsSelfTest extends IgfsCommonAbstractTest {
IgfsPath path = new IgfsPath("/file");
try {
- IgniteFs fs0 = grid(0).fileSystem("igfs");
- IgniteFs fs1 = grid(1).fileSystem("igfs");
- IgniteFs fs2 = grid(2).fileSystem("igfs");
+ IgniteFileSystem fs0 = grid(0).fileSystem("igfs");
+ IgniteFileSystem fs1 = grid(1).fileSystem("igfs");
+ IgniteFileSystem fs2 = grid(2).fileSystem("igfs");
try (IgfsOutputStream out = fs0.create(path, 128, false, 1, CFG_GRP_SIZE,
- F.asMap(IgniteFs.PROP_PREFER_LOCAL_WRITES, "true"))) {
+ F.asMap(IgfsEx.PROP_PREFER_LOCAL_WRITES, "true"))) {
// 1.5 blocks
byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java
index 5b7a636..d1778df 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java
@@ -68,7 +68,7 @@ public class IgfsTaskSelfTest extends IgfsCommonAbstractTest {
private static final int REPEAT_CNT = 10;
/** IGFS. */
- private static IgniteFs igfs;
+ private static IgniteFileSystem igfs;
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
@@ -97,7 +97,7 @@ public class IgfsTaskSelfTest extends IgfsCommonAbstractTest {
* @return Grid configuration
*/
private IgniteConfiguration config(int idx) {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName("dataCache");
igfsCfg.setMetaCacheName("metaCache");
@@ -133,7 +133,7 @@ public class IgfsTaskSelfTest extends IgfsCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
cfg.setGridName("node-" + idx);
@@ -172,7 +172,7 @@ public class IgfsTaskSelfTest extends IgfsCommonAbstractTest {
assertFalse(igfs.isAsync());
- IgniteFs igfsAsync = igfs.withAsync();
+ IgniteFileSystem igfsAsync = igfs.withAsync();
assertTrue(igfsAsync.isAsync());
@@ -269,7 +269,7 @@ public class IgfsTaskSelfTest extends IgfsCommonAbstractTest {
private ComputeJobContext ctx;
/** {@inheritDoc} */
- @Override public Object execute(IgniteFs igfs, IgfsFileRange range, IgfsInputStream in)
+ @Override public Object execute(IgniteFileSystem igfs, IgfsFileRange range, IgfsInputStream in)
throws IOException {
assert ignite != null;
assert ses != null;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java
index 05de61a..487d391 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java
@@ -46,11 +46,11 @@ public class IgfsAbstractRecordResolverSelfTest extends GridCommonAbstractTest {
private final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
/** IGFS. */
- protected static IgniteFs igfs;
+ protected static IgniteFileSystem igfs;
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName("dataCache");
igfsCfg.setMetaCacheName("metaCache");
@@ -87,7 +87,7 @@ public class IgfsAbstractRecordResolverSelfTest extends GridCommonAbstractTest {
cfg.setDiscoverySpi(discoSpi);
cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
Ignite g = G.start(cfg);
[32/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
new file mode 100644
index 0000000..e0c5916
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.hadoop.mapreduce.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
+import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
+import org.apache.ignite.internal.processors.hadoop.shuffle.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
+import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
+import org.apache.ignite.internal.util.tostring.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.util.*;
+import java.util.concurrent.atomic.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopClassLoader.*;
+
+/**
+ * Hadoop processor.
+ */
+public class HadoopProcessor extends HadoopProcessorAdapter {
+ /** Job ID counter. */
+ private final AtomicInteger idCtr = new AtomicInteger();
+
+ /** Hadoop context. */
+ @GridToStringExclude
+ private HadoopContext hctx;
+
+ /** Hadoop facade for public API. */
+ @GridToStringExclude
+ private Hadoop hadoop;
+
+ /**
+ * @param ctx Kernal context.
+ */
+ public HadoopProcessor(GridKernalContext ctx) {
+ super(ctx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start() throws IgniteCheckedException {
+ if (ctx.isDaemon())
+ return;
+
+ HadoopConfiguration cfg = ctx.config().getHadoopConfiguration();
+
+ if (cfg == null)
+ cfg = new HadoopConfiguration();
+ else
+ cfg = new HadoopConfiguration(cfg);
+
+ initializeDefaults(cfg);
+
+ validate(cfg);
+
+ if (hadoopHome() != null)
+ U.quietAndInfo(log, "HADOOP_HOME is set to " + hadoopHome());
+
+ boolean ok = false;
+
+ try { // Check for Hadoop installation.
+ hadoopUrls();
+
+ ok = true;
+ }
+ catch (IgniteCheckedException e) {
+ U.quietAndWarn(log, e.getMessage());
+ }
+
+ if (ok) {
+ hctx = new HadoopContext(
+ ctx,
+ cfg,
+ new HadoopJobTracker(),
+ cfg.isExternalExecution() ? new HadoopExternalTaskExecutor() : new HadoopEmbeddedTaskExecutor(),
+ new HadoopShuffle());
+
+
+ for (HadoopComponent c : hctx.components())
+ c.start(hctx);
+
+ hadoop = new HadoopImpl(this);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopProcessor.class, this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop(boolean cancel) throws IgniteCheckedException {
+ super.stop(cancel);
+
+ if (hctx == null)
+ return;
+
+ List<HadoopComponent> components = hctx.components();
+
+ for (ListIterator<HadoopComponent> it = components.listIterator(components.size()); it.hasPrevious();) {
+ HadoopComponent c = it.previous();
+
+ c.stop(cancel);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onKernalStart() throws IgniteCheckedException {
+ super.onKernalStart();
+
+ if (hctx == null)
+ return;
+
+ for (HadoopComponent c : hctx.components())
+ c.onKernalStart();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onKernalStop(boolean cancel) {
+ super.onKernalStop(cancel);
+
+ if (hctx == null)
+ return;
+
+ List<HadoopComponent> components = hctx.components();
+
+ for (ListIterator<HadoopComponent> it = components.listIterator(components.size()); it.hasPrevious();) {
+ HadoopComponent c = it.previous();
+
+ c.onKernalStop(cancel);
+ }
+ }
+
+ /**
+ * Gets Hadoop context.
+ *
+ * @return Hadoop context.
+ */
+ public HadoopContext context() {
+ return hctx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Hadoop hadoop() {
+ if (hadoop == null)
+ throw new IllegalStateException("Hadoop accelerator is disabled (Hadoop is not in classpath, " +
+ "is HADOOP_HOME environment variable set?)");
+
+ return hadoop;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopConfiguration config() {
+ return hctx.configuration();
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopJobId nextJobId() {
+ return new HadoopJobId(ctx.localNodeId(), idCtr.incrementAndGet());
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteInternalFuture<?> submit(HadoopJobId jobId, HadoopJobInfo jobInfo) {
+ return hctx.jobTracker().submit(jobId, jobInfo);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopJobStatus status(HadoopJobId jobId) throws IgniteCheckedException {
+ return hctx.jobTracker().status(jobId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopCounters counters(HadoopJobId jobId) throws IgniteCheckedException {
+ return hctx.jobTracker().jobCounters(jobId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteInternalFuture<?> finishFuture(HadoopJobId jobId) throws IgniteCheckedException {
+ return hctx.jobTracker().finishFuture(jobId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean kill(HadoopJobId jobId) throws IgniteCheckedException {
+ return hctx.jobTracker().killJob(jobId);
+ }
+
+ /**
+ * Initializes default hadoop configuration.
+ *
+ * @param cfg Hadoop configuration.
+ */
+ private void initializeDefaults(HadoopConfiguration cfg) {
+ if (cfg.getMapReducePlanner() == null)
+ cfg.setMapReducePlanner(new IgniteHadoopMapReducePlanner());
+ }
+
+ /**
+ * Validates Grid and Hadoop configuration for correctness.
+ *
+ * @param hadoopCfg Hadoop configuration.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void validate(HadoopConfiguration hadoopCfg) throws IgniteCheckedException {
+ if (ctx.config().isPeerClassLoadingEnabled())
+ throw new IgniteCheckedException("Peer class loading cannot be used with Hadoop (disable it using " +
+ "GridConfiguration.setPeerClassLoadingEnabled()).");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopSetup.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopSetup.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopSetup.java
new file mode 100644
index 0000000..35df5da
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopSetup.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.net.*;
+import java.nio.file.*;
+import java.text.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.IgniteVersionUtils.*;
+
+/**
+ * Setup tool to configure Hadoop client.
+ */
+public class HadoopSetup {
+ /** */
+ public static final String WINUTILS_EXE = "winutils.exe";
+
+ /** */
+ private static final FilenameFilter IGNITE_JARS = new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return name.startsWith("ignite-") && name.endsWith(".jar");
+ }
+ };
+
+ /**
+ * The main method.
+ * @param ignore Params.
+ */
+ public static void main(String[] ignore) {
+ X.println(
+ " __________ ________________ ",
+ " / _/ ___/ |/ / _/_ __/ __/ ",
+ " _/ // (_ / // / / / / _/ ",
+ "/___/\\___/_/|_/___/ /_/ /___/ ",
+ " for Apache Hadoop ",
+ " ");
+
+ println("Version " + ACK_VER_STR);
+
+ configureHadoop();
+ }
+
+ /**
+ * This operation prepares the clean unpacked Hadoop distributive to work as client with Ignite-Hadoop.
+ * It performs these operations:
+ * <ul>
+ * <li>Check for setting of HADOOP_HOME environment variable.</li>
+ * <li>Try to resolve HADOOP_COMMON_HOME or evaluate it relative to HADOOP_HOME.</li>
+ * <li>In Windows check if winutils.exe exists and try to fix issue with some restrictions.</li>
+ * <li>In Windows check new line character issues in CMD scripts.</li>
+ * <li>Scan Hadoop lib directory to detect Ignite JARs. If these don't exist tries to create ones.</li>
+ * </ul>
+ */
+ private static void configureHadoop() {
+ String igniteHome = U.getIgniteHome();
+
+ println("IGNITE_HOME is set to '" + igniteHome + "'.");
+
+ checkIgniteHome(igniteHome);
+
+ String homeVar = "HADOOP_HOME";
+ String hadoopHome = System.getenv(homeVar);
+
+ if (F.isEmpty(hadoopHome)) {
+ homeVar = "HADOOP_PREFIX";
+ hadoopHome = System.getenv(homeVar);
+ }
+
+ if (F.isEmpty(hadoopHome))
+ exit("Neither HADOOP_HOME nor HADOOP_PREFIX environment variable is set. Please set one of them to a " +
+ "valid Hadoop installation directory and run setup tool again.", null);
+
+ hadoopHome = hadoopHome.replaceAll("\"", "");
+
+ println(homeVar + " is set to '" + hadoopHome + "'.");
+
+ String hiveHome = System.getenv("HIVE_HOME");
+
+ if (!F.isEmpty(hiveHome)) {
+ hiveHome = hiveHome.replaceAll("\"", "");
+
+ println("HIVE_HOME is set to '" + hiveHome + "'.");
+ }
+
+ File hadoopDir = new File(hadoopHome);
+
+ if (!hadoopDir.exists())
+ exit("Hadoop installation folder does not exist.", null);
+
+ if (!hadoopDir.isDirectory())
+ exit("HADOOP_HOME must point to a directory.", null);
+
+ if (!hadoopDir.canRead())
+ exit("Hadoop installation folder can not be read. Please check permissions.", null);
+
+ File hadoopCommonDir;
+
+ String hadoopCommonHome = System.getenv("HADOOP_COMMON_HOME");
+
+ if (F.isEmpty(hadoopCommonHome)) {
+ hadoopCommonDir = new File(hadoopDir, "share/hadoop/common");
+
+ println("HADOOP_COMMON_HOME is not set, will use '" + hadoopCommonDir.getPath() + "'.");
+ }
+ else {
+ println("HADOOP_COMMON_HOME is set to '" + hadoopCommonHome + "'.");
+
+ hadoopCommonDir = new File(hadoopCommonHome);
+ }
+
+ if (!hadoopCommonDir.canRead())
+ exit("Failed to read Hadoop common dir in '" + hadoopCommonHome + "'.", null);
+
+ File hadoopCommonLibDir = new File(hadoopCommonDir, "lib");
+
+ if (!hadoopCommonLibDir.canRead())
+ exit("Failed to read Hadoop 'lib' folder in '" + hadoopCommonLibDir.getPath() + "'.", null);
+
+ if (U.isWindows()) {
+ checkJavaPathSpaces();
+
+ File hadoopBinDir = new File(hadoopDir, "bin");
+
+ if (!hadoopBinDir.canRead())
+ exit("Failed to read subdirectory 'bin' in HADOOP_HOME.", null);
+
+ File winutilsFile = new File(hadoopBinDir, WINUTILS_EXE);
+
+ if (!winutilsFile.exists()) {
+ if (ask("File '" + WINUTILS_EXE + "' does not exist. " +
+ "It may be replaced by a stub. Create it?")) {
+ println("Creating file stub '" + winutilsFile.getAbsolutePath() + "'.");
+
+ boolean ok = false;
+
+ try {
+ ok = winutilsFile.createNewFile();
+ }
+ catch (IOException ignore) {
+ // No-op.
+ }
+
+ if (!ok)
+ exit("Failed to create '" + WINUTILS_EXE + "' file. Please check permissions.", null);
+ }
+ else
+ println("Ok. But Hadoop client probably will not work on Windows this way...");
+ }
+
+ processCmdFiles(hadoopDir, "bin", "sbin", "libexec");
+ }
+
+ File igniteLibs = new File(new File(igniteHome), "libs");
+
+ if (!igniteLibs.exists())
+ exit("Ignite 'libs' folder is not found.", null);
+
+ Collection<File> jarFiles = new ArrayList<>();
+
+ addJarsInFolder(jarFiles, igniteLibs);
+ addJarsInFolder(jarFiles, new File(igniteLibs, "ignite-hadoop"));
+
+ boolean jarsLinksCorrect = true;
+
+ for (File file : jarFiles) {
+ File link = new File(hadoopCommonLibDir, file.getName());
+
+ jarsLinksCorrect &= isJarLinkCorrect(link, file);
+
+ if (!jarsLinksCorrect)
+ break;
+ }
+
+ if (!jarsLinksCorrect) {
+ if (ask("Ignite JAR files are not found in Hadoop 'lib' directory. " +
+ "Create appropriate symbolic links?")) {
+ File[] oldIgniteJarFiles = hadoopCommonLibDir.listFiles(IGNITE_JARS);
+
+ if (oldIgniteJarFiles.length > 0 && ask("The Hadoop 'lib' directory contains JARs from other Ignite " +
+ "installation. They must be deleted to continue. Continue?")) {
+ for (File file : oldIgniteJarFiles) {
+ println("Deleting file '" + file.getAbsolutePath() + "'.");
+
+ if (!file.delete())
+ exit("Failed to delete file '" + file.getPath() + "'.", null);
+ }
+ }
+
+ for (File file : jarFiles) {
+ File targetFile = new File(hadoopCommonLibDir, file.getName());
+
+ try {
+ println("Creating symbolic link '" + targetFile.getAbsolutePath() + "'.");
+
+ Files.createSymbolicLink(targetFile.toPath(), file.toPath());
+ }
+ catch (IOException e) {
+ if (U.isWindows()) {
+ warn("Ability to create symbolic links is required!");
+ warn("On Windows platform you have to grant permission 'Create symbolic links'");
+ warn("to your user or run the Accelerator as Administrator.");
+ }
+
+ exit("Creating symbolic link failed! Check permissions.", e);
+ }
+ }
+ }
+ else
+ println("Ok. But Hadoop client will not be able to talk to Ignite cluster without those JARs in classpath...");
+ }
+
+ File hadoopEtc = new File(hadoopDir, "etc" + File.separator + "hadoop");
+
+ File igniteDocs = new File(igniteHome, "docs");
+
+ if (!igniteDocs.canRead())
+ exit("Failed to read Ignite 'docs' folder at '" + igniteDocs.getAbsolutePath() + "'.", null);
+
+ if (hadoopEtc.canWrite()) { // TODO Bigtop
+ if (ask("Replace 'core-site.xml' and 'mapred-site.xml' files with preconfigured templates " +
+ "(existing files will be backed up)?")) {
+ replaceWithBackup(new File(igniteDocs, "core-site.ignite.xml"), new File(hadoopEtc, "core-site.xml"));
+
+ replaceWithBackup(new File(igniteDocs, "mapred-site.ignite.xml"), new File(hadoopEtc, "mapred-site.xml"));
+ }
+ else
+ println("Ok. You can configure them later, the templates are available at Ignite's 'docs' directory...");
+ }
+
+ if (!F.isEmpty(hiveHome)) {
+ File hiveConfDir = new File(hiveHome + File.separator + "conf");
+
+ if (!hiveConfDir.canWrite())
+ warn("Can not write to '" + hiveConfDir.getAbsolutePath() + "'. To run Hive queries you have to " +
+ "configure 'hive-site.xml' manually. The template is available at Ignite's 'docs' directory.");
+ else if (ask("Replace 'hive-site.xml' with preconfigured template (existing file will be backed up)?"))
+ replaceWithBackup(new File(igniteDocs, "hive-site.ignite.xml"), new File(hiveConfDir, "hive-site.xml"));
+ else
+ println("Ok. You can configure it later, the template is available at Ignite's 'docs' directory...");
+ }
+
+ println("Apache Hadoop setup is complete.");
+ }
+
+ /**
+ * @param jarFiles Jars.
+ * @param folder Folder.
+ */
+ private static void addJarsInFolder(Collection<File> jarFiles, File folder) {
+ if (!folder.exists())
+ exit("Folder '" + folder.getAbsolutePath() + "' is not found.", null);
+
+ jarFiles.addAll(Arrays.asList(folder.listFiles(IGNITE_JARS)));
+ }
+
+ /**
+ * Checks that JAVA_HOME does not contain space characters.
+ */
+ private static void checkJavaPathSpaces() {
+ String javaHome = System.getProperty("java.home");
+
+ if (javaHome.contains(" ")) {
+ warn("Java installation path contains space characters!");
+ warn("Hadoop client will not be able to start using '" + javaHome + "'.");
+ warn("Please install JRE to path which does not contain spaces and point JAVA_HOME to that installation.");
+ }
+ }
+
+ /**
+ * Checks Ignite home.
+ *
+ * @param igniteHome Ignite home.
+ */
+ private static void checkIgniteHome(String igniteHome) {
+ URL jarUrl = U.class.getProtectionDomain().getCodeSource().getLocation();
+
+ try {
+ Path jar = Paths.get(jarUrl.toURI());
+ Path igHome = Paths.get(igniteHome);
+
+ if (!jar.startsWith(igHome))
+ exit("Ignite JAR files are not under IGNITE_HOME.", null);
+ }
+ catch (Exception e) {
+ exit(e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Replaces target file with source file.
+ *
+ * @param from From.
+ * @param to To.
+ */
+ private static void replaceWithBackup(File from, File to) {
+ if (!from.canRead())
+ exit("Failed to read source file '" + from.getAbsolutePath() + "'.", null);
+
+ println("Replacing file '" + to.getAbsolutePath() + "'.");
+
+ try {
+ U.copy(from, renameToBak(to), true);
+ }
+ catch (IOException e) {
+ exit("Failed to replace file '" + to.getAbsolutePath() + "'.", e);
+ }
+ }
+
+ /**
+ * Renames file for backup.
+ *
+ * @param file File.
+ * @return File.
+ */
+ private static File renameToBak(File file) {
+ DateFormat fmt = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
+
+ if (file.exists() && !file.renameTo(new File(file.getAbsolutePath() + "." + fmt.format(new Date()) + ".bak")))
+ exit("Failed to rename file '" + file.getPath() + "'.", null);
+
+ return file;
+ }
+
+ /**
+ * Checks if link is correct.
+ *
+ * @param link Symbolic link.
+ * @param correctTarget Correct link target.
+ * @return {@code true} If link target is correct.
+ */
+ private static boolean isJarLinkCorrect(File link, File correctTarget) {
+ if (!Files.isSymbolicLink(link.toPath()))
+ return false; // It is a real file or it does not exist.
+
+ Path target = null;
+
+ try {
+ target = Files.readSymbolicLink(link.toPath());
+ }
+ catch (IOException e) {
+ exit("Failed to read symbolic link: " + link.getAbsolutePath(), e);
+ }
+
+ return Files.exists(target) && target.toFile().equals(correctTarget);
+ }
+
+ /**
+ * Writes the question end read the boolean answer from the console.
+ *
+ * @param question Question to write.
+ * @return {@code true} if user inputs 'Y' or 'y', {@code false} otherwise.
+ */
+ private static boolean ask(String question) {
+ X.println();
+ X.print(" < " + question + " (Y/N): ");
+
+ String answer = null;
+
+ if (!F.isEmpty(System.getenv("IGNITE_HADOOP_SETUP_YES")))
+ answer = "Y";
+ else {
+ BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+
+ try {
+ answer = br.readLine();
+ }
+ catch (IOException e) {
+ exit("Failed to read answer: " + e.getMessage(), e);
+ }
+ }
+
+ if (answer != null && "Y".equals(answer.toUpperCase().trim())) {
+ X.println(" > Yes.");
+
+ return true;
+ }
+ else {
+ X.println(" > No.");
+
+ return false;
+ }
+ }
+
+ /**
+ * Exit with message.
+ *
+ * @param msg Exit message.
+ */
+ private static void exit(String msg, Exception e) {
+ X.println(" ");
+ X.println(" # " + msg);
+ X.println(" # Setup failed, exiting... ");
+
+ if (e != null && !F.isEmpty(System.getenv("IGNITE_HADOOP_SETUP_DEBUG")))
+ e.printStackTrace();
+
+ System.exit(1);
+ }
+
+ /**
+ * Prints message.
+ *
+ * @param msg Message.
+ */
+ private static void println(String msg) {
+ X.println(" > " + msg);
+ }
+
+ /**
+ * Prints warning.
+ *
+ * @param msg Message.
+ */
+ private static void warn(String msg) {
+ X.println(" ! " + msg);
+ }
+
+ /**
+ * Checks that CMD files have valid MS Windows new line characters. If not, writes question to console and reads the
+ * answer. If it's 'Y' then backups original files and corrects invalid new line characters.
+ *
+ * @param rootDir Root directory to process.
+ * @param dirs Directories inside of the root to process.
+ */
+ private static void processCmdFiles(File rootDir, String... dirs) {
+ boolean answer = false;
+
+ for (String dir : dirs) {
+ File subDir = new File(rootDir, dir);
+
+ File[] cmdFiles = subDir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return name.toLowerCase().endsWith(".cmd");
+ }
+ });
+
+ for (File file : cmdFiles) {
+ String content = null;
+
+ try (Scanner scanner = new Scanner(file)) {
+ content = scanner.useDelimiter("\\Z").next();
+ }
+ catch (FileNotFoundException e) {
+ exit("Failed to read file '" + file + "'.", e);
+ }
+
+ boolean invalid = false;
+
+ for (int i = 0; i < content.length(); i++) {
+ if (content.charAt(i) == '\n' && (i == 0 || content.charAt(i - 1) != '\r')) {
+ invalid = true;
+
+ break;
+ }
+ }
+
+ if (invalid) {
+ answer = answer || ask("One or more *.CMD files has invalid new line character. Replace them?");
+
+ if (!answer) {
+ println("Ok. But Windows most probably will fail to execute them...");
+
+ return;
+ }
+
+ println("Fixing newline characters in file '" + file.getAbsolutePath() + "'.");
+
+ renameToBak(file);
+
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) {
+ for (int i = 0; i < content.length(); i++) {
+ if (content.charAt(i) == '\n' && (i == 0 || content.charAt(i - 1) != '\r'))
+ writer.write("\r");
+
+ writer.write(content.charAt(i));
+ }
+ }
+ catch (IOException e) {
+ exit("Failed to write file '" + file.getPath() + "': " + e.getMessage(), e);
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskCancelledException.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskCancelledException.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskCancelledException.java
new file mode 100644
index 0000000..bb3d1cc
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskCancelledException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+
+/**
+ * Exception that throws when the task is cancelling.
+ */
+public class HadoopTaskCancelledException extends IgniteException {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * @param msg Exception message.
+ */
+ public HadoopTaskCancelledException(String msg) {
+ super(msg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
new file mode 100644
index 0000000..00be422
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.JobPriority;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Hadoop utility methods.
+ */
+public class HadoopUtils {
+ /** Property to store timestamp of new job id request. */
+ public static final String REQ_NEW_JOBID_TS_PROPERTY = "ignite.job.requestNewIdTs";
+
+ /** Property to store timestamp of response of new job id request. */
+ public static final String RESPONSE_NEW_JOBID_TS_PROPERTY = "ignite.job.responseNewIdTs";
+
+ /** Property to store timestamp of job submission. */
+ public static final String JOB_SUBMISSION_START_TS_PROPERTY = "ignite.job.submissionStartTs";
+
+ /** Property to set custom writer of job statistics. */
+ public static final String JOB_COUNTER_WRITER_PROPERTY = "ignite.counters.writer";
+
+ /** Staging constant. */
+ private static final String STAGING_CONSTANT = ".staging";
+
+ /** Old mapper class attribute. */
+ private static final String OLD_MAP_CLASS_ATTR = "mapred.mapper.class";
+
+ /** Old reducer class attribute. */
+ private static final String OLD_REDUCE_CLASS_ATTR = "mapred.reducer.class";
+
+ /**
+ * Wraps native split.
+ *
+ * @param id Split ID.
+ * @param split Split.
+ * @param hosts Hosts.
+ * @throws IOException If failed.
+ */
+ public static HadoopSplitWrapper wrapSplit(int id, Object split, String[] hosts) throws IOException {
+ ByteArrayOutputStream arr = new ByteArrayOutputStream();
+ ObjectOutput out = new ObjectOutputStream(arr);
+
+ assert split instanceof Writable;
+
+ ((Writable)split).write(out);
+
+ out.flush();
+
+ return new HadoopSplitWrapper(id, split.getClass().getName(), arr.toByteArray(), hosts);
+ }
+
+ /**
+ * Unwraps native split.
+ *
+ * @param o Wrapper.
+ * @return Split.
+ */
+ public static Object unwrapSplit(HadoopSplitWrapper o) {
+ try {
+ Writable w = (Writable)HadoopUtils.class.getClassLoader().loadClass(o.className()).newInstance();
+
+ w.readFields(new ObjectInputStream(new ByteArrayInputStream(o.bytes())));
+
+ return w;
+ }
+ catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Convert Ignite job status to Hadoop job status.
+ *
+ * @param status Ignite job status.
+ * @return Hadoop job status.
+ */
+ public static JobStatus status(HadoopJobStatus status, Configuration conf) {
+ JobID jobId = new JobID(status.jobId().globalId().toString(), status.jobId().localId());
+
+ float setupProgress = 0;
+ float mapProgress = 0;
+ float reduceProgress = 0;
+ float cleanupProgress = 0;
+
+ JobStatus.State state = JobStatus.State.RUNNING;
+
+ switch (status.jobPhase()) {
+ case PHASE_SETUP:
+ setupProgress = 0.42f;
+
+ break;
+
+ case PHASE_MAP:
+ setupProgress = 1;
+ mapProgress = 1f - status.pendingMapperCnt() / (float)status.totalMapperCnt();
+
+ break;
+
+ case PHASE_REDUCE:
+ assert status.totalReducerCnt() > 0;
+
+ setupProgress = 1;
+ mapProgress = 1;
+ reduceProgress = 1f - status.pendingReducerCnt() / (float)status.totalReducerCnt();
+
+ break;
+
+ case PHASE_CANCELLING:
+ case PHASE_COMPLETE:
+ if (!status.isFailed()) {
+ setupProgress = 1;
+ mapProgress = 1;
+ reduceProgress = 1;
+ cleanupProgress = 1;
+
+ state = JobStatus.State.SUCCEEDED;
+ }
+ else
+ state = JobStatus.State.FAILED;
+
+ break;
+
+ default:
+ assert false;
+ }
+
+ return new JobStatus(jobId, setupProgress, mapProgress, reduceProgress, cleanupProgress, state,
+ JobPriority.NORMAL, status.user(), status.jobName(), jobFile(conf, status.user(), jobId).toString(), "N/A");
+ }
+
+ /**
+ * Gets staging area directory.
+ *
+ * @param conf Configuration.
+ * @param usr User.
+ * @return Staging area directory.
+ */
+ public static Path stagingAreaDir(Configuration conf, String usr) {
+ return new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR, MRJobConfig.DEFAULT_MR_AM_STAGING_DIR)
+ + Path.SEPARATOR + usr + Path.SEPARATOR + STAGING_CONSTANT);
+ }
+
+ /**
+ * Gets job file.
+ *
+ * @param conf Configuration.
+ * @param usr User.
+ * @param jobId Job ID.
+ * @return Job file.
+ */
+ public static Path jobFile(Configuration conf, String usr, JobID jobId) {
+ return new Path(stagingAreaDir(conf, usr), jobId.toString() + Path.SEPARATOR + MRJobConfig.JOB_CONF_FILE);
+ }
+
+ /**
+ * Checks the attribute in configuration is not set.
+ *
+ * @param attr Attribute name.
+ * @param msg Message for creation of exception.
+ * @throws IgniteCheckedException If attribute is set.
+ */
+ public static void ensureNotSet(Configuration cfg, String attr, String msg) throws IgniteCheckedException {
+ if (cfg.get(attr) != null)
+ throw new IgniteCheckedException(attr + " is incompatible with " + msg + " mode.");
+ }
+
+ /**
+ * Creates JobInfo from hadoop configuration.
+ *
+ * @param cfg Hadoop configuration.
+ * @return Job info.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static HadoopDefaultJobInfo createJobInfo(Configuration cfg) throws IgniteCheckedException {
+ JobConf jobConf = new JobConf(cfg);
+
+ boolean hasCombiner = jobConf.get("mapred.combiner.class") != null
+ || jobConf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null;
+
+ int numReduces = jobConf.getNumReduceTasks();
+
+ jobConf.setBooleanIfUnset("mapred.mapper.new-api", jobConf.get(OLD_MAP_CLASS_ATTR) == null);
+
+ if (jobConf.getUseNewMapper()) {
+ String mode = "new map API";
+
+ ensureNotSet(jobConf, "mapred.input.format.class", mode);
+ ensureNotSet(jobConf, OLD_MAP_CLASS_ATTR, mode);
+
+ if (numReduces != 0)
+ ensureNotSet(jobConf, "mapred.partitioner.class", mode);
+ else
+ ensureNotSet(jobConf, "mapred.output.format.class", mode);
+ }
+ else {
+ String mode = "map compatibility";
+
+ ensureNotSet(jobConf, MRJobConfig.INPUT_FORMAT_CLASS_ATTR, mode);
+ ensureNotSet(jobConf, MRJobConfig.MAP_CLASS_ATTR, mode);
+
+ if (numReduces != 0)
+ ensureNotSet(jobConf, MRJobConfig.PARTITIONER_CLASS_ATTR, mode);
+ else
+ ensureNotSet(jobConf, MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, mode);
+ }
+
+ if (numReduces != 0) {
+ jobConf.setBooleanIfUnset("mapred.reducer.new-api", jobConf.get(OLD_REDUCE_CLASS_ATTR) == null);
+
+ if (jobConf.getUseNewReducer()) {
+ String mode = "new reduce API";
+
+ ensureNotSet(jobConf, "mapred.output.format.class", mode);
+ ensureNotSet(jobConf, OLD_REDUCE_CLASS_ATTR, mode);
+ }
+ else {
+ String mode = "reduce compatibility";
+
+ ensureNotSet(jobConf, MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, mode);
+ ensureNotSet(jobConf, MRJobConfig.REDUCE_CLASS_ATTR, mode);
+ }
+ }
+
+ Map<String, String> props = new HashMap<>();
+
+ for (Map.Entry<String, String> entry : jobConf)
+ props.put(entry.getKey(), entry.getValue());
+
+ return new HadoopDefaultJobInfo(jobConf.getJobName(), jobConf.getUser(), hasCombiner, numReduces, props);
+ }
+
+ /**
+ * Throws new {@link IgniteCheckedException} with original exception is serialized into string.
+ * This is needed to transfer error outside the current class loader.
+ *
+ * @param e Original exception.
+ * @return IgniteCheckedException New exception.
+ */
+ public static IgniteCheckedException transformException(Throwable e) {
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+
+ e.printStackTrace(new PrintStream(os, true));
+
+ return new IgniteCheckedException(os.toString());
+ }
+
+ /**
+ * Returns work directory for job execution.
+ *
+ * @param locNodeId Local node ID.
+ * @param jobId Job ID.
+ * @return Working directory for job.
+ * @throws IgniteCheckedException If Failed.
+ */
+ public static File jobLocalDir(UUID locNodeId, HadoopJobId jobId) throws IgniteCheckedException {
+ return new File(new File(U.resolveWorkDirectory("hadoop", false), "node-" + locNodeId), "job_" + jobId);
+ }
+
+ /**
+ * Returns subdirectory of job working directory for task execution.
+ *
+ * @param locNodeId Local node ID.
+ * @param info Task info.
+ * @return Working directory for task.
+ * @throws IgniteCheckedException If Failed.
+ */
+ public static File taskLocalDir(UUID locNodeId, HadoopTaskInfo info) throws IgniteCheckedException {
+ File jobLocDir = jobLocalDir(locNodeId, info.jobId());
+
+ return new File(jobLocDir, info.type() + "_" + info.taskNumber() + "_" + info.attempt());
+ }
+
+ /**
+ * Constructor.
+ */
+ private HadoopUtils() {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessor.java
deleted file mode 100644
index 4ef9e35..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessor.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
-import org.apache.ignite.internal.processors.hadoop.planner.*;
-import org.apache.ignite.internal.processors.hadoop.shuffle.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
-import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
-import org.apache.ignite.internal.util.tostring.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.util.*;
-import java.util.concurrent.atomic.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopClassLoader.*;
-
-/**
- * Hadoop processor.
- */
-public class IgniteHadoopProcessor extends IgniteHadoopProcessorAdapter {
- /** Job ID counter. */
- private final AtomicInteger idCtr = new AtomicInteger();
-
- /** Hadoop context. */
- @GridToStringExclude
- private GridHadoopContext hctx;
-
- /** Hadoop facade for public API. */
- @GridToStringExclude
- private GridHadoop hadoop;
-
- /**
- * @param ctx Kernal context.
- */
- public IgniteHadoopProcessor(GridKernalContext ctx) {
- super(ctx);
- }
-
- /** {@inheritDoc} */
- @Override public void start() throws IgniteCheckedException {
- if (ctx.isDaemon())
- return;
-
- GridHadoopConfiguration cfg = ctx.config().getHadoopConfiguration();
-
- if (cfg == null)
- cfg = new GridHadoopConfiguration();
- else
- cfg = new GridHadoopConfiguration(cfg);
-
- initializeDefaults(cfg);
-
- validate(cfg);
-
- if (hadoopHome() != null)
- U.quietAndInfo(log, "HADOOP_HOME is set to " + hadoopHome());
-
- boolean ok = false;
-
- try { // Check for Hadoop installation.
- hadoopUrls();
-
- ok = true;
- }
- catch (IgniteCheckedException e) {
- U.quietAndWarn(log, e.getMessage());
- }
-
- if (ok) {
- hctx = new GridHadoopContext(
- ctx,
- cfg,
- new GridHadoopJobTracker(),
- cfg.isExternalExecution() ? new GridHadoopExternalTaskExecutor() : new GridHadoopEmbeddedTaskExecutor(),
- new GridHadoopShuffle());
-
-
- for (GridHadoopComponent c : hctx.components())
- c.start(hctx);
-
- hadoop = new GridHadoopImpl(this);
- }
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(IgniteHadoopProcessor.class, this);
- }
-
- /** {@inheritDoc} */
- @Override public void stop(boolean cancel) throws IgniteCheckedException {
- super.stop(cancel);
-
- if (hctx == null)
- return;
-
- List<GridHadoopComponent> components = hctx.components();
-
- for (ListIterator<GridHadoopComponent> it = components.listIterator(components.size()); it.hasPrevious();) {
- GridHadoopComponent c = it.previous();
-
- c.stop(cancel);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void onKernalStart() throws IgniteCheckedException {
- super.onKernalStart();
-
- if (hctx == null)
- return;
-
- for (GridHadoopComponent c : hctx.components())
- c.onKernalStart();
- }
-
- /** {@inheritDoc} */
- @Override public void onKernalStop(boolean cancel) {
- super.onKernalStop(cancel);
-
- if (hctx == null)
- return;
-
- List<GridHadoopComponent> components = hctx.components();
-
- for (ListIterator<GridHadoopComponent> it = components.listIterator(components.size()); it.hasPrevious();) {
- GridHadoopComponent c = it.previous();
-
- c.onKernalStop(cancel);
- }
- }
-
- /**
- * Gets Hadoop context.
- *
- * @return Hadoop context.
- */
- public GridHadoopContext context() {
- return hctx;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoop hadoop() {
- if (hadoop == null)
- throw new IllegalStateException("Hadoop accelerator is disabled (Hadoop is not in classpath, " +
- "is HADOOP_HOME environment variable set?)");
-
- return hadoop;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopConfiguration config() {
- return hctx.configuration();
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopJobId nextJobId() {
- return new GridHadoopJobId(ctx.localNodeId(), idCtr.incrementAndGet());
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<?> submit(GridHadoopJobId jobId, GridHadoopJobInfo jobInfo) {
- return hctx.jobTracker().submit(jobId, jobInfo);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopJobStatus status(GridHadoopJobId jobId) throws IgniteCheckedException {
- return hctx.jobTracker().status(jobId);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopCounters counters(GridHadoopJobId jobId) throws IgniteCheckedException {
- return hctx.jobTracker().jobCounters(jobId);
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<?> finishFuture(GridHadoopJobId jobId) throws IgniteCheckedException {
- return hctx.jobTracker().finishFuture(jobId);
- }
-
- /** {@inheritDoc} */
- @Override public boolean kill(GridHadoopJobId jobId) throws IgniteCheckedException {
- return hctx.jobTracker().killJob(jobId);
- }
-
- /**
- * Initializes default hadoop configuration.
- *
- * @param cfg Hadoop configuration.
- */
- private void initializeDefaults(GridHadoopConfiguration cfg) {
- if (cfg.getMapReducePlanner() == null)
- cfg.setMapReducePlanner(new GridHadoopDefaultMapReducePlanner());
- }
-
- /**
- * Validates Grid and Hadoop configuration for correctness.
- *
- * @param hadoopCfg Hadoop configuration.
- * @throws IgniteCheckedException If failed.
- */
- private void validate(GridHadoopConfiguration hadoopCfg) throws IgniteCheckedException {
- if (ctx.config().isPeerClassLoadingEnabled())
- throw new IgniteCheckedException("Peer class loading cannot be used with Hadoop (disable it using " +
- "GridConfiguration.setPeerClassLoadingEnabled()).");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCounterAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCounterAdapter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCounterAdapter.java
deleted file mode 100644
index 9e46846..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCounterAdapter.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.counter;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * Default Hadoop counter implementation.
- */
-public abstract class GridHadoopCounterAdapter implements GridHadoopCounter, Externalizable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Counter group name. */
- private String grp;
-
- /** Counter name. */
- private String name;
-
- /**
- * Default constructor required by {@link Externalizable}.
- */
- protected GridHadoopCounterAdapter() {
- // No-op.
- }
-
- /**
- * Creates new counter with given group and name.
- *
- * @param grp Counter group name.
- * @param name Counter name.
- */
- protected GridHadoopCounterAdapter(String grp, String name) {
- assert grp != null : "counter must have group";
- assert name != null : "counter must have name";
-
- this.grp = grp;
- this.name = name;
- }
-
- /** {@inheritDoc} */
- @Override public String name() {
- return name;
- }
-
- /** {@inheritDoc} */
- @Override @Nullable public String group() {
- return grp;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeUTF(grp);
- out.writeUTF(name);
- writeValue(out);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- grp = in.readUTF();
- name = in.readUTF();
- readValue(in);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- GridHadoopCounterAdapter cntr = (GridHadoopCounterAdapter)o;
-
- if (!grp.equals(cntr.grp))
- return false;
- if (!name.equals(cntr.name))
- return false;
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = grp.hashCode();
- res = 31 * res + name.hashCode();
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopCounterAdapter.class, this);
- }
-
- /**
- * Writes value of this counter to output.
- *
- * @param out Output.
- * @throws IOException If failed.
- */
- protected abstract void writeValue(ObjectOutput out) throws IOException;
-
- /**
- * Read value of this counter from input.
- *
- * @param in Input.
- * @throws IOException If failed.
- */
- protected abstract void readValue(ObjectInput in) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCountersImpl.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCountersImpl.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCountersImpl.java
deleted file mode 100644
index 92d54af..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCountersImpl.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.counter;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jdk8.backport.*;
-
-import java.io.*;
-import java.lang.reflect.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-/**
- * Default in-memory counters store.
- */
-public class GridHadoopCountersImpl implements GridHadoopCounters, Externalizable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private final ConcurrentMap<CounterKey, GridHadoopCounter> cntrsMap = new ConcurrentHashMap8<>();
-
- /**
- * Default constructor. Creates new instance without counters.
- */
- public GridHadoopCountersImpl() {
- // No-op.
- }
-
- /**
- * Creates new instance that contain given counters.
- *
- * @param cntrs Counters to store.
- */
- public GridHadoopCountersImpl(Iterable<GridHadoopCounter> cntrs) {
- addCounters(cntrs, true);
- }
-
- /**
- * Copy constructor.
- *
- * @param cntrs Counters to copy.
- */
- public GridHadoopCountersImpl(GridHadoopCounters cntrs) {
- this(cntrs.all());
- }
-
- /**
- * Creates counter instance.
- *
- * @param cls Class of the counter.
- * @param grp Group name.
- * @param name Counter name.
- * @return Counter.
- */
- private <T extends GridHadoopCounter> T createCounter(Class<? extends GridHadoopCounter> cls, String grp,
- String name) {
- try {
- Constructor constructor = cls.getConstructor(String.class, String.class);
-
- return (T)constructor.newInstance(grp, name);
- }
- catch (Exception e) {
- throw new IgniteException(e);
- }
- }
-
- /**
- * Adds counters collection in addition to existing counters.
- *
- * @param cntrs Counters to add.
- * @param cp Whether to copy counters or not.
- */
- private void addCounters(Iterable<GridHadoopCounter> cntrs, boolean cp) {
- assert cntrs != null;
-
- for (GridHadoopCounter cntr : cntrs) {
- if (cp) {
- GridHadoopCounter cntrCp = createCounter(cntr.getClass(), cntr.group(), cntr.name());
-
- cntrCp.merge(cntr);
-
- cntr = cntrCp;
- }
-
- cntrsMap.put(new CounterKey(cntr.getClass(), cntr.group(), cntr.name()), cntr);
- }
- }
-
- /** {@inheritDoc} */
- @Override public <T extends GridHadoopCounter> T counter(String grp, String name, Class<T> cls) {
- assert cls != null;
-
- CounterKey mapKey = new CounterKey(cls, grp, name);
-
- T cntr = (T)cntrsMap.get(mapKey);
-
- if (cntr == null) {
- cntr = createCounter(cls, grp, name);
-
- T old = (T)cntrsMap.putIfAbsent(mapKey, cntr);
-
- if (old != null)
- return old;
- }
-
- return cntr;
- }
-
- /** {@inheritDoc} */
- @Override public Collection<GridHadoopCounter> all() {
- return cntrsMap.values();
- }
-
- /** {@inheritDoc} */
- @Override public void merge(GridHadoopCounters other) {
- for (GridHadoopCounter counter : other.all())
- counter(counter.group(), counter.name(), counter.getClass()).merge(counter);
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- U.writeCollection(out, cntrsMap.values());
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- addCounters(U.<GridHadoopCounter>readCollection(in), false);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- GridHadoopCountersImpl counters = (GridHadoopCountersImpl)o;
-
- return cntrsMap.equals(counters.cntrsMap);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return cntrsMap.hashCode();
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopCountersImpl.class, this, "counters", cntrsMap.values());
- }
-
- /**
- * The tuple of counter identifier components for more readable code.
- */
- private static class CounterKey extends GridTuple3<Class<? extends GridHadoopCounter>, String, String> {
- /** */
- private static final long serialVersionUID = 0L;
-
- /**
- * Constructor.
- *
- * @param cls Class of the counter.
- * @param grp Group name.
- * @param name Counter name.
- */
- private CounterKey(Class<? extends GridHadoopCounter> cls, String grp, String name) {
- super(cls, grp, name);
- }
-
- /**
- * Empty constructor required by {@link Externalizable}.
- */
- public CounterKey() {
- // No-op.
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopFSCounterWriter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopFSCounterWriter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopFSCounterWriter.java
deleted file mode 100644
index 55dcc4c..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopFSCounterWriter.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.counter;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.typedef.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Statistic writer implementation that writes info into any Hadoop file system.
- */
-public class GridHadoopFSCounterWriter implements GridHadoopCounterWriter {
- /** */
- public static final String PERFORMANCE_COUNTER_FILE_NAME = "performance";
-
- /** */
- private static final String DEFAULT_USER_NAME = "anonymous";
-
- /** */
- public static final String COUNTER_WRITER_DIR_PROPERTY = "ignite.counters.fswriter.directory";
-
- /** */
- private static final String USER_MACRO = "${USER}";
-
- /** */
- private static final String DEFAULT_COUNTER_WRITER_DIR = "/user/" + USER_MACRO;
-
- /** {@inheritDoc} */
- @Override public void write(GridHadoopJobInfo jobInfo, GridHadoopJobId jobId, GridHadoopCounters cntrs)
- throws IgniteCheckedException {
-
- Configuration hadoopCfg = new Configuration();
-
- for (Map.Entry<String, String> e : ((GridHadoopDefaultJobInfo)jobInfo).properties().entrySet())
- hadoopCfg.set(e.getKey(), e.getValue());
-
- String user = jobInfo.user();
-
- if (F.isEmpty(user))
- user = DEFAULT_USER_NAME;
-
- String dir = jobInfo.property(COUNTER_WRITER_DIR_PROPERTY);
-
- if (dir == null)
- dir = DEFAULT_COUNTER_WRITER_DIR;
-
- Path jobStatPath = new Path(new Path(dir.replace(USER_MACRO, user)), jobId.toString());
-
- GridHadoopPerformanceCounter perfCntr = GridHadoopPerformanceCounter.getCounter(cntrs, null);
-
- try {
- FileSystem fs = jobStatPath.getFileSystem(hadoopCfg);
-
- fs.mkdirs(jobStatPath);
-
- try (PrintStream out = new PrintStream(fs.create(new Path(jobStatPath, PERFORMANCE_COUNTER_FILE_NAME)))) {
- for (T2<String, Long> evt : perfCntr.evts()) {
- out.print(evt.get1());
- out.print(':');
- out.println(evt.get2().toString());
- }
-
- out.flush();
- }
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopLongCounter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopLongCounter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopLongCounter.java
deleted file mode 100644
index 67af49f..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopLongCounter.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.counter;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-
-import java.io.*;
-
-/**
- * Standard hadoop counter to use via original Hadoop API in Hadoop jobs.
- */
-public class GridHadoopLongCounter extends GridHadoopCounterAdapter {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** The counter value. */
- private long val;
-
- /**
- * Default constructor required by {@link Externalizable}.
- */
- public GridHadoopLongCounter() {
- // No-op.
- }
-
- /**
- * Constructor.
- *
- * @param grp Group name.
- * @param name Counter name.
- */
- public GridHadoopLongCounter(String grp, String name) {
- super(grp, name);
- }
-
- /** {@inheritDoc} */
- @Override protected void writeValue(ObjectOutput out) throws IOException {
- out.writeLong(val);
- }
-
- /** {@inheritDoc} */
- @Override protected void readValue(ObjectInput in) throws IOException {
- val = in.readLong();
- }
-
- /** {@inheritDoc} */
- @Override public void merge(GridHadoopCounter cntr) {
- val += ((GridHadoopLongCounter)cntr).val;
- }
-
- /**
- * Gets current value of this counter.
- *
- * @return Current value.
- */
- public long value() {
- return val;
- }
-
- /**
- * Sets current value by the given value.
- *
- * @param val Value to set.
- */
- public void value(long val) {
- this.val = val;
- }
-
- /**
- * Increment this counter by the given value.
- *
- * @param i Value to increase this counter by.
- */
- public void increment(long i) {
- val += i;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopPerformanceCounter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopPerformanceCounter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopPerformanceCounter.java
deleted file mode 100644
index d5ceebf..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopPerformanceCounter.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.counter;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Counter for the job statistics accumulation.
- */
-public class GridHadoopPerformanceCounter extends GridHadoopCounterAdapter {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** The group name for this counter. */
- private static final String GROUP_NAME = "SYSTEM";
-
- /** The counter name for this counter. */
- private static final String COUNTER_NAME = "PERFORMANCE";
-
- /** Events collections. */
- private Collection<T2<String,Long>> evts = new ArrayList<>();
-
- /** Node id to insert into the event info. */
- private UUID nodeId;
-
- /** */
- private int reducerNum;
-
- /** */
- private volatile Long firstShuffleMsg;
-
- /** */
- private volatile Long lastShuffleMsg;
-
- /**
- * Default constructor required by {@link Externalizable}.
- */
- public GridHadoopPerformanceCounter() {
- // No-op.
- }
-
- /**
- * Constructor.
- *
- * @param grp Group name.
- * @param name Counter name.
- */
- public GridHadoopPerformanceCounter(String grp, String name) {
- super(grp, name);
- }
-
- /**
- * Constructor to create instance to use this as helper.
- *
- * @param nodeId Id of the work node.
- */
- public GridHadoopPerformanceCounter(UUID nodeId) {
- this.nodeId = nodeId;
- }
-
- /** {@inheritDoc} */
- @Override protected void writeValue(ObjectOutput out) throws IOException {
- U.writeCollection(out, evts);
- }
-
- /** {@inheritDoc} */
- @Override protected void readValue(ObjectInput in) throws IOException {
- try {
- evts = U.readCollection(in);
- }
- catch (ClassNotFoundException e) {
- throw new IOException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void merge(GridHadoopCounter cntr) {
- evts.addAll(((GridHadoopPerformanceCounter)cntr).evts);
- }
-
- /**
- * Gets the events collection.
- *
- * @return Collection of event.
- */
- public Collection<T2<String, Long>> evts() {
- return evts;
- }
-
- /**
- * Generate name that consists of some event information.
- *
- * @param info Task info.
- * @param evtType The type of the event.
- * @return String contains necessary event information.
- */
- private String eventName(GridHadoopTaskInfo info, String evtType) {
- return eventName(info.type().toString(), info.taskNumber(), evtType);
- }
-
- /**
- * Generate name that consists of some event information.
- *
- * @param taskType Task type.
- * @param taskNum Number of the task.
- * @param evtType The type of the event.
- * @return String contains necessary event information.
- */
- private String eventName(String taskType, int taskNum, String evtType) {
- assert nodeId != null;
-
- return taskType + " " + taskNum + " " + evtType + " " + nodeId;
- }
-
- /**
- * Adds event of the task submission (task instance creation).
- *
- * @param info Task info.
- * @param ts Timestamp of the event.
- */
- public void onTaskSubmit(GridHadoopTaskInfo info, long ts) {
- evts.add(new T2<>(eventName(info, "submit"), ts));
- }
-
- /**
- * Adds event of the task preparation.
- *
- * @param info Task info.
- * @param ts Timestamp of the event.
- */
- public void onTaskPrepare(GridHadoopTaskInfo info, long ts) {
- evts.add(new T2<>(eventName(info, "prepare"), ts));
- }
-
- /**
- * Adds event of the task finish.
- *
- * @param info Task info.
- * @param ts Timestamp of the event.
- */
- public void onTaskFinish(GridHadoopTaskInfo info, long ts) {
- if (info.type() == GridHadoopTaskType.REDUCE && lastShuffleMsg != null) {
- evts.add(new T2<>(eventName("SHUFFLE", reducerNum, "start"), firstShuffleMsg));
- evts.add(new T2<>(eventName("SHUFFLE", reducerNum, "finish"), lastShuffleMsg));
-
- lastShuffleMsg = null;
- }
-
- evts.add(new T2<>(eventName(info, "finish"), ts));
- }
-
- /**
- * Adds event of the task run.
- *
- * @param info Task info.
- * @param ts Timestamp of the event.
- */
- public void onTaskStart(GridHadoopTaskInfo info, long ts) {
- evts.add(new T2<>(eventName(info, "start"), ts));
- }
-
- /**
- * Adds event of the job preparation.
- *
- * @param ts Timestamp of the event.
- */
- public void onJobPrepare(long ts) {
- assert nodeId != null;
-
- evts.add(new T2<>("JOB prepare " + nodeId, ts));
- }
-
- /**
- * Adds event of the job start.
- *
- * @param ts Timestamp of the event.
- */
- public void onJobStart(long ts) {
- assert nodeId != null;
-
- evts.add(new T2<>("JOB start " + nodeId, ts));
- }
-
- /**
- * Adds client submission events from job info.
- *
- * @param info Job info.
- */
- public void clientSubmissionEvents(GridHadoopJobInfo info) {
- assert nodeId != null;
-
- addEventFromProperty("JOB requestId", info, REQ_NEW_JOBID_TS_PROPERTY);
- addEventFromProperty("JOB responseId", info, RESPONSE_NEW_JOBID_TS_PROPERTY);
- addEventFromProperty("JOB submit", info, JOB_SUBMISSION_START_TS_PROPERTY);
- }
-
- /**
- * Adds event with timestamp from some property in job info.
- *
- * @param evt Event type and phase.
- * @param info Job info.
- * @param propName Property name to get timestamp.
- */
- private void addEventFromProperty(String evt, GridHadoopJobInfo info, String propName) {
- String val = info.property(propName);
-
- if (!F.isEmpty(val)) {
- try {
- evts.add(new T2<>(evt + " " + nodeId, Long.parseLong(val)));
- }
- catch (NumberFormatException e) {
- throw new IllegalStateException("Invalid value '" + val + "' of property '" + propName + "'", e);
- }
- }
- }
-
- /**
- * Registers shuffle message event.
- *
- * @param reducerNum Number of reducer that receives the data.
- * @param ts Timestamp of the event.
- */
- public void onShuffleMessage(int reducerNum, long ts) {
- this.reducerNum = reducerNum;
-
- if (firstShuffleMsg == null)
- firstShuffleMsg = ts;
-
- lastShuffleMsg = ts;
- }
-
- /**
- * Gets system predefined performance counter from the GridHadoopCounters object.
- *
- * @param cntrs GridHadoopCounters object.
- * @param nodeId Node id for methods that adds events. It may be null if you don't use ones.
- * @return Predefined performance counter.
- */
- public static GridHadoopPerformanceCounter getCounter(GridHadoopCounters cntrs, @Nullable UUID nodeId) {
- GridHadoopPerformanceCounter cntr = cntrs.counter(GROUP_NAME, COUNTER_NAME, GridHadoopPerformanceCounter.class);
-
- if (nodeId != null)
- cntr.nodeId(nodeId);
-
- return cntrs.counter(GROUP_NAME, COUNTER_NAME, GridHadoopPerformanceCounter.class);
- }
-
- /**
- * Sets the nodeId field.
- *
- * @param nodeId Node id.
- */
- private void nodeId(UUID nodeId) {
- this.nodeId = nodeId;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounterAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounterAdapter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounterAdapter.java
new file mode 100644
index 0000000..c2ed5bb
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCounterAdapter.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.counter;
+
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * Default Hadoop counter implementation.
+ */
+public abstract class HadoopCounterAdapter implements HadoopCounter, Externalizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Counter group name. */
+ private String grp;
+
+ /** Counter name. */
+ private String name;
+
+ /**
+ * Default constructor required by {@link Externalizable}.
+ */
+ protected HadoopCounterAdapter() {
+ // No-op.
+ }
+
+ /**
+ * Creates new counter with given group and name.
+ *
+ * @param grp Counter group name.
+ * @param name Counter name.
+ */
+ protected HadoopCounterAdapter(String grp, String name) {
+ assert grp != null : "counter must have group";
+ assert name != null : "counter must have name";
+
+ this.grp = grp;
+ this.name = name;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String name() {
+ return name;
+ }
+
+ /** {@inheritDoc} */
+ @Override @Nullable public String group() {
+ return grp;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeUTF(grp);
+ out.writeUTF(name);
+ writeValue(out);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ grp = in.readUTF();
+ name = in.readUTF();
+ readValue(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ HadoopCounterAdapter cntr = (HadoopCounterAdapter)o;
+
+ if (!grp.equals(cntr.grp))
+ return false;
+ if (!name.equals(cntr.name))
+ return false;
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = grp.hashCode();
+ res = 31 * res + name.hashCode();
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopCounterAdapter.class, this);
+ }
+
+ /**
+ * Writes value of this counter to output.
+ *
+ * @param out Output.
+ * @throws IOException If failed.
+ */
+ protected abstract void writeValue(ObjectOutput out) throws IOException;
+
+ /**
+ * Read value of this counter from input.
+ *
+ * @param in Input.
+ * @throws IOException If failed.
+ */
+ protected abstract void readValue(ObjectInput in) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCountersImpl.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCountersImpl.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCountersImpl.java
new file mode 100644
index 0000000..78e1c26
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopCountersImpl.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.counter;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jdk8.backport.*;
+
+import java.io.*;
+import java.lang.reflect.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+/**
+ * Default in-memory counters store.
+ */
+public class HadoopCountersImpl implements HadoopCounters, Externalizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final ConcurrentMap<CounterKey, HadoopCounter> cntrsMap = new ConcurrentHashMap8<>();
+
+ /**
+ * Default constructor. Creates new instance without counters.
+ */
+ public HadoopCountersImpl() {
+ // No-op.
+ }
+
+ /**
+ * Creates new instance that contain given counters.
+ *
+ * @param cntrs Counters to store.
+ */
+ public HadoopCountersImpl(Iterable<HadoopCounter> cntrs) {
+ addCounters(cntrs, true);
+ }
+
+ /**
+ * Copy constructor.
+ *
+ * @param cntrs Counters to copy.
+ */
+ public HadoopCountersImpl(HadoopCounters cntrs) {
+ this(cntrs.all());
+ }
+
+ /**
+ * Creates counter instance.
+ *
+ * @param cls Class of the counter.
+ * @param grp Group name.
+ * @param name Counter name.
+ * @return Counter.
+ */
+ private <T extends HadoopCounter> T createCounter(Class<? extends HadoopCounter> cls, String grp,
+ String name) {
+ try {
+ Constructor constructor = cls.getConstructor(String.class, String.class);
+
+ return (T)constructor.newInstance(grp, name);
+ }
+ catch (Exception e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /**
+ * Adds counters collection in addition to existing counters.
+ *
+ * @param cntrs Counters to add.
+ * @param cp Whether to copy counters or not.
+ */
+ private void addCounters(Iterable<HadoopCounter> cntrs, boolean cp) {
+ assert cntrs != null;
+
+ for (HadoopCounter cntr : cntrs) {
+ if (cp) {
+ HadoopCounter cntrCp = createCounter(cntr.getClass(), cntr.group(), cntr.name());
+
+ cntrCp.merge(cntr);
+
+ cntr = cntrCp;
+ }
+
+ cntrsMap.put(new CounterKey(cntr.getClass(), cntr.group(), cntr.name()), cntr);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T extends HadoopCounter> T counter(String grp, String name, Class<T> cls) {
+ assert cls != null;
+
+ CounterKey mapKey = new CounterKey(cls, grp, name);
+
+ T cntr = (T)cntrsMap.get(mapKey);
+
+ if (cntr == null) {
+ cntr = createCounter(cls, grp, name);
+
+ T old = (T)cntrsMap.putIfAbsent(mapKey, cntr);
+
+ if (old != null)
+ return old;
+ }
+
+ return cntr;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<HadoopCounter> all() {
+ return cntrsMap.values();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void merge(HadoopCounters other) {
+ for (HadoopCounter counter : other.all())
+ counter(counter.group(), counter.name(), counter.getClass()).merge(counter);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ U.writeCollection(out, cntrsMap.values());
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ addCounters(U.<HadoopCounter>readCollection(in), false);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ HadoopCountersImpl counters = (HadoopCountersImpl)o;
+
+ return cntrsMap.equals(counters.cntrsMap);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return cntrsMap.hashCode();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopCountersImpl.class, this, "counters", cntrsMap.values());
+ }
+
+ /**
+ * The tuple of counter identifier components for more readable code.
+ */
+ private static class CounterKey extends GridTuple3<Class<? extends HadoopCounter>, String, String> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * Constructor.
+ *
+ * @param cls Class of the counter.
+ * @param grp Group name.
+ * @param name Counter name.
+ */
+ private CounterKey(Class<? extends HadoopCounter> cls, String grp, String name) {
+ super(cls, grp, name);
+ }
+
+ /**
+ * Empty constructor required by {@link Externalizable}.
+ */
+ public CounterKey() {
+ // No-op.
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopLongCounter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopLongCounter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopLongCounter.java
new file mode 100644
index 0000000..ce86edb
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/HadoopLongCounter.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.counter;
+
+import java.io.*;
+
+/**
+ * Standard hadoop counter to use via original Hadoop API in Hadoop jobs.
+ */
+public class HadoopLongCounter extends HadoopCounterAdapter {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** The counter value. */
+ private long val;
+
+ /**
+ * Default constructor required by {@link Externalizable}.
+ */
+ public HadoopLongCounter() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param grp Group name.
+ * @param name Counter name.
+ */
+ public HadoopLongCounter(String grp, String name) {
+ super(grp, name);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void writeValue(ObjectOutput out) throws IOException {
+ out.writeLong(val);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void readValue(ObjectInput in) throws IOException {
+ val = in.readLong();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void merge(HadoopCounter cntr) {
+ val += ((HadoopLongCounter)cntr).val;
+ }
+
+ /**
+ * Gets current value of this counter.
+ *
+ * @return Current value.
+ */
+ public long value() {
+ return val;
+ }
+
+ /**
+ * Sets current value by the given value.
+ *
+ * @param val Value to set.
+ */
+ public void value(long val) {
+ this.val = val;
+ }
+
+ /**
+ * Increment this counter by the given value.
+ *
+ * @param i Value to increase this counter by.
+ */
+ public void increment(long i) {
+ val += i;
+ }
+}
[37/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java
new file mode 100644
index 0000000..9a6b4d8
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java
@@ -0,0 +1,435 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.mapreduce;
+
+import org.apache.ignite.*;
+import org.apache.ignite.cluster.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.igfs.*;
+import org.apache.ignite.internal.processors.hadoop.planner.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.resources.*;
+import org.jetbrains.annotations.*;
+
+import java.util.*;
+
+import static org.apache.ignite.IgniteFileSystem.*;
+
+/**
+ * Default map-reduce planner implementation.
+ */
+public class IgniteHadoopMapReducePlanner implements HadoopMapReducePlanner {
+ /** Injected grid. */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** Logger. */
+ @SuppressWarnings("UnusedDeclaration")
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** {@inheritDoc} */
+ @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> top,
+ @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
+ // Convert collection of topology nodes to collection of topology node IDs.
+ Collection<UUID> topIds = new HashSet<>(top.size(), 1.0f);
+
+ for (ClusterNode topNode : top)
+ topIds.add(topNode.id());
+
+ Map<UUID, Collection<HadoopInputSplit>> mappers = mappers(top, topIds, job.input());
+
+ int rdcCnt = job.info().reducers();
+
+ if (rdcCnt < 0)
+ throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + rdcCnt);
+
+ Map<UUID, int[]> reducers = reducers(top, mappers, rdcCnt);
+
+ return new HadoopDefaultMapReducePlan(mappers, reducers);
+ }
+
+ /**
+ * Create plan for mappers.
+ *
+ * @param top Topology nodes.
+ * @param topIds Topology node IDs.
+ * @param splits Splits.
+ * @return Mappers map.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Map<UUID, Collection<HadoopInputSplit>> mappers(Collection<ClusterNode> top, Collection<UUID> topIds,
+ Iterable<HadoopInputSplit> splits) throws IgniteCheckedException {
+ Map<UUID, Collection<HadoopInputSplit>> mappers = new HashMap<>();
+
+ Map<String, Collection<UUID>> nodes = hosts(top);
+
+ Map<UUID, Integer> nodeLoads = new HashMap<>(top.size(), 1.0f); // Track node load.
+
+ for (UUID nodeId : topIds)
+ nodeLoads.put(nodeId, 0);
+
+ for (HadoopInputSplit split : splits) {
+ UUID nodeId = nodeForSplit(split, topIds, nodes, nodeLoads);
+
+ if (log.isDebugEnabled())
+ log.debug("Mapped split to node [split=" + split + ", nodeId=" + nodeId + ']');
+
+ Collection<HadoopInputSplit> nodeSplits = mappers.get(nodeId);
+
+ if (nodeSplits == null) {
+ nodeSplits = new ArrayList<>();
+
+ mappers.put(nodeId, nodeSplits);
+ }
+
+ nodeSplits.add(split);
+
+ // Updated node load.
+ nodeLoads.put(nodeId, nodeLoads.get(nodeId) + 1);
+ }
+
+ return mappers;
+ }
+
+ /**
+ * Groups nodes by host names.
+ *
+ * @param top Topology to group.
+ * @return Map.
+ */
+ private static Map<String, Collection<UUID>> hosts(Collection<ClusterNode> top) {
+ Map<String, Collection<UUID>> grouped = U.newHashMap(top.size());
+
+ for (ClusterNode node : top) {
+ for (String host : node.hostNames()) {
+ Collection<UUID> nodeIds = grouped.get(host);
+
+ if (nodeIds == null) {
+ // Expecting 1-2 nodes per host.
+ nodeIds = new ArrayList<>(2);
+
+ grouped.put(host, nodeIds);
+ }
+
+ nodeIds.add(node.id());
+ }
+ }
+
+ return grouped;
+ }
+
+ /**
+ * Determine the best node for this split.
+ *
+ * @param split Split.
+ * @param topIds Topology node IDs.
+ * @param nodes Nodes.
+ * @param nodeLoads Node load tracker.
+ * @return Node ID.
+ */
+ @SuppressWarnings("unchecked")
+ private UUID nodeForSplit(HadoopInputSplit split, Collection<UUID> topIds, Map<String, Collection<UUID>> nodes,
+ Map<UUID, Integer> nodeLoads) throws IgniteCheckedException {
+ if (split instanceof HadoopFileBlock) {
+ HadoopFileBlock split0 = (HadoopFileBlock)split;
+
+ if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
+ HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
+
+ IgfsEx igfs = null;
+
+ if (F.eq(ignite.name(), endpoint.grid()))
+ igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs());
+
+ if (igfs != null && !igfs.isProxy(split0.file())) {
+ Collection<IgfsBlockLocation> blocks;
+
+ try {
+ blocks = igfs.affinity(new IgfsPath(split0.file()), split0.start(), split0.length());
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+
+ assert blocks != null;
+
+ if (blocks.size() == 1)
+ // Fast-path, split consists of one IGFS block (as in most cases).
+ return bestNode(blocks.iterator().next().nodeIds(), topIds, nodeLoads, false);
+ else {
+ // Slow-path, file consists of multiple IGFS blocks. First, find the most co-located nodes.
+ Map<UUID, Long> nodeMap = new HashMap<>();
+
+ List<UUID> bestNodeIds = null;
+ long bestLen = -1L;
+
+ for (IgfsBlockLocation block : blocks) {
+ for (UUID blockNodeId : block.nodeIds()) {
+ if (topIds.contains(blockNodeId)) {
+ Long oldLen = nodeMap.get(blockNodeId);
+ long newLen = oldLen == null ? block.length() : oldLen + block.length();
+
+ nodeMap.put(blockNodeId, newLen);
+
+ if (bestNodeIds == null || bestLen < newLen) {
+ bestNodeIds = new ArrayList<>(1);
+
+ bestNodeIds.add(blockNodeId);
+
+ bestLen = newLen;
+ }
+ else if (bestLen == newLen) {
+ assert !F.isEmpty(bestNodeIds);
+
+ bestNodeIds.add(blockNodeId);
+ }
+ }
+ }
+ }
+
+ if (bestNodeIds != null) {
+ return bestNodeIds.size() == 1 ? bestNodeIds.get(0) :
+ bestNode(bestNodeIds, topIds, nodeLoads, true);
+ }
+ }
+ }
+ }
+ }
+
+ // Cannot use local IGFS for some reason, try selecting the node by host.
+ Collection<UUID> blockNodes = null;
+
+ for (String host : split.hosts()) {
+ Collection<UUID> hostNodes = nodes.get(host);
+
+ if (!F.isEmpty(hostNodes)) {
+ if (blockNodes == null)
+ blockNodes = new ArrayList<>(hostNodes);
+ else
+ blockNodes.addAll(hostNodes);
+ }
+ }
+
+ return bestNode(blockNodes, topIds, nodeLoads, false);
+ }
+
+ /**
+ * Finds the best (the least loaded) node among the candidates.
+ *
+ * @param candidates Candidates.
+ * @param topIds Topology node IDs.
+ * @param nodeLoads Known node loads.
+ * @param skipTopCheck Whether to skip topology check.
+ * @return The best node.
+ */
+ private UUID bestNode(@Nullable Collection<UUID> candidates, Collection<UUID> topIds, Map<UUID, Integer> nodeLoads,
+ boolean skipTopCheck) {
+ UUID bestNode = null;
+ int bestLoad = Integer.MAX_VALUE;
+
+ if (candidates != null) {
+ for (UUID candidate : candidates) {
+ if (skipTopCheck || topIds.contains(candidate)) {
+ int load = nodeLoads.get(candidate);
+
+ if (bestNode == null || bestLoad > load) {
+ bestNode = candidate;
+ bestLoad = load;
+
+ if (bestLoad == 0)
+ break; // Minimum load possible, no need for further iterations.
+ }
+ }
+ }
+ }
+
+ if (bestNode == null) {
+ // Blocks are located on nodes which are not Hadoop-enabled, assign to the least loaded one.
+ bestLoad = Integer.MAX_VALUE;
+
+ for (UUID nodeId : topIds) {
+ int load = nodeLoads.get(nodeId);
+
+ if (bestNode == null || bestLoad > load) {
+ bestNode = nodeId;
+ bestLoad = load;
+
+ if (bestLoad == 0)
+ break; // Minimum load possible, no need for further iterations.
+ }
+ }
+ }
+
+ assert bestNode != null;
+
+ return bestNode;
+ }
+
+ /**
+ * Create plan for reducers.
+ *
+ * @param top Topology.
+ * @param mappers Mappers map.
+ * @param reducerCnt Reducers count.
+ * @return Reducers map.
+ */
+ private Map<UUID, int[]> reducers(Collection<ClusterNode> top,
+ Map<UUID, Collection<HadoopInputSplit>> mappers, int reducerCnt) {
+ // Determine initial node weights.
+ int totalWeight = 0;
+
+ List<WeightedNode> nodes = new ArrayList<>(top.size());
+
+ for (ClusterNode node : top) {
+ Collection<HadoopInputSplit> split = mappers.get(node.id());
+
+ int weight = reducerNodeWeight(node, split != null ? split.size() : 0);
+
+ nodes.add(new WeightedNode(node.id(), weight, weight));
+
+ totalWeight += weight;
+ }
+
+ // Adjust weights.
+ int totalAdjustedWeight = 0;
+
+ for (WeightedNode node : nodes) {
+ node.floatWeight = ((float)node.weight * reducerCnt) / totalWeight;
+
+ node.weight = Math.round(node.floatWeight);
+
+ totalAdjustedWeight += node.weight;
+ }
+
+ // Apply redundant/lost reducers.
+ Collections.sort(nodes);
+
+ if (totalAdjustedWeight > reducerCnt) {
+ // Too much reducers set.
+ ListIterator<WeightedNode> iter = nodes.listIterator(nodes.size() - 1);
+
+ while (totalAdjustedWeight != reducerCnt) {
+ if (!iter.hasPrevious())
+ iter = nodes.listIterator(nodes.size() - 1);
+
+ WeightedNode node = iter.previous();
+
+ if (node.weight > 0) {
+ node.weight -= 1;
+
+ totalAdjustedWeight--;
+ }
+ }
+ }
+ else if (totalAdjustedWeight < reducerCnt) {
+ // Not enough reducers set.
+ ListIterator<WeightedNode> iter = nodes.listIterator(0);
+
+ while (totalAdjustedWeight != reducerCnt) {
+ if (!iter.hasNext())
+ iter = nodes.listIterator(0);
+
+ WeightedNode node = iter.next();
+
+ if (node.floatWeight > 0.0f) {
+ node.weight += 1;
+
+ totalAdjustedWeight++;
+ }
+ }
+ }
+
+ int idx = 0;
+
+ Map<UUID, int[]> reducers = new HashMap<>(nodes.size(), 1.0f);
+
+ for (WeightedNode node : nodes) {
+ if (node.weight > 0) {
+ int[] arr = new int[node.weight];
+
+ for (int i = 0; i < arr.length; i++)
+ arr[i] = idx++;
+
+ reducers.put(node.nodeId, arr);
+ }
+ }
+
+ return reducers;
+ }
+
+ /**
+ * Calculate node weight based on node metrics and data co-location.
+ *
+ * @param node Node.
+ * @param splitCnt Splits mapped to this node.
+ * @return Node weight.
+ */
+ @SuppressWarnings("UnusedParameters")
+ protected int reducerNodeWeight(ClusterNode node, int splitCnt) {
+ return splitCnt;
+ }
+
+ /**
+ * Weighted node.
+ */
+ private static class WeightedNode implements Comparable<WeightedNode> {
+ /** Node ID. */
+ private final UUID nodeId;
+
+ /** Weight. */
+ private int weight;
+
+ /** Floating point weight. */
+ private float floatWeight;
+
+ /**
+ * Constructor.
+ *
+ * @param nodeId Node ID.
+ * @param weight Weight.
+ * @param floatWeight Floating point weight.
+ */
+ private WeightedNode(UUID nodeId, int weight, float floatWeight) {
+ this.nodeId = nodeId;
+ this.weight = weight;
+ this.floatWeight = floatWeight;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ return obj != null && obj instanceof WeightedNode && F.eq(nodeId, ((WeightedNode)obj).nodeId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return nodeId.hashCode();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compareTo(@NotNull WeightedNode other) {
+ float res = other.floatWeight - floatWeight;
+
+ return res > 0.0f ? 1 : res < 0.0f ? -1 : nodeId.compareTo(other.nodeId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/package.html
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/package.html b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/package.html
new file mode 100644
index 0000000..e289841
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/package.html
@@ -0,0 +1,24 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<body>
+ <!-- Package description. -->
+ Ignite Hadoop Accelerator map-reduce classes.
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/hadoop/package.html
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/package.html b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/package.html
new file mode 100644
index 0000000..d687e32
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/package.html
@@ -0,0 +1,24 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<body>
+ <!-- Package description. -->
+ Ignite Hadoop Accelerator API.
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java
deleted file mode 100644
index bdab61d..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java
+++ /dev/null
@@ -1,414 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs.hadoop;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.*;
-import org.apache.hadoop.ipc.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.igfs.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.jetbrains.annotations.*;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-
-/**
- * Adapter to use any Hadoop file system {@link org.apache.hadoop.fs.FileSystem} as {@link org.apache.ignite.igfs.Igfs}.
- */
-public class IgfsHadoopFileSystemWrapper implements Igfs, AutoCloseable {
-
- /** Hadoop file system. */
- private final FileSystem fileSys;
-
- /** Properties of file system */
- private final Map<String, String> props = new HashMap<>();
-
- /**
- * Simple constructor that is to be used by default.
- *
- * @param uri URI of file system.
- * @throws IgniteCheckedException In case of error.
- */
- public IgfsHadoopFileSystemWrapper(String uri) throws IgniteCheckedException {
- this(uri, null);
- }
-
- /**
- * Constructor.
- *
- * @param uri URI of file system.
- * @param cfgPath Additional path to Hadoop configuration.
- * @throws IgniteCheckedException In case of error.
- */
- public IgfsHadoopFileSystemWrapper(@Nullable String uri, @Nullable String cfgPath) throws IgniteCheckedException {
- try {
- SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(uri, cfgPath);
-
- fileSys = secProvider.createFileSystem();
-
- uri = secProvider.uri().toString();
-
- if (!uri.endsWith("/"))
- uri += "/";
-
- props.put(SECONDARY_FS_CONFIG_PATH, cfgPath);
- props.put(SECONDARY_FS_URI, uri);
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /**
- * Convert IGFS path into Hadoop path.
- *
- * @param path IGFS path.
- * @return Hadoop path.
- */
- private Path convert(IgfsPath path) {
- URI uri = fileSys.getUri();
-
- return new Path(uri.getScheme(), uri.getAuthority(), path.toString());
- }
-
- /**
- * Heuristically checks if exception was caused by invalid HDFS version and returns appropriate exception.
- *
- * @param e Exception to check.
- * @param detailMsg Detailed error message.
- * @return Appropriate exception.
- */
- private IgfsException handleSecondaryFsError(IOException e, String detailMsg) {
- boolean wrongVer = X.hasCause(e, RemoteException.class) ||
- (e.getMessage() != null && e.getMessage().contains("Failed on local"));
-
- IgfsException igfsErr = !wrongVer ? cast(detailMsg, e) :
- new IgfsInvalidHdfsVersionException("HDFS version you are connecting to differs from local " +
- "version.", e);
-
- return igfsErr;
- }
-
- /**
- * Cast IO exception to IGFS exception.
- *
- * @param e IO exception.
- * @return IGFS exception.
- */
- public static IgfsException cast(String msg, IOException e) {
- if (e instanceof FileNotFoundException)
- return new IgfsFileNotFoundException(e);
- else if (e instanceof ParentNotDirectoryException)
- return new IgfsParentNotDirectoryException(msg, e);
- else if (e instanceof PathIsNotEmptyDirectoryException)
- return new IgfsDirectoryNotEmptyException(e);
- else if (e instanceof PathExistsException)
- return new IgfsPathAlreadyExistsException(msg, e);
- else
- return new IgfsException(msg, e);
- }
-
- /**
- * Convert Hadoop FileStatus properties to map.
- *
- * @param status File status.
- * @return IGFS attributes.
- */
- private static Map<String, String> properties(FileStatus status) {
- FsPermission perm = status.getPermission();
-
- if (perm == null)
- perm = FsPermission.getDefault();
-
- return F.asMap(PROP_PERMISSION, String.format("%04o", perm.toShort()), PROP_USER_NAME, status.getOwner(),
- PROP_GROUP_NAME, status.getGroup());
- }
-
- /** {@inheritDoc} */
- @Override public boolean exists(IgfsPath path) {
- try {
- return fileSys.exists(convert(path));
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to check file existence [path=" + path + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public IgfsFile update(IgfsPath path, Map<String, String> props) {
- IgfsHadoopFSProperties props0 = new IgfsHadoopFSProperties(props);
-
- try {
- if (props0.userName() != null || props0.groupName() != null)
- fileSys.setOwner(convert(path), props0.userName(), props0.groupName());
-
- if (props0.permission() != null)
- fileSys.setPermission(convert(path), props0.permission());
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to update file properties [path=" + path + "]");
- }
-
- //Result is not used in case of secondary FS.
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void rename(IgfsPath src, IgfsPath dest) {
- // Delegate to the secondary file system.
- try {
- if (!fileSys.rename(convert(src), convert(dest)))
- throw new IgfsException("Failed to rename (secondary file system returned false) " +
- "[src=" + src + ", dest=" + dest + ']');
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to rename file [src=" + src + ", dest=" + dest + ']');
- }
- }
-
- /** {@inheritDoc} */
- @Override public boolean delete(IgfsPath path, boolean recursive) {
- try {
- return fileSys.delete(convert(path), recursive);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to delete file [path=" + path + ", recursive=" + recursive + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public void mkdirs(IgfsPath path) {
- try {
- if (!fileSys.mkdirs(convert(path)))
- throw new IgniteException("Failed to make directories [path=" + path + "]");
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to make directories [path=" + path + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) {
- try {
- if (!fileSys.mkdirs(convert(path), new IgfsHadoopFSProperties(props).permission()))
- throw new IgniteException("Failed to make directories [path=" + path + ", props=" + props + "]");
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to make directories [path=" + path + ", props=" + props + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsPath> listPaths(IgfsPath path) {
- try {
- FileStatus[] statuses = fileSys.listStatus(convert(path));
-
- if (statuses == null)
- throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
-
- Collection<IgfsPath> res = new ArrayList<>(statuses.length);
-
- for (FileStatus status : statuses)
- res.add(new IgfsPath(path, status.getPath().getName()));
-
- return res;
- }
- catch (FileNotFoundException ignored) {
- throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsFile> listFiles(IgfsPath path) {
- try {
- FileStatus[] statuses = fileSys.listStatus(convert(path));
-
- if (statuses == null)
- throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
-
- Collection<IgfsFile> res = new ArrayList<>(statuses.length);
-
- for (FileStatus status : statuses) {
- IgfsFileInfo fsInfo = status.isDirectory() ? new IgfsFileInfo(true, properties(status)) :
- new IgfsFileInfo((int)status.getBlockSize(), status.getLen(), null, null, false,
- properties(status));
-
- res.add(new IgfsFileImpl(new IgfsPath(path, status.getPath().getName()), fsInfo, 1));
- }
-
- return res;
- }
- catch (FileNotFoundException ignored) {
- throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsReader open(IgfsPath path, int bufSize) {
- return new IgfsHadoopReader(fileSys, convert(path), bufSize);
- }
-
- /** {@inheritDoc} */
- @Override public OutputStream create(IgfsPath path, boolean overwrite) {
- try {
- return fileSys.create(convert(path), overwrite);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to create file [path=" + path + ", overwrite=" + overwrite + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
- long blockSize, @Nullable Map<String, String> props) {
- IgfsHadoopFSProperties props0 =
- new IgfsHadoopFSProperties(props != null ? props : Collections.<String, String>emptyMap());
-
- try {
- return fileSys.create(convert(path), props0.permission(), overwrite, bufSize, (short)replication, blockSize,
- null);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to create file [path=" + path + ", props=" + props +
- ", overwrite=" + overwrite + ", bufSize=" + bufSize + ", replication=" + replication +
- ", blockSize=" + blockSize + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public OutputStream append(IgfsPath path, int bufSize, boolean create,
- @Nullable Map<String, String> props) {
- try {
- return fileSys.append(convert(path), bufSize);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to append file [path=" + path + ", bufSize=" + bufSize + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsFile info(final IgfsPath path) {
- try {
- final FileStatus status = fileSys.getFileStatus(convert(path));
-
- if (status == null)
- return null;
-
- final Map<String, String> props = properties(status);
-
- return new IgfsFile() {
- @Override public IgfsPath path() {
- return path;
- }
-
- @Override public boolean isFile() {
- return status.isFile();
- }
-
- @Override public boolean isDirectory() {
- return status.isDirectory();
- }
-
- @Override public int blockSize() {
- return (int)status.getBlockSize();
- }
-
- @Override public long groupBlockSize() {
- return status.getBlockSize();
- }
-
- @Override public long accessTime() {
- return status.getAccessTime();
- }
-
- @Override public long modificationTime() {
- return status.getModificationTime();
- }
-
- @Override public String property(String name) throws IllegalArgumentException {
- String val = props.get(name);
-
- if (val == null)
- throw new IllegalArgumentException("File property not found [path=" + path + ", name=" + name + ']');
-
- return val;
- }
-
- @Nullable @Override public String property(String name, @Nullable String dfltVal) {
- String val = props.get(name);
-
- return val == null ? dfltVal : val;
- }
-
- @Override public long length() {
- return status.getLen();
- }
-
- /** {@inheritDoc} */
- @Override public Map<String, String> properties() {
- return props;
- }
- };
-
- }
- catch (FileNotFoundException ignore) {
- return null;
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public long usedSpaceSize() {
- try {
- return fileSys.getContentSummary(new Path(fileSys.getUri())).getSpaceConsumed();
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to get used space size of file system.");
- }
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public Map<String, String> properties() {
- return props;
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteCheckedException {
- try {
- fileSys.close();
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopParameters.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopParameters.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopParameters.java
deleted file mode 100644
index 6237dd4..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopParameters.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs.hadoop;
-
-/**
- * This class lists parameters that can be specified in Hadoop configuration.
- * Hadoop configuration can be specified in {@code core-site.xml} file
- * or passed to map-reduce task directly when using Hadoop driver for IGFS file system:
- * <ul>
- * <li>
- * {@code fs.igfs.[name].open.sequential_reads_before_prefetch} - this parameter overrides
- * the one specified in {@link org.apache.ignite.configuration.IgfsConfiguration#getSequentialReadsBeforePrefetch()}
- * IGFS data node configuration property.
- * </li>
- * <li>
- * {@code fs.igfs.[name].log.enabled} - specifies whether IGFS sampling logger is enabled. If
- * {@code true}, then all file system operations will be logged to a file.
- * </li>
- * <li>{@code fs.igfs.[name].log.dir} - specifies log directory where sampling log files should be placed.</li>
- * <li>
- * {@code fs.igfs.[name].log.batch_size} - specifies how many log entries are accumulated in a batch before
- * it gets flushed to log file. Higher values will imply greater performance, but will increase delay
- * before record appears in the log file.
- * </li>
- * <li>
- * {@code fs.igfs.[name].colocated.writes} - specifies whether written files should be colocated on data
- * node to which client is connected. If {@code true}, file will not be distributed and will be written
- * to a single data node. Default value is {@code true}.
- * </li>
- * <li>
- * {@code fs.igfs.prefer.local.writes} - specifies whether file preferably should be written to
- * local data node if it has enough free space. After some time it can be redistributed across nodes though.
- * </li>
- * </ul>
- * Where {@code [name]} is file system endpoint which you specify in file system URI authority part. E.g. in
- * case your file system URI is {@code igfs://127.0.0.1:10500} then {@code name} will be {@code 127.0.0.1:10500}.
- * <p>
- * Sample configuration that can be placed to {@code core-site.xml} file:
- * <pre name="code" class="xml">
- * <property>
- * <name>fs.igfs.127.0.0.1:10500.log.enabled</name>
- * <value>true</value>
- * </property>
- * <property>
- * <name>fs.igfs.127.0.0.1:10500.log.dir</name>
- * <value>/home/apache/ignite/log/sampling</value>
- * </property>
- * <property>
- * <name>fs.igfs.127.0.0.1:10500.log.batch_size</name>
- * <value>16</value>
- * </property>
- * </pre>
- * Parameters could also be specified per mapreduce job, e.g.
- * <pre name="code" class="bash">
- * hadoop jar myjarfile.jar MyMapReduceJob -Dfs.igfs.open.sequential_reads_before_prefetch=4
- * </pre>
- * If you want to use these parameters in code, then you have to substitute you file system name in it. The easiest
- * way to do that is {@code String.format(PARAM_IGFS_COLOCATED_WRITES, [name])}.
- */
-public class IgfsHadoopParameters {
- /** Parameter name for control over file colocation write mode. */
- public static final String PARAM_IGFS_COLOCATED_WRITES = "fs.igfs.%s.colocated.writes";
-
- /** Parameter name for custom sequential reads before prefetch value. */
- public static final String PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH =
- "fs.igfs.%s.open.sequential_reads_before_prefetch";
-
- /** Parameter name for client logger directory. */
- public static final String PARAM_IGFS_LOG_DIR = "fs.igfs.%s.log.dir";
-
- /** Parameter name for log batch size. */
- public static final String PARAM_IGFS_LOG_BATCH_SIZE = "fs.igfs.%s.log.batch_size";
-
- /** Parameter name for log enabled flag. */
- public static final String PARAM_IGFS_LOG_ENABLED = "fs.igfs.%s.log.enabled";
-
- /** Parameter name for prefer local writes flag. */
- public static final String PARAM_IGFS_PREFER_LOCAL_WRITES = "fs.igfs.prefer.local.writes";
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/package.html
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/package.html b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/package.html
deleted file mode 100644
index 137055b..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/package.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<body>
- <!-- Package description. -->
- Contains common files for Hadoop 1.x and Hadoop 2.x distros.
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java
deleted file mode 100644
index c4d2f5e..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java
+++ /dev/null
@@ -1,1234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs.hadoop.v1;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.*;
-import org.apache.hadoop.hdfs.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.util.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.igfs.common.*;
-import org.apache.ignite.internal.igfs.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-import java.util.concurrent.atomic.*;
-
-import static org.apache.ignite.IgniteFs.*;
-import static org.apache.ignite.configuration.IgfsConfiguration.*;
-import static org.apache.ignite.igfs.IgfsMode.*;
-import static org.apache.ignite.igfs.hadoop.IgfsHadoopParameters.*;
-import static org.apache.ignite.internal.igfs.hadoop.IgfsHadoopUtils.*;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
-
-/**
- * {@code IGFS} Hadoop 1.x file system driver over file system API. To use
- * {@code IGFS} as Hadoop file system, you should configure this class
- * in Hadoop's {@code core-site.xml} as follows:
- * <pre name="code" class="xml">
- * <property>
- * <name>fs.default.name</name>
- * <value>igfs://ipc</value>
- * </property>
- *
- * <property>
- * <name>fs.igfs.impl</name>
- * <value>org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem</value>
- * </property>
- * </pre>
- * You should also add Ignite JAR and all libraries to Hadoop classpath. To
- * do this, add following lines to {@code conf/hadoop-env.sh} script in Hadoop
- * distribution:
- * <pre name="code" class="bash">
- * export IGNITE_HOME=/path/to/Ignite/distribution
- * export HADOOP_CLASSPATH=$IGNITE_HOME/ignite*.jar
- *
- * for f in $IGNITE_HOME/libs/*.jar; do
- * export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f;
- * done
- * </pre>
- * <h1 class="header">Data vs Clients Nodes</h1>
- * Hadoop needs to use its FileSystem remotely from client nodes as well as directly on
- * data nodes. Client nodes are responsible for basic file system operations as well as
- * accessing data nodes remotely. Usually, client nodes are started together
- * with {@code job-submitter} or {@code job-scheduler} processes, while data nodes are usually
- * started together with Hadoop {@code task-tracker} processes.
- * <p>
- * For sample client and data node configuration refer to {@code config/hadoop/default-config-client.xml}
- * and {@code config/hadoop/default-config.xml} configuration files in Ignite installation.
- */
-public class IgfsHadoopFileSystem extends FileSystem {
- /** Internal property to indicate management connection. */
- public static final String IGFS_MANAGEMENT = "fs.igfs.management.connection";
-
- /** Empty array of file block locations. */
- private static final BlockLocation[] EMPTY_BLOCK_LOCATIONS = new BlockLocation[0];
-
- /** Empty array of file statuses. */
- public static final FileStatus[] EMPTY_FILE_STATUS = new FileStatus[0];
-
- /** Ensures that close routine is invoked at most once. */
- private final AtomicBoolean closeGuard = new AtomicBoolean();
-
- /** Grid remote client. */
- private IgfsHadoopWrapper rmtClient;
-
- /** User name for each thread. */
- private final ThreadLocal<String> userName = new ThreadLocal<String>(){
- /** {@inheritDoc} */
- @Override protected String initialValue() {
- return DFLT_USER_NAME;
- }
- };
-
- /** Working directory for each thread. */
- private final ThreadLocal<Path> workingDir = new ThreadLocal<Path>(){
- /** {@inheritDoc} */
- @Override protected Path initialValue() {
- return getHomeDirectory();
- }
- };
-
- /** Default replication factor. */
- private short dfltReplication;
-
- /** Base file system uri. */
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private URI uri;
-
- /** Authority. */
- private String uriAuthority;
-
- /** Client logger. */
- private IgfsLogger clientLog;
-
- /** Secondary URI string. */
- private URI secondaryUri;
-
- /** IGFS mode resolver. */
- private IgfsModeResolver modeRslvr;
-
- /** Secondary file system instance. */
- private FileSystem secondaryFs;
-
- /** Management connection flag. */
- private boolean mgmt;
-
- /** Whether custom sequential reads before prefetch value is provided. */
- private boolean seqReadsBeforePrefetchOverride;
-
- /** IGFS group block size. */
- private long igfsGrpBlockSize;
-
- /** Flag that controls whether file writes should be colocated. */
- private boolean colocateFileWrites;
-
- /** Prefer local writes. */
- private boolean preferLocFileWrites;
-
- /** Custom-provided sequential reads before prefetch. */
- private int seqReadsBeforePrefetch;
-
- /** The cache was disabled when the instance was creating. */
- private boolean cacheEnabled;
-
- /** {@inheritDoc} */
- @Override public URI getUri() {
- if (uri == null)
- throw new IllegalStateException("URI is null (was IgfsHadoopFileSystem properly initialized?).");
-
- return uri;
- }
-
- /**
- * Enter busy state.
- *
- * @throws IOException If file system is stopped.
- */
- private void enterBusy() throws IOException {
- if (closeGuard.get())
- throw new IOException("File system is stopped.");
- }
-
- /**
- * Leave busy state.
- */
- private void leaveBusy() {
- // No-op.
- }
-
- /**
- * Public setter that can be used by direct users of FS or Visor.
- *
- * @param colocateFileWrites Whether all ongoing file writes should be colocated.
- */
- @SuppressWarnings("UnusedDeclaration")
- public void colocateFileWrites(boolean colocateFileWrites) {
- this.colocateFileWrites = colocateFileWrites;
- }
-
- /** {@inheritDoc} */
- @Override public void initialize(URI name, Configuration cfg) throws IOException {
- enterBusy();
-
- try {
- if (rmtClient != null)
- throw new IOException("File system is already initialized: " + rmtClient);
-
- A.notNull(name, "name");
- A.notNull(cfg, "cfg");
-
- super.initialize(name, cfg);
-
- setConf(cfg);
-
- String disableCacheName = String.format("fs.%s.impl.disable.cache", name.getScheme());
-
- cacheEnabled = !cfg.getBoolean(disableCacheName, false);
-
- mgmt = cfg.getBoolean(IGFS_MANAGEMENT, false);
-
- if (!IGFS_SCHEME.equals(name.getScheme()))
- throw new IOException("Illegal file system URI [expected=" + IGFS_SCHEME +
- "://[name]/[optional_path], actual=" + name + ']');
-
- uri = name;
-
- uriAuthority = uri.getAuthority();
-
- setUser(cfg.get(MRJobConfig.USER_NAME, DFLT_USER_NAME));
-
- // Override sequential reads before prefetch if needed.
- seqReadsBeforePrefetch = parameter(cfg, PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH, uriAuthority, 0);
-
- if (seqReadsBeforePrefetch > 0)
- seqReadsBeforePrefetchOverride = true;
-
- // In Ignite replication factor is controlled by data cache affinity.
- // We use replication factor to force the whole file to be stored on local node.
- dfltReplication = (short)cfg.getInt("dfs.replication", 3);
-
- // Get file colocation control flag.
- colocateFileWrites = parameter(cfg, PARAM_IGFS_COLOCATED_WRITES, uriAuthority, false);
- preferLocFileWrites = cfg.getBoolean(PARAM_IGFS_PREFER_LOCAL_WRITES, false);
-
- // Get log directory.
- String logDirCfg = parameter(cfg, PARAM_IGFS_LOG_DIR, uriAuthority, DFLT_IGFS_LOG_DIR);
-
- File logDirFile = U.resolveIgnitePath(logDirCfg);
-
- String logDir = logDirFile != null ? logDirFile.getAbsolutePath() : null;
-
- rmtClient = new IgfsHadoopWrapper(uriAuthority, logDir, cfg, LOG);
-
- // Handshake.
- IgfsHandshakeResponse handshake = rmtClient.handshake(logDir);
-
- igfsGrpBlockSize = handshake.blockSize();
-
- IgfsPaths paths = handshake.secondaryPaths();
-
- // Initialize client logger.
- Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);
-
- if (handshake.sampling() != null ? handshake.sampling() : logEnabled) {
- // Initiate client logger.
- if (logDir == null)
- throw new IOException("Failed to resolve log directory: " + logDirCfg);
-
- Integer batchSize = parameter(cfg, PARAM_IGFS_LOG_BATCH_SIZE, uriAuthority, DFLT_IGFS_LOG_BATCH_SIZE);
-
- clientLog = IgfsLogger.logger(uriAuthority, handshake.igfsName(), logDir, batchSize);
- }
- else
- clientLog = IgfsLogger.disabledLogger();
-
- modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes());
-
- boolean initSecondary = paths.defaultMode() == PROXY;
-
- if (!initSecondary && paths.pathModes() != null && !paths.pathModes().isEmpty()) {
- for (T2<IgfsPath, IgfsMode> pathMode : paths.pathModes()) {
- IgfsMode mode = pathMode.getValue();
-
- if (mode == PROXY) {
- initSecondary = true;
-
- break;
- }
- }
- }
-
- if (initSecondary) {
- Map<String, String> props = paths.properties();
-
- String secUri = props.get(SECONDARY_FS_URI);
- String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
-
- try {
- SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
-
- secondaryFs = secProvider.createFileSystem();
- secondaryUri = secProvider.uri();
- }
- catch (IOException e) {
- if (!mgmt)
- throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
- else
- LOG.warn("Visor failed to create secondary file system (operations on paths with PROXY mode " +
- "will have no effect): " + e.getMessage());
- }
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override protected void checkPath(Path path) {
- URI uri = path.toUri();
-
- if (uri.isAbsolute()) {
- if (!F.eq(uri.getScheme(), IGFS_SCHEME))
- throw new InvalidPathException("Wrong path scheme [expected=" + IGFS_SCHEME + ", actual=" +
- uri.getAuthority() + ']');
-
- if (!F.eq(uri.getAuthority(), uriAuthority))
- throw new InvalidPathException("Wrong path authority [expected=" + uriAuthority + ", actual=" +
- uri.getAuthority() + ']');
- }
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("deprecation")
- @Override public short getDefaultReplication() {
- return dfltReplication;
- }
-
- /** {@inheritDoc} */
- @Override protected void finalize() throws Throwable {
- super.finalize();
-
- close0();
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IOException {
- if (cacheEnabled && get(getUri(), getConf()) == this)
- return;
-
- close0();
- }
-
- /**
- * Closes file system.
- *
- * @throws IOException If failed.
- */
- private void close0() throws IOException {
- if (closeGuard.compareAndSet(false, true)) {
- if (LOG.isDebugEnabled())
- LOG.debug("File system closed [uri=" + uri + ", endpoint=" + uriAuthority + ']');
-
- if (rmtClient == null)
- return;
-
- super.close();
-
- rmtClient.close(false);
-
- if (clientLog.isLogEnabled())
- clientLog.close();
-
- if (secondaryFs != null)
- U.closeQuiet(secondaryFs);
-
- // Reset initialized resources.
- uri = null;
- rmtClient = null;
- }
- }
-
- /** {@inheritDoc} */
- @Override public void setTimes(Path p, long mtime, long atime) throws IOException {
- enterBusy();
-
- try {
- A.notNull(p, "p");
-
- if (mode(p) == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- // No-op for management connection.
- return;
- }
-
- secondaryFs.setTimes(toSecondary(p), mtime, atime);
- }
- else {
- IgfsPath path = convert(p);
-
- rmtClient.setTimes(path, atime, mtime);
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void setPermission(Path p, FsPermission perm) throws IOException {
- enterBusy();
-
- try {
- A.notNull(p, "p");
-
- if (mode(p) == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- // No-op for management connection.
- return;
- }
-
- secondaryFs.setPermission(toSecondary(p), perm);
- }
- else if (rmtClient.update(convert(p), permission(perm)) == null) {
- throw new IOException("Failed to set file permission (file not found?)" +
- " [path=" + p + ", perm=" + perm + ']');
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void setOwner(Path p, String username, String grpName) throws IOException {
- A.notNull(p, "p");
- A.notNull(username, "username");
- A.notNull(grpName, "grpName");
-
- enterBusy();
-
- try {
- if (mode(p) == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- // No-op for management connection.
- return;
- }
-
- secondaryFs.setOwner(toSecondary(p), username, grpName);
- }
- else if (rmtClient.update(convert(p), F.asMap(PROP_USER_NAME, username, PROP_GROUP_NAME, grpName)) == null)
- throw new IOException("Failed to set file permission (file not found?)" +
- " [path=" + p + ", userName=" + username + ", groupName=" + grpName + ']');
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public FSDataInputStream open(Path f, int bufSize) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- try {
- IgfsPath path = convert(f);
- IgfsMode mode = mode(path);
-
- if (mode == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- throw new IOException("Failed to open file (secondary file system is not initialized): " + f);
- }
-
- FSDataInputStream is = secondaryFs.open(toSecondary(f), bufSize);
-
- if (clientLog.isLogEnabled()) {
- // At this point we do not know file size, so we perform additional request to remote FS to get it.
- FileStatus status = secondaryFs.getFileStatus(toSecondary(f));
-
- long size = status != null ? status.getLen() : -1;
-
- long logId = IgfsLogger.nextId();
-
- clientLog.logOpen(logId, path, PROXY, bufSize, size);
-
- return new FSDataInputStream(new IgfsHadoopProxyInputStream(is, clientLog, logId));
- }
- else
- return is;
- }
- else {
- IgfsHadoopStreamDelegate stream = seqReadsBeforePrefetchOverride ?
- rmtClient.open(path, seqReadsBeforePrefetch) : rmtClient.open(path);
-
- long logId = -1;
-
- if (clientLog.isLogEnabled()) {
- logId = IgfsLogger.nextId();
-
- clientLog.logOpen(logId, path, mode, bufSize, stream.length());
- }
-
- if (LOG.isDebugEnabled())
- LOG.debug("Opening input stream [thread=" + Thread.currentThread().getName() + ", path=" + path +
- ", bufSize=" + bufSize + ']');
-
- IgfsHadoopInputStream igfsIn = new IgfsHadoopInputStream(stream, stream.length(),
- bufSize, LOG, clientLog, logId);
-
- if (LOG.isDebugEnabled())
- LOG.debug("Opened input stream [path=" + path + ", delegate=" + stream + ']');
-
- return new FSDataInputStream(igfsIn);
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("deprecation")
- @Override public FSDataOutputStream create(Path f, FsPermission perm, boolean overwrite, int bufSize,
- short replication, long blockSize, Progressable progress) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- OutputStream out = null;
-
- try {
- IgfsPath path = convert(f);
- IgfsMode mode = mode(path);
-
- if (LOG.isDebugEnabled())
- LOG.debug("Opening output stream in create [thread=" + Thread.currentThread().getName() + "path=" +
- path + ", overwrite=" + overwrite + ", bufSize=" + bufSize + ']');
-
- if (mode == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- throw new IOException("Failed to create file (secondary file system is not initialized): " + f);
- }
-
- FSDataOutputStream os =
- secondaryFs.create(toSecondary(f), perm, overwrite, bufSize, replication, blockSize, progress);
-
- if (clientLog.isLogEnabled()) {
- long logId = IgfsLogger.nextId();
-
- clientLog.logCreate(logId, path, PROXY, overwrite, bufSize, replication, blockSize);
-
- return new FSDataOutputStream(new IgfsHadoopProxyOutputStream(os, clientLog, logId));
- }
- else
- return os;
- }
- else {
- // Create stream and close it in the 'finally' section if any sequential operation failed.
- IgfsHadoopStreamDelegate stream = rmtClient.create(path, overwrite, colocateFileWrites,
- replication, blockSize, F.asMap(PROP_PERMISSION, toString(perm),
- PROP_PREFER_LOCAL_WRITES, Boolean.toString(preferLocFileWrites)));
-
- assert stream != null;
-
- long logId = -1;
-
- if (clientLog.isLogEnabled()) {
- logId = IgfsLogger.nextId();
-
- clientLog.logCreate(logId, path, mode, overwrite, bufSize, replication, blockSize);
- }
-
- if (LOG.isDebugEnabled())
- LOG.debug("Opened output stream in create [path=" + path + ", delegate=" + stream + ']');
-
- IgfsHadoopOutputStream igfsOut = new IgfsHadoopOutputStream(stream, LOG, clientLog,
- logId);
-
- bufSize = Math.max(64 * 1024, bufSize);
-
- out = new BufferedOutputStream(igfsOut, bufSize);
-
- FSDataOutputStream res = new FSDataOutputStream(out, null, 0);
-
- // Mark stream created successfully.
- out = null;
-
- return res;
- }
- }
- finally {
- // Close if failed during stream creation.
- if (out != null)
- U.closeQuiet(out);
-
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("deprecation")
- @Override public FSDataOutputStream append(Path f, int bufSize, Progressable progress) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- try {
- IgfsPath path = convert(f);
- IgfsMode mode = mode(path);
-
- if (LOG.isDebugEnabled())
- LOG.debug("Opening output stream in append [thread=" + Thread.currentThread().getName() +
- ", path=" + path + ", bufSize=" + bufSize + ']');
-
- if (mode == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- throw new IOException("Failed to append file (secondary file system is not initialized): " + f);
- }
-
- FSDataOutputStream os = secondaryFs.append(toSecondary(f), bufSize, progress);
-
- if (clientLog.isLogEnabled()) {
- long logId = IgfsLogger.nextId();
-
- clientLog.logAppend(logId, path, PROXY, bufSize); // Don't have stream ID.
-
- return new FSDataOutputStream(new IgfsHadoopProxyOutputStream(os, clientLog, logId));
- }
- else
- return os;
- }
- else {
- IgfsHadoopStreamDelegate stream = rmtClient.append(path, false, null);
-
- assert stream != null;
-
- long logId = -1;
-
- if (clientLog.isLogEnabled()) {
- logId = IgfsLogger.nextId();
-
- clientLog.logAppend(logId, path, mode, bufSize);
- }
-
- if (LOG.isDebugEnabled())
- LOG.debug("Opened output stream in append [path=" + path + ", delegate=" + stream + ']');
-
- IgfsHadoopOutputStream igfsOut = new IgfsHadoopOutputStream(stream, LOG, clientLog,
- logId);
-
- bufSize = Math.max(64 * 1024, bufSize);
-
- BufferedOutputStream out = new BufferedOutputStream(igfsOut, bufSize);
-
- return new FSDataOutputStream(out, null, 0);
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public boolean rename(Path src, Path dst) throws IOException {
- A.notNull(src, "src");
- A.notNull(dst, "dst");
-
- enterBusy();
-
- try {
- IgfsPath srcPath = convert(src);
- IgfsPath dstPath = convert(dst);
- IgfsMode mode = mode(srcPath);
-
- if (mode == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- return false;
- }
-
- if (clientLog.isLogEnabled())
- clientLog.logRename(srcPath, PROXY, dstPath);
-
- return secondaryFs.rename(toSecondary(src), toSecondary(dst));
- }
- else {
- // Will throw exception if failed.
- rmtClient.rename(srcPath, dstPath);
-
- if (clientLog.isLogEnabled())
- clientLog.logRename(srcPath, mode, dstPath);
-
- return true;
- }
- }
- catch (IOException e) {
- // Intentionally ignore IGFS exceptions here to follow Hadoop contract.
- if (F.eq(IOException.class, e.getClass()) && (e.getCause() == null ||
- !X.hasCause(e.getCause(), IgfsException.class)))
- throw e;
- else
- return false;
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("deprecation")
- @Override public boolean delete(Path f) throws IOException {
- return delete(f, false);
- }
-
- /** {@inheritDoc} */
- @Override public boolean delete(Path f, boolean recursive) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- try {
- IgfsPath path = convert(f);
- IgfsMode mode = mode(path);
-
- if (mode == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- return false;
- }
-
- if (clientLog.isLogEnabled())
- clientLog.logDelete(path, PROXY, recursive);
-
- return secondaryFs.delete(toSecondary(f), recursive);
- }
- else {
- // Will throw exception if delete failed.
- boolean res = rmtClient.delete(path, recursive);
-
- if (clientLog.isLogEnabled())
- clientLog.logDelete(path, mode, recursive);
-
- return res;
- }
- }
- catch (IOException e) {
- // Intentionally ignore IGFS exceptions here to follow Hadoop contract.
- if (F.eq(IOException.class, e.getClass()) && (e.getCause() == null ||
- !X.hasCause(e.getCause(), IgfsException.class)))
- throw e;
- else
- return false;
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public FileStatus[] listStatus(Path f) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- try {
- IgfsPath path = convert(f);
- IgfsMode mode = mode(path);
-
- if (mode == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- return EMPTY_FILE_STATUS;
- }
-
- FileStatus[] arr = secondaryFs.listStatus(toSecondary(f));
-
- if (arr == null)
- throw new FileNotFoundException("File " + f + " does not exist.");
-
- for (int i = 0; i < arr.length; i++)
- arr[i] = toPrimary(arr[i]);
-
- if (clientLog.isLogEnabled()) {
- String[] fileArr = new String[arr.length];
-
- for (int i = 0; i < arr.length; i++)
- fileArr[i] = arr[i].getPath().toString();
-
- clientLog.logListDirectory(path, PROXY, fileArr);
- }
-
- return arr;
- }
- else {
- Collection<IgfsFile> list = rmtClient.listFiles(path);
-
- if (list == null)
- throw new FileNotFoundException("File " + f + " does not exist.");
-
- List<IgfsFile> files = new ArrayList<>(list);
-
- FileStatus[] arr = new FileStatus[files.size()];
-
- for (int i = 0; i < arr.length; i++)
- arr[i] = convert(files.get(i));
-
- if (clientLog.isLogEnabled()) {
- String[] fileArr = new String[arr.length];
-
- for (int i = 0; i < arr.length; i++)
- fileArr[i] = arr[i].getPath().toString();
-
- clientLog.logListDirectory(path, mode, fileArr);
- }
-
- return arr;
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public Path getHomeDirectory() {
- Path path = new Path("/user/" + userName.get());
-
- return path.makeQualified(getUri(), null);
- }
-
- /**
- * Set user name and default working directory for current thread.
- *
- * @param userName User name.
- */
- public void setUser(String userName) {
- this.userName.set(userName);
-
- setWorkingDirectory(null);
- }
-
- /** {@inheritDoc} */
- @Override public void setWorkingDirectory(Path newPath) {
- if (newPath == null) {
- Path homeDir = getHomeDirectory();
-
- if (secondaryFs != null)
- secondaryFs.setWorkingDirectory(toSecondary(homeDir));
-
- workingDir.set(homeDir);
- }
- else {
- Path fixedNewPath = fixRelativePart(newPath);
-
- String res = fixedNewPath.toUri().getPath();
-
- if (!DFSUtil.isValidName(res))
- throw new IllegalArgumentException("Invalid DFS directory name " + res);
-
- if (secondaryFs != null)
- secondaryFs.setWorkingDirectory(toSecondary(fixedNewPath));
-
- workingDir.set(fixedNewPath);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Path getWorkingDirectory() {
- return workingDir.get();
- }
-
- /** {@inheritDoc} */
- @Override public boolean mkdirs(Path f, FsPermission perm) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- try {
- IgfsPath path = convert(f);
- IgfsMode mode = mode(path);
-
- if (mode == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- return false;
- }
-
- if (clientLog.isLogEnabled())
- clientLog.logMakeDirectory(path, PROXY);
-
- return secondaryFs.mkdirs(toSecondary(f), perm);
- }
- else {
- boolean mkdirRes = rmtClient.mkdirs(path, permission(perm));
-
- if (clientLog.isLogEnabled())
- clientLog.logMakeDirectory(path, mode);
-
- return mkdirRes;
- }
- }
- catch (IOException e) {
- // Intentionally ignore IGFS exceptions here to follow Hadoop contract.
- if (F.eq(IOException.class, e.getClass()) && (e.getCause() == null ||
- !X.hasCause(e.getCause(), IgfsException.class)))
- throw e;
- else
- return false;
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public FileStatus getFileStatus(Path f) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- try {
- if (mode(f) == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- throw new IOException("Failed to get file status (secondary file system is not initialized): " + f);
- }
-
- return toPrimary(secondaryFs.getFileStatus(toSecondary(f)));
- }
- else {
- IgfsFile info = rmtClient.info(convert(f));
-
- if (info == null)
- throw new FileNotFoundException("File not found: " + f);
-
- return convert(info);
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public ContentSummary getContentSummary(Path f) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- try {
- if (mode(f) == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- throw new IOException("Failed to get content summary (secondary file system is not initialized): " +
- f);
- }
-
- return secondaryFs.getContentSummary(toSecondary(f));
- }
- else {
- IgfsPathSummary sum = rmtClient.contentSummary(convert(f));
-
- return new ContentSummary(sum.totalLength(), sum.filesCount(), sum.directoriesCount(),
- -1, sum.totalLength(), rmtClient.fsStatus().spaceTotal());
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public BlockLocation[] getFileBlockLocations(FileStatus status, long start, long len) throws IOException {
- A.notNull(status, "status");
-
- enterBusy();
-
- try {
- IgfsPath path = convert(status.getPath());
-
- if (mode(status.getPath()) == PROXY) {
- if (secondaryFs == null) {
- assert mgmt;
-
- return EMPTY_BLOCK_LOCATIONS;
- }
-
- Path secPath = toSecondary(status.getPath());
-
- return secondaryFs.getFileBlockLocations(secondaryFs.getFileStatus(secPath), start, len);
- }
- else {
- long now = System.currentTimeMillis();
-
- List<IgfsBlockLocation> affinity = new ArrayList<>(rmtClient.affinity(path, start, len));
-
- BlockLocation[] arr = new BlockLocation[affinity.size()];
-
- for (int i = 0; i < arr.length; i++)
- arr[i] = convert(affinity.get(i));
-
- if (LOG.isDebugEnabled())
- LOG.debug("Fetched file locations [path=" + path + ", fetchTime=" +
- (System.currentTimeMillis() - now) + ", locations=" + Arrays.asList(arr) + ']');
-
- return arr;
- }
- }
- catch (FileNotFoundException ignored) {
- return EMPTY_BLOCK_LOCATIONS;
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("deprecation")
- @Override public long getDefaultBlockSize() {
- return igfsGrpBlockSize;
- }
-
- /**
- * Resolve path mode.
- *
- * @param path HDFS path.
- * @return Path mode.
- */
- public IgfsMode mode(Path path) {
- return mode(convert(path));
- }
-
- /**
- * Resolve path mode.
- *
- * @param path IGFS path.
- * @return Path mode.
- */
- public IgfsMode mode(IgfsPath path) {
- return modeRslvr.resolveMode(path);
- }
-
- /**
- * Convert the given path to path acceptable by the primary file system.
- *
- * @param path Path.
- * @return Primary file system path.
- */
- private Path toPrimary(Path path) {
- return convertPath(path, uri);
- }
-
- /**
- * Convert the given path to path acceptable by the secondary file system.
- *
- * @param path Path.
- * @return Secondary file system path.
- */
- private Path toSecondary(Path path) {
- assert secondaryFs != null;
- assert secondaryUri != null;
-
- return convertPath(path, secondaryUri);
- }
-
- /**
- * Convert path using the given new URI.
- *
- * @param path Old path.
- * @param newUri New URI.
- * @return New path.
- */
- private Path convertPath(Path path, URI newUri) {
- assert newUri != null;
-
- if (path != null) {
- URI pathUri = path.toUri();
-
- try {
- return new Path(new URI(pathUri.getScheme() != null ? newUri.getScheme() : null,
- pathUri.getAuthority() != null ? newUri.getAuthority() : null, pathUri.getPath(), null, null));
- }
- catch (URISyntaxException e) {
- throw new IgniteException("Failed to construct secondary file system path from the primary file " +
- "system path: " + path, e);
- }
- }
- else
- return null;
- }
-
- /**
- * Convert a file status obtained from the secondary file system to a status of the primary file system.
- *
- * @param status Secondary file system status.
- * @return Primary file system status.
- */
- @SuppressWarnings("deprecation")
- private FileStatus toPrimary(FileStatus status) {
- return status != null ? new FileStatus(status.getLen(), status.isDir(), status.getReplication(),
- status.getBlockSize(), status.getModificationTime(), status.getAccessTime(), status.getPermission(),
- status.getOwner(), status.getGroup(), toPrimary(status.getPath())) : null;
- }
-
- /**
- * Convert IGFS path into Hadoop path.
- *
- * @param path IGFS path.
- * @return Hadoop path.
- */
- private Path convert(IgfsPath path) {
- return new Path(IGFS_SCHEME, uriAuthority, path.toString());
- }
-
- /**
- * Convert Hadoop path into IGFS path.
- *
- * @param path Hadoop path.
- * @return IGFS path.
- */
- @Nullable private IgfsPath convert(@Nullable Path path) {
- if (path == null)
- return null;
-
- return path.isAbsolute() ? new IgfsPath(path.toUri().getPath()) :
- new IgfsPath(convert(workingDir.get()), path.toUri().getPath());
- }
-
- /**
- * Convert IGFS affinity block location into Hadoop affinity block location.
- *
- * @param block IGFS affinity block location.
- * @return Hadoop affinity block location.
- */
- private BlockLocation convert(IgfsBlockLocation block) {
- Collection<String> names = block.names();
- Collection<String> hosts = block.hosts();
-
- return new BlockLocation(
- names.toArray(new String[names.size()]) /* hostname:portNumber of data nodes */,
- hosts.toArray(new String[hosts.size()]) /* hostnames of data nodes */,
- block.start(), block.length()
- ) {
- @Override public String toString() {
- try {
- return "BlockLocation [offset=" + getOffset() + ", length=" + getLength() +
- ", hosts=" + Arrays.asList(getHosts()) + ", names=" + Arrays.asList(getNames()) + ']';
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- };
- }
-
- /**
- * Convert IGFS file information into Hadoop file status.
- *
- * @param file IGFS file information.
- * @return Hadoop file status.
- */
- @SuppressWarnings("deprecation")
- private FileStatus convert(IgfsFile file) {
- return new FileStatus(file.length(), file.isDirectory(), getDefaultReplication(),
- file.groupBlockSize(), file.modificationTime(), file.accessTime(), permission(file),
- file.property(PROP_USER_NAME, DFLT_USER_NAME), file.property(PROP_GROUP_NAME, "users"),
- convert(file.path())) {
- @Override public String toString() {
- return "FileStatus [path=" + getPath() + ", isDir=" + isDir() + ", len=" + getLen() +
- ", mtime=" + getModificationTime() + ", atime=" + getAccessTime() + ']';
- }
- };
- }
-
- /**
- * Convert Hadoop permission into IGFS file attribute.
- *
- * @param perm Hadoop permission.
- * @return IGFS attributes.
- */
- private Map<String, String> permission(FsPermission perm) {
- if (perm == null)
- perm = FsPermission.getDefault();
-
- return F.asMap(PROP_PERMISSION, toString(perm));
- }
-
- /**
- * @param perm Permission.
- * @return String.
- */
- private static String toString(FsPermission perm) {
- return String.format("%04o", perm.toShort());
- }
-
- /**
- * Convert IGFS file attributes into Hadoop permission.
- *
- * @param file File info.
- * @return Hadoop permission.
- */
- private FsPermission permission(IgfsFile file) {
- String perm = file.property(PROP_PERMISSION, null);
-
- if (perm == null)
- return FsPermission.getDefault();
-
- try {
- return new FsPermission((short)Integer.parseInt(perm, 8));
- }
- catch (NumberFormatException ignore) {
- return FsPermission.getDefault();
- }
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(IgfsHadoopFileSystem.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/package.html
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/package.html b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/package.html
deleted file mode 100644
index 4b62db1..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/package.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<body>
- <!-- Package description. -->
- Contains Hadoop 1.x <code>FileSystem</code> wrapper for Ignite file system.
-</body>
-</html>
[36/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java
deleted file mode 100644
index 0759203..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java
+++ /dev/null
@@ -1,982 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs.hadoop.v2;
-
-import org.apache.commons.logging.*;
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.permission.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.util.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.igfs.common.*;
-import org.apache.ignite.internal.igfs.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-import java.util.concurrent.atomic.*;
-
-import static org.apache.ignite.IgniteFs.*;
-import static org.apache.ignite.configuration.IgfsConfiguration.*;
-import static org.apache.ignite.igfs.IgfsMode.*;
-import static org.apache.ignite.igfs.hadoop.IgfsHadoopParameters.*;
-import static org.apache.ignite.internal.igfs.hadoop.IgfsHadoopUtils.*;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
-
-/**
- * {@code IGFS} Hadoop 2.x file system driver over file system API. To use
- * {@code IGFS} as Hadoop file system, you should configure this class
- * in Hadoop's {@code core-site.xml} as follows:
- * <pre name="code" class="xml">
- * <property>
- * <name>fs.default.name</name>
- * <value>igfs://ipc</value>
- * </property>
- *
- * <property>
- * <name>fs.igfs.impl</name>
- * <value>org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem</value>
- * </property>
- * </pre>
- * You should also add Ignite JAR and all libraries to Hadoop classpath. To
- * do this, add following lines to {@code conf/hadoop-env.sh} script in Hadoop
- * distribution:
- * <pre name="code" class="bash">
- * export IGNITE_HOME=/path/to/Ignite/distribution
- * export HADOOP_CLASSPATH=$IGNITE_HOME/ignite*.jar
- *
- * for f in $IGNITE_HOME/libs/*.jar; do
- * export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f;
- * done
- * </pre>
- * <h1 class="header">Data vs Clients Nodes</h1>
- * Hadoop needs to use its FileSystem remotely from client nodes as well as directly on
- * data nodes. Client nodes are responsible for basic file system operations as well as
- * accessing data nodes remotely. Usually, client nodes are started together
- * with {@code job-submitter} or {@code job-scheduler} processes, while data nodes are usually
- * started together with Hadoop {@code task-tracker} processes.
- * <p>
- * For sample client and data node configuration refer to {@code config/hadoop/default-config-client.xml}
- * and {@code config/hadoop/default-config.xml} configuration files in Ignite installation.
- */
-public class IgfsHadoopFileSystem extends AbstractFileSystem implements Closeable {
- /** Logger. */
- private static final Log LOG = LogFactory.getLog(IgfsHadoopFileSystem.class);
-
- /** Ensures that close routine is invoked at most once. */
- private final AtomicBoolean closeGuard = new AtomicBoolean();
-
- /** Grid remote client. */
- private IgfsHadoopWrapper rmtClient;
-
- /** Working directory. */
- private IgfsPath workingDir;
-
- /** URI. */
- private URI uri;
-
- /** Authority. */
- private String uriAuthority;
-
- /** Client logger. */
- private IgfsLogger clientLog;
-
- /** Server block size. */
- private long grpBlockSize;
-
- /** Default replication factor. */
- private short dfltReplication;
-
- /** Secondary URI string. */
- private URI secondaryUri;
-
- /** Mode resolver. */
- private IgfsModeResolver modeRslvr;
-
- /** Secondary file system instance. */
- private AbstractFileSystem secondaryFs;
-
- /** Whether custom sequential reads before prefetch value is provided. */
- private boolean seqReadsBeforePrefetchOverride;
-
- /** Custom-provided sequential reads before prefetch. */
- private int seqReadsBeforePrefetch;
-
- /** Flag that controls whether file writes should be colocated on data node. */
- private boolean colocateFileWrites;
-
- /** Prefer local writes. */
- private boolean preferLocFileWrites;
-
- /**
- * @param name URI for file system.
- * @param cfg Configuration.
- * @throws URISyntaxException if name has invalid syntax.
- * @throws IOException If initialization failed.
- */
- public IgfsHadoopFileSystem(URI name, Configuration cfg) throws URISyntaxException, IOException {
- super(IgfsHadoopEndpoint.normalize(name), IGFS_SCHEME, false, -1);
-
- uri = name;
-
- try {
- initialize(name, cfg);
- }
- catch (IOException e) {
- // Close client if exception occurred.
- if (rmtClient != null)
- rmtClient.close(false);
-
- throw e;
- }
-
- workingDir = new IgfsPath("/user/" + cfg.get(MRJobConfig.USER_NAME, DFLT_USER_NAME));
- }
-
- /** {@inheritDoc} */
- @Override public void checkPath(Path path) {
- URI uri = path.toUri();
-
- if (uri.isAbsolute()) {
- if (!F.eq(uri.getScheme(), IGFS_SCHEME))
- throw new InvalidPathException("Wrong path scheme [expected=" + IGFS_SCHEME + ", actual=" +
- uri.getAuthority() + ']');
-
- if (!F.eq(uri.getAuthority(), uriAuthority))
- throw new InvalidPathException("Wrong path authority [expected=" + uriAuthority + ", actual=" +
- uri.getAuthority() + ']');
- }
- }
-
- /**
- * Public setter that can be used by direct users of FS or Visor.
- *
- * @param colocateFileWrites Whether all ongoing file writes should be colocated.
- */
- @SuppressWarnings("UnusedDeclaration")
- public void colocateFileWrites(boolean colocateFileWrites) {
- this.colocateFileWrites = colocateFileWrites;
- }
-
- /**
- * Enter busy state.
- *
- * @throws IOException If file system is stopped.
- */
- private void enterBusy() throws IOException {
- if (closeGuard.get())
- throw new IOException("File system is stopped.");
- }
-
- /**
- * Leave busy state.
- */
- private void leaveBusy() {
- // No-op.
- }
-
- /**
- * @param name URI passed to constructor.
- * @param cfg Configuration passed to constructor.
- * @throws IOException If initialization failed.
- */
- private void initialize(URI name, Configuration cfg) throws IOException {
- enterBusy();
-
- try {
- if (rmtClient != null)
- throw new IOException("File system is already initialized: " + rmtClient);
-
- A.notNull(name, "name");
- A.notNull(cfg, "cfg");
-
- if (!IGFS_SCHEME.equals(name.getScheme()))
- throw new IOException("Illegal file system URI [expected=" + IGFS_SCHEME +
- "://[name]/[optional_path], actual=" + name + ']');
-
- uriAuthority = name.getAuthority();
-
- // Override sequential reads before prefetch if needed.
- seqReadsBeforePrefetch = parameter(cfg, PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH, uriAuthority, 0);
-
- if (seqReadsBeforePrefetch > 0)
- seqReadsBeforePrefetchOverride = true;
-
- // In Ignite replication factor is controlled by data cache affinity.
- // We use replication factor to force the whole file to be stored on local node.
- dfltReplication = (short)cfg.getInt("dfs.replication", 3);
-
- // Get file colocation control flag.
- colocateFileWrites = parameter(cfg, PARAM_IGFS_COLOCATED_WRITES, uriAuthority, false);
- preferLocFileWrites = cfg.getBoolean(PARAM_IGFS_PREFER_LOCAL_WRITES, false);
-
- // Get log directory.
- String logDirCfg = parameter(cfg, PARAM_IGFS_LOG_DIR, uriAuthority, DFLT_IGFS_LOG_DIR);
-
- File logDirFile = U.resolveIgnitePath(logDirCfg);
-
- String logDir = logDirFile != null ? logDirFile.getAbsolutePath() : null;
-
- rmtClient = new IgfsHadoopWrapper(uriAuthority, logDir, cfg, LOG);
-
- // Handshake.
- IgfsHandshakeResponse handshake = rmtClient.handshake(logDir);
-
- grpBlockSize = handshake.blockSize();
-
- IgfsPaths paths = handshake.secondaryPaths();
-
- Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);
-
- if (handshake.sampling() != null ? handshake.sampling() : logEnabled) {
- // Initiate client logger.
- if (logDir == null)
- throw new IOException("Failed to resolve log directory: " + logDirCfg);
-
- Integer batchSize = parameter(cfg, PARAM_IGFS_LOG_BATCH_SIZE, uriAuthority, DFLT_IGFS_LOG_BATCH_SIZE);
-
- clientLog = IgfsLogger.logger(uriAuthority, handshake.igfsName(), logDir, batchSize);
- }
- else
- clientLog = IgfsLogger.disabledLogger();
-
- modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes());
-
- boolean initSecondary = paths.defaultMode() == PROXY;
-
- if (!initSecondary && paths.pathModes() != null) {
- for (T2<IgfsPath, IgfsMode> pathMode : paths.pathModes()) {
- IgfsMode mode = pathMode.getValue();
-
- if (mode == PROXY) {
- initSecondary = true;
-
- break;
- }
- }
- }
-
- if (initSecondary) {
- Map<String, String> props = paths.properties();
-
- String secUri = props.get(SECONDARY_FS_URI);
- String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
-
- try {
- SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
-
- secondaryFs = secProvider.createAbstractFileSystem();
- secondaryUri = secProvider.uri();
- }
- catch (IOException e) {
- throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
- }
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IOException {
- if (closeGuard.compareAndSet(false, true)) {
- if (rmtClient == null)
- return;
-
- rmtClient.close(false);
-
- if (clientLog.isLogEnabled())
- clientLog.close();
-
- // Reset initialized resources.
- rmtClient = null;
- }
- }
-
- /** {@inheritDoc} */
- @Override public URI getUri() {
- return uri;
- }
-
- /** {@inheritDoc} */
- @Override public int getUriDefaultPort() {
- return -1;
- }
-
- /** {@inheritDoc} */
- @Override public FsServerDefaults getServerDefaults() throws IOException {
- return new FsServerDefaults(grpBlockSize, (int)grpBlockSize, (int)grpBlockSize, dfltReplication, 64 * 1024,
- false, 0, DataChecksum.Type.NULL);
- }
-
- /** {@inheritDoc} */
- @Override public boolean setReplication(Path f, short replication) throws IOException {
- return mode(f) == PROXY && secondaryFs.setReplication(f, replication);
- }
-
- /** {@inheritDoc} */
- @Override public void setTimes(Path f, long mtime, long atime) throws IOException {
- if (mode(f) == PROXY)
- secondaryFs.setTimes(f, mtime, atime);
- else {
- if (mtime == -1 && atime == -1)
- return;
-
- rmtClient.setTimes(convert(f), atime, mtime);
- }
- }
-
- /** {@inheritDoc} */
- @Override public FsStatus getFsStatus() throws IOException {
- IgfsStatus status = rmtClient.fsStatus();
-
- return new FsStatus(status.spaceTotal(), status.spaceUsed(), status.spaceTotal() - status.spaceUsed());
- }
-
- /** {@inheritDoc} */
- @Override public void setPermission(Path p, FsPermission perm) throws IOException {
- enterBusy();
-
- try {
- A.notNull(p, "p");
-
- if (mode(p) == PROXY)
- secondaryFs.setPermission(toSecondary(p), perm);
- else {
- if (rmtClient.update(convert(p), permission(perm)) == null)
- throw new IOException("Failed to set file permission (file not found?)" +
- " [path=" + p + ", perm=" + perm + ']');
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void setOwner(Path p, String usr, String grp) throws IOException {
- A.notNull(p, "p");
- A.notNull(usr, "username");
- A.notNull(grp, "grpName");
-
- enterBusy();
-
- try {
- if (mode(p) == PROXY)
- secondaryFs.setOwner(toSecondary(p), usr, grp);
- else if (rmtClient.update(convert(p), F.asMap(PROP_USER_NAME, usr, PROP_GROUP_NAME, grp)) == null)
- throw new IOException("Failed to set file permission (file not found?)" +
- " [path=" + p + ", username=" + usr + ", grpName=" + grp + ']');
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public FSDataInputStream open(Path f, int bufSize) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- try {
- IgfsPath path = convert(f);
- IgfsMode mode = modeRslvr.resolveMode(path);
-
- if (mode == PROXY) {
- FSDataInputStream is = secondaryFs.open(toSecondary(f), bufSize);
-
- if (clientLog.isLogEnabled()) {
- // At this point we do not know file size, so we perform additional request to remote FS to get it.
- FileStatus status = secondaryFs.getFileStatus(toSecondary(f));
-
- long size = status != null ? status.getLen() : -1;
-
- long logId = IgfsLogger.nextId();
-
- clientLog.logOpen(logId, path, PROXY, bufSize, size);
-
- return new FSDataInputStream(new IgfsHadoopProxyInputStream(is, clientLog, logId));
- }
- else
- return is;
- }
- else {
- IgfsHadoopStreamDelegate stream = seqReadsBeforePrefetchOverride ?
- rmtClient.open(path, seqReadsBeforePrefetch) : rmtClient.open(path);
-
- long logId = -1;
-
- if (clientLog.isLogEnabled()) {
- logId = IgfsLogger.nextId();
-
- clientLog.logOpen(logId, path, mode, bufSize, stream.length());
- }
-
- if (LOG.isDebugEnabled())
- LOG.debug("Opening input stream [thread=" + Thread.currentThread().getName() + ", path=" + path +
- ", bufSize=" + bufSize + ']');
-
- IgfsHadoopInputStream igfsIn = new IgfsHadoopInputStream(stream, stream.length(),
- bufSize, LOG, clientLog, logId);
-
- if (LOG.isDebugEnabled())
- LOG.debug("Opened input stream [path=" + path + ", delegate=" + stream + ']');
-
- return new FSDataInputStream(igfsIn);
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("deprecation")
- @Override public FSDataOutputStream createInternal(
- Path f,
- EnumSet<CreateFlag> flag,
- FsPermission perm,
- int bufSize,
- short replication,
- long blockSize,
- Progressable progress,
- Options.ChecksumOpt checksumOpt,
- boolean createParent
- ) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
- boolean append = flag.contains(CreateFlag.APPEND);
- boolean create = flag.contains(CreateFlag.CREATE);
-
- OutputStream out = null;
-
- try {
- IgfsPath path = convert(f);
- IgfsMode mode = modeRslvr.resolveMode(path);
-
- if (LOG.isDebugEnabled())
- LOG.debug("Opening output stream in create [thread=" + Thread.currentThread().getName() + "path=" +
- path + ", overwrite=" + overwrite + ", bufSize=" + bufSize + ']');
-
- if (mode == PROXY) {
- FSDataOutputStream os = secondaryFs.createInternal(toSecondary(f), flag, perm, bufSize,
- replication, blockSize, progress, checksumOpt, createParent);
-
- if (clientLog.isLogEnabled()) {
- long logId = IgfsLogger.nextId();
-
- if (append)
- clientLog.logAppend(logId, path, PROXY, bufSize); // Don't have stream ID.
- else
- clientLog.logCreate(logId, path, PROXY, overwrite, bufSize, replication, blockSize);
-
- return new FSDataOutputStream(new IgfsHadoopProxyOutputStream(os, clientLog, logId));
- }
- else
- return os;
- }
- else {
- Map<String, String> permMap = F.asMap(PROP_PERMISSION, toString(perm),
- PROP_PREFER_LOCAL_WRITES, Boolean.toString(preferLocFileWrites));
-
- // Create stream and close it in the 'finally' section if any sequential operation failed.
- IgfsHadoopStreamDelegate stream;
-
- long logId = -1;
-
- if (append) {
- stream = rmtClient.append(path, create, permMap);
-
- if (clientLog.isLogEnabled()) {
- logId = IgfsLogger.nextId();
-
- clientLog.logAppend(logId, path, mode, bufSize);
- }
-
- if (LOG.isDebugEnabled())
- LOG.debug("Opened output stream in append [path=" + path + ", delegate=" + stream + ']');
- }
- else {
- stream = rmtClient.create(path, overwrite, colocateFileWrites, replication, blockSize,
- permMap);
-
- if (clientLog.isLogEnabled()) {
- logId = IgfsLogger.nextId();
-
- clientLog.logCreate(logId, path, mode, overwrite, bufSize, replication, blockSize);
- }
-
- if (LOG.isDebugEnabled())
- LOG.debug("Opened output stream in create [path=" + path + ", delegate=" + stream + ']');
- }
-
- assert stream != null;
-
- IgfsHadoopOutputStream igfsOut = new IgfsHadoopOutputStream(stream, LOG,
- clientLog, logId);
-
- bufSize = Math.max(64 * 1024, bufSize);
-
- out = new BufferedOutputStream(igfsOut, bufSize);
-
- FSDataOutputStream res = new FSDataOutputStream(out, null, 0);
-
- // Mark stream created successfully.
- out = null;
-
- return res;
- }
- }
- finally {
- // Close if failed during stream creation.
- if (out != null)
- U.closeQuiet(out);
-
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public boolean supportsSymlinks() {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public void renameInternal(Path src, Path dst) throws IOException {
- A.notNull(src, "src");
- A.notNull(dst, "dst");
-
- enterBusy();
-
- try {
- IgfsPath srcPath = convert(src);
- IgfsPath dstPath = convert(dst);
- Set<IgfsMode> childrenModes = modeRslvr.resolveChildrenModes(srcPath);
-
- if (childrenModes.contains(PROXY)) {
- if (clientLog.isLogEnabled())
- clientLog.logRename(srcPath, PROXY, dstPath);
-
- secondaryFs.renameInternal(toSecondary(src), toSecondary(dst));
- }
-
- rmtClient.rename(srcPath, dstPath);
-
- if (clientLog.isLogEnabled())
- clientLog.logRename(srcPath, modeRslvr.resolveMode(srcPath), dstPath);
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public boolean delete(Path f, boolean recursive) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- try {
- IgfsPath path = convert(f);
- IgfsMode mode = modeRslvr.resolveMode(path);
- Set<IgfsMode> childrenModes = modeRslvr.resolveChildrenModes(path);
-
- if (childrenModes.contains(PROXY)) {
- if (clientLog.isLogEnabled())
- clientLog.logDelete(path, PROXY, recursive);
-
- return secondaryFs.delete(toSecondary(f), recursive);
- }
-
- boolean res = rmtClient.delete(path, recursive);
-
- if (clientLog.isLogEnabled())
- clientLog.logDelete(path, mode, recursive);
-
- return res;
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void setVerifyChecksum(boolean verifyChecksum) throws IOException {
- // Checksum has effect for secondary FS only.
- if (secondaryFs != null)
- secondaryFs.setVerifyChecksum(verifyChecksum);
- }
-
- /** {@inheritDoc} */
- @Override public FileChecksum getFileChecksum(Path f) throws IOException {
- if (mode(f) == PROXY)
- return secondaryFs.getFileChecksum(f);
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public FileStatus[] listStatus(Path f) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- try {
- IgfsPath path = convert(f);
- IgfsMode mode = modeRslvr.resolveMode(path);
-
- if (mode == PROXY) {
- FileStatus[] arr = secondaryFs.listStatus(toSecondary(f));
-
- if (arr == null)
- throw new FileNotFoundException("File " + f + " does not exist.");
-
- for (int i = 0; i < arr.length; i++)
- arr[i] = toPrimary(arr[i]);
-
- if (clientLog.isLogEnabled()) {
- String[] fileArr = new String[arr.length];
-
- for (int i = 0; i < arr.length; i++)
- fileArr[i] = arr[i].getPath().toString();
-
- clientLog.logListDirectory(path, PROXY, fileArr);
- }
-
- return arr;
- }
- else {
- Collection<IgfsFile> list = rmtClient.listFiles(path);
-
- if (list == null)
- throw new FileNotFoundException("File " + f + " does not exist.");
-
- List<IgfsFile> files = new ArrayList<>(list);
-
- FileStatus[] arr = new FileStatus[files.size()];
-
- for (int i = 0; i < arr.length; i++)
- arr[i] = convert(files.get(i));
-
- if (clientLog.isLogEnabled()) {
- String[] fileArr = new String[arr.length];
-
- for (int i = 0; i < arr.length; i++)
- fileArr[i] = arr[i].getPath().toString();
-
- clientLog.logListDirectory(path, mode, fileArr);
- }
-
- return arr;
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void mkdir(Path f, FsPermission perm, boolean createParent) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- try {
- IgfsPath path = convert(f);
- IgfsMode mode = modeRslvr.resolveMode(path);
-
- if (mode == PROXY) {
- if (clientLog.isLogEnabled())
- clientLog.logMakeDirectory(path, PROXY);
-
- secondaryFs.mkdir(toSecondary(f), perm, createParent);
- }
- else {
- rmtClient.mkdirs(path, permission(perm));
-
- if (clientLog.isLogEnabled())
- clientLog.logMakeDirectory(path, mode);
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public FileStatus getFileStatus(Path f) throws IOException {
- A.notNull(f, "f");
-
- enterBusy();
-
- try {
- if (mode(f) == PROXY)
- return toPrimary(secondaryFs.getFileStatus(toSecondary(f)));
- else {
- IgfsFile info = rmtClient.info(convert(f));
-
- if (info == null)
- throw new FileNotFoundException("File not found: " + f);
-
- return convert(info);
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /** {@inheritDoc} */
- @Override public BlockLocation[] getFileBlockLocations(Path path, long start, long len) throws IOException {
- A.notNull(path, "path");
-
- IgfsPath igfsPath = convert(path);
-
- enterBusy();
-
- try {
- if (modeRslvr.resolveMode(igfsPath) == PROXY)
- return secondaryFs.getFileBlockLocations(path, start, len);
- else {
- long now = System.currentTimeMillis();
-
- List<IgfsBlockLocation> affinity = new ArrayList<>(
- rmtClient.affinity(igfsPath, start, len));
-
- BlockLocation[] arr = new BlockLocation[affinity.size()];
-
- for (int i = 0; i < arr.length; i++)
- arr[i] = convert(affinity.get(i));
-
- if (LOG.isDebugEnabled())
- LOG.debug("Fetched file locations [path=" + path + ", fetchTime=" +
- (System.currentTimeMillis() - now) + ", locations=" + Arrays.asList(arr) + ']');
-
- return arr;
- }
- }
- finally {
- leaveBusy();
- }
- }
-
- /**
- * Resolve path mode.
- *
- * @param path HDFS path.
- * @return Path mode.
- */
- public IgfsMode mode(Path path) {
- return modeRslvr.resolveMode(convert(path));
- }
-
- /**
- * Convert the given path to path acceptable by the primary file system.
- *
- * @param path Path.
- * @return Primary file system path.
- */
- private Path toPrimary(Path path) {
- return convertPath(path, getUri());
- }
-
- /**
- * Convert the given path to path acceptable by the secondary file system.
- *
- * @param path Path.
- * @return Secondary file system path.
- */
- private Path toSecondary(Path path) {
- assert secondaryFs != null;
- assert secondaryUri != null;
-
- return convertPath(path, secondaryUri);
- }
-
- /**
- * Convert path using the given new URI.
- *
- * @param path Old path.
- * @param newUri New URI.
- * @return New path.
- */
- private Path convertPath(Path path, URI newUri) {
- assert newUri != null;
-
- if (path != null) {
- URI pathUri = path.toUri();
-
- try {
- return new Path(new URI(pathUri.getScheme() != null ? newUri.getScheme() : null,
- pathUri.getAuthority() != null ? newUri.getAuthority() : null, pathUri.getPath(), null, null));
- }
- catch (URISyntaxException e) {
- throw new IgniteException("Failed to construct secondary file system path from the primary file " +
- "system path: " + path, e);
- }
- }
- else
- return null;
- }
-
- /**
- * Convert a file status obtained from the secondary file system to a status of the primary file system.
- *
- * @param status Secondary file system status.
- * @return Primary file system status.
- */
- private FileStatus toPrimary(FileStatus status) {
- return status != null ? new FileStatus(status.getLen(), status.isDirectory(), status.getReplication(),
- status.getBlockSize(), status.getModificationTime(), status.getAccessTime(), status.getPermission(),
- status.getOwner(), status.getGroup(), toPrimary(status.getPath())) : null;
- }
-
- /**
- * Convert IGFS path into Hadoop path.
- *
- * @param path IGFS path.
- * @return Hadoop path.
- */
- private Path convert(IgfsPath path) {
- return new Path(IGFS_SCHEME, uriAuthority, path.toString());
- }
-
- /**
- * Convert Hadoop path into IGFS path.
- *
- * @param path Hadoop path.
- * @return IGFS path.
- */
- @Nullable private IgfsPath convert(Path path) {
- if (path == null)
- return null;
-
- return path.isAbsolute() ? new IgfsPath(path.toUri().getPath()) :
- new IgfsPath(workingDir, path.toUri().getPath());
- }
-
- /**
- * Convert IGFS affinity block location into Hadoop affinity block location.
- *
- * @param block IGFS affinity block location.
- * @return Hadoop affinity block location.
- */
- private BlockLocation convert(IgfsBlockLocation block) {
- Collection<String> names = block.names();
- Collection<String> hosts = block.hosts();
-
- return new BlockLocation(
- names.toArray(new String[names.size()]) /* hostname:portNumber of data nodes */,
- hosts.toArray(new String[hosts.size()]) /* hostnames of data nodes */,
- block.start(), block.length()
- ) {
- @Override public String toString() {
- try {
- return "BlockLocation [offset=" + getOffset() + ", length=" + getLength() +
- ", hosts=" + Arrays.asList(getHosts()) + ", names=" + Arrays.asList(getNames()) + ']';
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- };
- }
-
- /**
- * Convert IGFS file information into Hadoop file status.
- *
- * @param file IGFS file information.
- * @return Hadoop file status.
- */
- private FileStatus convert(IgfsFile file) {
- return new FileStatus(
- file.length(),
- file.isDirectory(),
- dfltReplication,
- file.groupBlockSize(),
- file.modificationTime(),
- file.accessTime(),
- permission(file),
- file.property(PROP_USER_NAME, DFLT_USER_NAME),
- file.property(PROP_GROUP_NAME, "users"),
- convert(file.path())) {
- @Override public String toString() {
- return "FileStatus [path=" + getPath() + ", isDir=" + isDirectory() + ", len=" + getLen() + "]";
- }
- };
- }
-
- /**
- * Convert Hadoop permission into IGFS file attribute.
- *
- * @param perm Hadoop permission.
- * @return IGFS attributes.
- */
- private Map<String, String> permission(FsPermission perm) {
- if (perm == null)
- perm = FsPermission.getDefault();
-
- return F.asMap(PROP_PERMISSION, toString(perm));
- }
-
- /**
- * @param perm Permission.
- * @return String.
- */
- private static String toString(FsPermission perm) {
- return String.format("%04o", perm.toShort());
- }
-
- /**
- * Convert IGFS file attributes into Hadoop permission.
- *
- * @param file File info.
- * @return Hadoop permission.
- */
- private FsPermission permission(IgfsFile file) {
- String perm = file.property(PROP_PERMISSION, null);
-
- if (perm == null)
- return FsPermission.getDefault();
-
- try {
- return new FsPermission((short)Integer.parseInt(perm, 8));
- }
- catch (NumberFormatException ignore) {
- return FsPermission.getDefault();
- }
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(IgfsHadoopFileSystem.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/package.html
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/package.html b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/package.html
deleted file mode 100644
index 6df66f4..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/package.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<body>
- <!-- Package description. -->
- Contains Hadoop 2.x <code>FileSystem</code> wrapper for Ignite file system.
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/igfs/package.html
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/package.html b/modules/hadoop/src/main/java/org/apache/ignite/igfs/package.html
deleted file mode 100644
index ec38a21..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/package.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<body>
- <!-- Package description. -->
- Contains <b>IG</b>nite <b>F</b>ile <b>S</b>ystem APIs.
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoop.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoop.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoop.java
deleted file mode 100644
index 27d6e33..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoop.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Facade for communication with grid.
- */
-public interface IgfsHadoop {
- /**
- * Perform handshake.
- *
- * @param logDir Log directory.
- * @return Future with handshake result.
- * @throws IgniteCheckedException If failed.
- */
- public IgfsHandshakeResponse handshake(String logDir) throws IgniteCheckedException, IOException;
-
- /**
- * Close connection.
- *
- * @param force Force flag.
- */
- public void close(boolean force);
-
- /**
- * Command to retrieve file info for some IGFS path.
- *
- * @param path Path to get file info for.
- * @return Future for info operation.
- * @throws IgniteCheckedException If failed.
- */
- public IgfsFile info(IgfsPath path) throws IgniteCheckedException, IOException;
-
- /**
- * Command to update file properties.
- *
- * @param path IGFS path to update properties.
- * @param props Properties to update.
- * @return Future for update operation.
- * @throws IgniteCheckedException If failed.
- */
- public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteCheckedException, IOException;
-
- /**
- * Sets last access time and last modification time for a file.
- *
- * @param path Path to update times.
- * @param accessTime Last access time to set.
- * @param modificationTime Last modification time to set.
- * @throws IgniteCheckedException If failed.
- */
- public Boolean setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteCheckedException,
- IOException;
-
- /**
- * Command to rename given path.
- *
- * @param src Source path.
- * @param dest Destination path.
- * @return Future for rename operation.
- * @throws IgniteCheckedException If failed.
- */
- public Boolean rename(IgfsPath src, IgfsPath dest) throws IgniteCheckedException, IOException;
-
- /**
- * Command to delete given path.
- *
- * @param path Path to delete.
- * @param recursive {@code True} if deletion is recursive.
- * @return Future for delete operation.
- * @throws IgniteCheckedException If failed.
- */
- public Boolean delete(IgfsPath path, boolean recursive) throws IgniteCheckedException, IOException;
-
- /**
- * Command to get affinity for given path, offset and length.
- *
- * @param path Path to get affinity for.
- * @param start Start position (offset).
- * @param len Data length.
- * @return Future for affinity command.
- * @throws IgniteCheckedException If failed.
- */
- public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) throws IgniteCheckedException,
- IOException;
-
- /**
- * Gets path summary.
- *
- * @param path Path to get summary for.
- * @return Future that will be completed when summary is received.
- * @throws IgniteCheckedException If failed.
- */
- public IgfsPathSummary contentSummary(IgfsPath path) throws IgniteCheckedException, IOException;
-
- /**
- * Command to create directories.
- *
- * @param path Path to create.
- * @return Future for mkdirs operation.
- * @throws IgniteCheckedException If failed.
- */
- public Boolean mkdirs(IgfsPath path, Map<String, String> props) throws IgniteCheckedException, IOException;
-
- /**
- * Command to get list of files in directory.
- *
- * @param path Path to list.
- * @return Future for listFiles operation.
- * @throws IgniteCheckedException If failed.
- */
- public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteCheckedException, IOException;
-
- /**
- * Command to get directory listing.
- *
- * @param path Path to list.
- * @return Future for listPaths operation.
- * @throws IgniteCheckedException If failed.
- */
- public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteCheckedException, IOException;
-
- /**
- * Performs status request.
- *
- * @return Status response.
- * @throws IgniteCheckedException If failed.
- */
- public IgfsStatus fsStatus() throws IgniteCheckedException, IOException;
-
- /**
- * Command to open file for reading.
- *
- * @param path File path to open.
- * @return Future for open operation.
- * @throws IgniteCheckedException If failed.
- */
- public IgfsHadoopStreamDelegate open(IgfsPath path) throws IgniteCheckedException, IOException;
-
- /**
- * Command to open file for reading.
- *
- * @param path File path to open.
- * @return Future for open operation.
- * @throws IgniteCheckedException If failed.
- */
- public IgfsHadoopStreamDelegate open(IgfsPath path, int seqReadsBeforePrefetch) throws IgniteCheckedException,
- IOException;
-
- /**
- * Command to create file and open it for output.
- *
- * @param path Path to file.
- * @param overwrite If {@code true} then old file contents will be lost.
- * @param colocate If {@code true} and called on data node, file will be written on that node.
- * @param replication Replication factor.
- * @param props File properties for creation.
- * @return Stream descriptor.
- * @throws IgniteCheckedException If failed.
- */
- public IgfsHadoopStreamDelegate create(IgfsPath path, boolean overwrite, boolean colocate,
- int replication, long blockSize, @Nullable Map<String, String> props) throws IgniteCheckedException, IOException;
-
- /**
- * Open file for output appending data to the end of a file.
- *
- * @param path Path to file.
- * @param create If {@code true}, file will be created if does not exist.
- * @param props File properties.
- * @return Stream descriptor.
- * @throws IgniteCheckedException If failed.
- */
- public IgfsHadoopStreamDelegate append(IgfsPath path, boolean create,
- @Nullable Map<String, String> props) throws IgniteCheckedException, IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopCommunicationException.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopCommunicationException.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopCommunicationException.java
deleted file mode 100644
index 03bf733..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopCommunicationException.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.ignite.*;
-
-/**
- * Communication exception indicating a problem between file system and IGFS instance.
- */
-public class IgfsHadoopCommunicationException extends IgniteCheckedException {
- /** */
- private static final long serialVersionUID = 0L;
-
- /**
- * Creates new exception with given throwable as a nested cause and
- * source of error message.
- *
- * @param cause Non-null throwable cause.
- */
- public IgfsHadoopCommunicationException(Exception cause) {
- super(cause);
- }
-
- /**
- * Creates a new exception with given error message and optional nested cause exception.
- *
- * @param msg Error message.
- */
- public IgfsHadoopCommunicationException(String msg) {
- super(msg);
- }
-
- /**
- * Creates a new exception with given error message and optional nested cause exception.
- *
- * @param msg Error message.
- * @param cause Cause.
- */
- public IgfsHadoopCommunicationException(String msg, Exception cause) {
- super(msg, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopEndpoint.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopEndpoint.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopEndpoint.java
deleted file mode 100644
index 35638ea..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopEndpoint.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.lang.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.net.*;
-
-import static org.apache.ignite.configuration.IgfsConfiguration.*;
-
-/**
- * IGFS endpoint abstraction.
- */
-public class IgfsHadoopEndpoint {
- /** Localhost. */
- public static final String LOCALHOST = "127.0.0.1";
-
- /** IGFS name. */
- private final String igfsName;
-
- /** Grid name. */
- private final String gridName;
-
- /** Host. */
- private final String host;
-
- /** Port. */
- private final int port;
-
- /**
- * Normalize IGFS URI.
- *
- * @param uri URI.
- * @return Normalized URI.
- * @throws IOException If failed.
- */
- public static URI normalize(URI uri) throws IOException {
- try {
- if (!F.eq(IgniteFs.IGFS_SCHEME, uri.getScheme()))
- throw new IOException("Failed to normalize UIR because it has non IGFS scheme: " + uri);
-
- IgfsHadoopEndpoint endpoint = new IgfsHadoopEndpoint(uri.getAuthority());
-
- StringBuilder sb = new StringBuilder();
-
- if (endpoint.igfs() != null)
- sb.append(endpoint.igfs());
-
- if (endpoint.grid() != null)
- sb.append(":").append(endpoint.grid());
-
- return new URI(uri.getScheme(), sb.length() != 0 ? sb.toString() : null, endpoint.host(), endpoint.port(),
- uri.getPath(), uri.getQuery(), uri.getFragment());
- }
- catch (URISyntaxException | IgniteCheckedException e) {
- throw new IOException("Failed to normalize URI: " + uri, e);
- }
- }
-
- /**
- * Constructor.
- *
- * @param connStr Connection string.
- * @throws IgniteCheckedException If failed to parse connection string.
- */
- public IgfsHadoopEndpoint(@Nullable String connStr) throws IgniteCheckedException {
- if (connStr == null)
- connStr = "";
-
- String[] tokens = connStr.split("@", -1);
-
- IgniteBiTuple<String, Integer> hostPort;
-
- if (tokens.length == 1) {
- igfsName = null;
- gridName = null;
-
- hostPort = hostPort(connStr, connStr);
- }
- else if (tokens.length == 2) {
- String authStr = tokens[0];
-
- if (authStr.isEmpty()) {
- gridName = null;
- igfsName = null;
- }
- else {
- String[] authTokens = authStr.split(":", -1);
-
- igfsName = F.isEmpty(authTokens[0]) ? null : authTokens[0];
-
- if (authTokens.length == 1)
- gridName = null;
- else if (authTokens.length == 2)
- gridName = F.isEmpty(authTokens[1]) ? null : authTokens[1];
- else
- throw new IgniteCheckedException("Invalid connection string format: " + connStr);
- }
-
- hostPort = hostPort(connStr, tokens[1]);
- }
- else
- throw new IgniteCheckedException("Invalid connection string format: " + connStr);
-
- host = hostPort.get1();
-
- assert hostPort.get2() != null;
-
- port = hostPort.get2();
- }
-
- /**
- * Parse host and port.
- *
- * @param connStr Full connection string.
- * @param hostPortStr Host/port connection string part.
- * @return Tuple with host and port.
- * @throws IgniteCheckedException If failed to parse connection string.
- */
- private IgniteBiTuple<String, Integer> hostPort(String connStr, String hostPortStr) throws IgniteCheckedException {
- String[] tokens = hostPortStr.split(":", -1);
-
- String host = tokens[0];
-
- if (F.isEmpty(host))
- host = LOCALHOST;
-
- int port;
-
- if (tokens.length == 1)
- port = DFLT_IPC_PORT;
- else if (tokens.length == 2) {
- String portStr = tokens[1];
-
- try {
- port = Integer.valueOf(portStr);
-
- if (port < 0 || port > 65535)
- throw new IgniteCheckedException("Invalid port number: " + connStr);
- }
- catch (NumberFormatException e) {
- throw new IgniteCheckedException("Invalid port number: " + connStr);
- }
- }
- else
- throw new IgniteCheckedException("Invalid connection string format: " + connStr);
-
- return F.t(host, port);
- }
-
- /**
- * @return IGFS name.
- */
- @Nullable public String igfs() {
- return igfsName;
- }
-
- /**
- * @return Grid name.
- */
- @Nullable public String grid() {
- return gridName;
- }
-
- /**
- * @return Host.
- */
- public String host() {
- return host;
- }
-
- /**
- * @return Host.
- */
- public boolean isLocal() {
- return F.eq(LOCALHOST, host);
- }
-
- /**
- * @return Port.
- */
- public int port() {
- return port;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(IgfsHadoopEndpoint.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopEx.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopEx.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopEx.java
deleted file mode 100644
index da86e37..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopEx.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.ignite.internal.util.lang.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * Extended IGFS server interface.
- */
-public interface IgfsHadoopEx extends IgfsHadoop {
- /**
- * Adds event listener that will be invoked when connection with server is lost or remote error has occurred.
- * If connection is closed already, callback will be invoked synchronously inside this method.
- *
- * @param delegate Stream delegate.
- * @param lsnr Event listener.
- */
- public void addEventListener(IgfsHadoopStreamDelegate delegate, IgfsHadoopStreamEventListener lsnr);
-
- /**
- * Removes event listener that will be invoked when connection with server is lost or remote error has occurred.
- *
- * @param delegate Stream delegate.
- */
- public void removeEventListener(IgfsHadoopStreamDelegate delegate);
-
- /**
- * Asynchronously reads specified amount of bytes from opened input stream.
- *
- * @param delegate Stream delegate.
- * @param pos Position to read from.
- * @param len Data length to read.
- * @param outBuf Optional output buffer. If buffer length is less then {@code len}, all remaining
- * bytes will be read into new allocated buffer of length {len - outBuf.length} and this buffer will
- * be the result of read future.
- * @param outOff Output offset.
- * @param outLen Output length.
- * @return Read data.
- */
- public GridPlainFuture<byte[]> readData(IgfsHadoopStreamDelegate delegate, long pos, int len,
- @Nullable final byte[] outBuf, final int outOff, final int outLen);
-
- /**
- * Writes data to the stream with given streamId. This method does not return any future since
- * no response to write request is sent.
- *
- * @param delegate Stream delegate.
- * @param data Data to write.
- * @param off Offset.
- * @param len Length.
- * @throws IOException If failed.
- */
- public void writeData(IgfsHadoopStreamDelegate delegate, byte[] data, int off, int len) throws IOException;
-
- /**
- * Close server stream.
- *
- * @param delegate Stream delegate.
- * @throws IOException If failed.
- */
- public void closeStream(IgfsHadoopStreamDelegate delegate) throws IOException;
-
- /**
- * Flush output stream.
- *
- * @param delegate Stream delegate.
- * @throws IOException If failed.
- */
- public void flush(IgfsHadoopStreamDelegate delegate) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopFSProperties.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopFSProperties.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopFSProperties.java
deleted file mode 100644
index c9d1322..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopFSProperties.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.hadoop.fs.permission.*;
-import org.apache.ignite.*;
-
-import java.util.*;
-
-import static org.apache.ignite.IgniteFs.*;
-
-/**
- * Hadoop file system properties.
- */
-public class IgfsHadoopFSProperties {
- /** Username. */
- private String usrName;
-
- /** Group name. */
- private String grpName;
-
- /** Permissions. */
- private FsPermission perm;
-
- /**
- * Constructor.
- *
- * @param props Properties.
- * @throws IgniteException In case of error.
- */
- public IgfsHadoopFSProperties(Map<String, String> props) throws IgniteException {
- usrName = props.get(PROP_USER_NAME);
- grpName = props.get(PROP_GROUP_NAME);
-
- String permStr = props.get(PROP_PERMISSION);
-
- if (permStr != null) {
- try {
- perm = new FsPermission((short)Integer.parseInt(permStr, 8));
- }
- catch (NumberFormatException ignore) {
- throw new IgniteException("Permissions cannot be parsed: " + permStr);
- }
- }
- }
-
- /**
- * Get user name.
- *
- * @return User name.
- */
- public String userName() {
- return usrName;
- }
-
- /**
- * Get group name.
- *
- * @return Group name.
- */
- public String groupName() {
- return grpName;
- }
-
- /**
- * Get permission.
- *
- * @return Permission.
- */
- public FsPermission permission() {
- return perm;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopFuture.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopFuture.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopFuture.java
deleted file mode 100644
index 476641c..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopFuture.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.ignite.internal.util.lang.*;
-import org.jetbrains.annotations.*;
-
-/**
- * IGFS client future that holds response parse closure.
- */
-public class IgfsHadoopFuture<T> extends GridPlainFutureAdapter<T> {
- /** Output buffer. */
- private byte[] outBuf;
-
- /** Output offset. */
- private int outOff;
-
- /** Output length. */
- private int outLen;
-
- /** Read future flag. */
- private boolean read;
-
- /**
- * @return Output buffer.
- */
- public byte[] outputBuffer() {
- return outBuf;
- }
-
- /**
- * @param outBuf Output buffer.
- */
- public void outputBuffer(@Nullable byte[] outBuf) {
- this.outBuf = outBuf;
- }
-
- /**
- * @return Offset in output buffer to write from.
- */
- public int outputOffset() {
- return outOff;
- }
-
- /**
- * @param outOff Offset in output buffer to write from.
- */
- public void outputOffset(int outOff) {
- this.outOff = outOff;
- }
-
- /**
- * @return Length to write to output buffer.
- */
- public int outputLength() {
- return outLen;
- }
-
- /**
- * @param outLen Length to write to output buffer.
- */
- public void outputLength(int outLen) {
- this.outLen = outLen;
- }
-
- /**
- * @param read {@code True} if this is a read future.
- */
- public void read(boolean read) {
- this.read = read;
- }
-
- /**
- * @return {@code True} if this is a read future.
- */
- public boolean read() {
- return read;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopInProc.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopInProc.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopInProc.java
deleted file mode 100644
index 8245125..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/igfs/hadoop/IgfsHadoopInProc.java
+++ /dev/null
@@ -1,409 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.igfs.hadoop;
-
-import org.apache.commons.logging.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-/**
- * Communication with grid in the same process.
- */
-public class IgfsHadoopInProc implements IgfsHadoopEx {
- /** Target IGFS. */
- private final IgfsEx igfs;
-
- /** Buffer size. */
- private final int bufSize;
-
- /** Event listeners. */
- private final Map<IgfsHadoopStreamDelegate, IgfsHadoopStreamEventListener> lsnrs =
- new ConcurrentHashMap<>();
-
- /** Logger. */
- private final Log log;
-
- /**
- * Constructor.
- *
- * @param igfs Target IGFS.
- * @param log Log.
- */
- public IgfsHadoopInProc(IgfsEx igfs, Log log) {
- this.igfs = igfs;
- this.log = log;
-
- bufSize = igfs.configuration().getBlockSize() * 2;
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHandshakeResponse handshake(String logDir) {
- igfs.clientLogDirectory(logDir);
-
- return new IgfsHandshakeResponse(igfs.name(), igfs.proxyPaths(), igfs.groupBlockSize(),
- igfs.globalSampling());
- }
-
- /** {@inheritDoc} */
- @Override public void close(boolean force) {
- // Perform cleanup.
- for (IgfsHadoopStreamEventListener lsnr : lsnrs.values()) {
- try {
- lsnr.onClose();
- }
- catch (IgniteCheckedException e) {
- if (log.isDebugEnabled())
- log.debug("Failed to notify stream event listener", e);
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsFile info(IgfsPath path) throws IgniteCheckedException {
- try {
- return igfs.info(path);
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to get file info because Grid is stopping: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteCheckedException {
- try {
- return igfs.update(path, props);
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to update file because Grid is stopping: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Boolean setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteCheckedException {
- try {
- igfs.setTimes(path, accessTime, modificationTime);
-
- return true;
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to set path times because Grid is stopping: " +
- path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Boolean rename(IgfsPath src, IgfsPath dest) throws IgniteCheckedException {
- try {
- igfs.rename(src, dest);
-
- return true;
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to rename path because Grid is stopping: " + src);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Boolean delete(IgfsPath path, boolean recursive) throws IgniteCheckedException {
- try {
- return igfs.delete(path, recursive);
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to delete path because Grid is stopping: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsStatus fsStatus() throws IgniteCheckedException {
- try {
- return igfs.globalSpace();
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to get file system status because Grid is " +
- "stopping.");
- }
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteCheckedException {
- try {
- return igfs.listPaths(path);
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to list paths because Grid is stopping: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteCheckedException {
- try {
- return igfs.listFiles(path);
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to list files because Grid is stopping: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Boolean mkdirs(IgfsPath path, Map<String, String> props) throws IgniteCheckedException {
- try {
- igfs.mkdirs(path, props);
-
- return true;
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to create directory because Grid is stopping: " +
- path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsPathSummary contentSummary(IgfsPath path) throws IgniteCheckedException {
- try {
- return igfs.summary(path);
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to get content summary because Grid is stopping: " +
- path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len)
- throws IgniteCheckedException {
- try {
- return igfs.affinity(path, start, len);
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to get affinity because Grid is stopping: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHadoopStreamDelegate open(IgfsPath path) throws IgniteCheckedException {
- try {
- IgfsInputStreamAdapter stream = igfs.open(path, bufSize);
-
- return new IgfsHadoopStreamDelegate(this, stream, stream.fileInfo().length());
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to open file because Grid is stopping: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHadoopStreamDelegate open(IgfsPath path, int seqReadsBeforePrefetch)
- throws IgniteCheckedException {
- try {
- IgfsInputStreamAdapter stream = igfs.open(path, bufSize, seqReadsBeforePrefetch);
-
- return new IgfsHadoopStreamDelegate(this, stream, stream.fileInfo().length());
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to open file because Grid is stopping: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHadoopStreamDelegate create(IgfsPath path, boolean overwrite, boolean colocate,
- int replication, long blockSize, @Nullable Map<String, String> props) throws IgniteCheckedException {
- try {
- IgfsOutputStream stream = igfs.create(path, bufSize, overwrite,
- colocate ? igfs.nextAffinityKey() : null, replication, blockSize, props);
-
- return new IgfsHadoopStreamDelegate(this, stream);
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to create file because Grid is stopping: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsHadoopStreamDelegate append(IgfsPath path, boolean create,
- @Nullable Map<String, String> props) throws IgniteCheckedException {
- try {
- IgfsOutputStream stream = igfs.append(path, bufSize, create, props);
-
- return new IgfsHadoopStreamDelegate(this, stream);
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
- catch (IllegalStateException e) {
- throw new IgfsHadoopCommunicationException("Failed to append file because Grid is stopping: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public GridPlainFuture<byte[]> readData(IgfsHadoopStreamDelegate delegate, long pos, int len,
- @Nullable byte[] outBuf, int outOff, int outLen) {
- IgfsInputStreamAdapter stream = delegate.target();
-
- try {
- byte[] res = null;
-
- if (outBuf != null) {
- int outTailLen = outBuf.length - outOff;
-
- if (len <= outTailLen)
- stream.readFully(pos, outBuf, outOff, len);
- else {
- stream.readFully(pos, outBuf, outOff, outTailLen);
-
- int remainderLen = len - outTailLen;
-
- res = new byte[remainderLen];
-
- stream.readFully(pos, res, 0, remainderLen);
- }
- } else {
- res = new byte[len];
-
- stream.readFully(pos, res, 0, len);
- }
-
- return new GridPlainFutureAdapter<>(res);
- }
- catch (IllegalStateException | IOException e) {
- IgfsHadoopStreamEventListener lsnr = lsnrs.get(delegate);
-
- if (lsnr != null)
- lsnr.onError(e.getMessage());
-
- return new GridPlainFutureAdapter<>(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void writeData(IgfsHadoopStreamDelegate delegate, byte[] data, int off, int len)
- throws IOException {
- try {
- IgfsOutputStream stream = delegate.target();
-
- stream.write(data, off, len);
- }
- catch (IllegalStateException | IOException e) {
- IgfsHadoopStreamEventListener lsnr = lsnrs.get(delegate);
-
- if (lsnr != null)
- lsnr.onError(e.getMessage());
-
- if (e instanceof IllegalStateException)
- throw new IOException("Failed to write data to IGFS stream because Grid is stopping.", e);
- else
- throw e;
- }
- }
-
- /** {@inheritDoc} */
- @Override public void flush(IgfsHadoopStreamDelegate delegate) throws IOException {
- try {
- IgfsOutputStream stream = delegate.target();
-
- stream.flush();
- }
- catch (IllegalStateException | IOException e) {
- IgfsHadoopStreamEventListener lsnr = lsnrs.get(delegate);
-
- if (lsnr != null)
- lsnr.onError(e.getMessage());
-
- if (e instanceof IllegalStateException)
- throw new IOException("Failed to flush data to IGFS stream because Grid is stopping.", e);
- else
- throw e;
- }
- }
-
- /** {@inheritDoc} */
- @Override public void closeStream(IgfsHadoopStreamDelegate desc) throws IOException {
- Closeable closeable = desc.target();
-
- try {
- closeable.close();
- }
- catch (IllegalStateException e) {
- throw new IOException("Failed to close IGFS stream because Grid is stopping.", e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void addEventListener(IgfsHadoopStreamDelegate delegate,
- IgfsHadoopStreamEventListener lsnr) {
- IgfsHadoopStreamEventListener lsnr0 = lsnrs.put(delegate, lsnr);
-
- assert lsnr0 == null || lsnr0 == lsnr;
-
- if (log.isDebugEnabled())
- log.debug("Added stream event listener [delegate=" + delegate + ']');
- }
-
- /** {@inheritDoc} */
- @Override public void removeEventListener(IgfsHadoopStreamDelegate delegate) {
- IgfsHadoopStreamEventListener lsnr0 = lsnrs.remove(delegate);
-
- if (lsnr0 != null && log.isDebugEnabled())
- log.debug("Removed stream event listener [delegate=" + delegate + ']');
- }
-}
[24/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipList.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipList.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipList.java
deleted file mode 100644
index a2c626c..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipList.java
+++ /dev/null
@@ -1,726 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.atomic.*;
-
-/**
- * Skip list.
- */
-public class GridHadoopSkipList extends GridHadoopMultimapBase {
- /** */
- private static final int HEADS_SIZE = 24 + 33 * 8; // Offset + max level is from 0 to 32 inclusive.
-
- /** Top level. */
- private final AtomicInteger topLevel = new AtomicInteger(-1);
-
- /** Heads for all the lists. */
- private final long heads;
-
- /** */
- private final AtomicBoolean visitGuard = new AtomicBoolean();
-
- /**
- * @param jobInfo Job info.
- * @param mem Memory.
- */
- public GridHadoopSkipList(GridHadoopJobInfo jobInfo, GridUnsafeMemory mem) {
- super(jobInfo, mem);
-
- heads = mem.allocate(HEADS_SIZE, true);
- }
-
- /** {@inheritDoc} */
- @Override public void close() {
- super.close();
-
- mem.release(heads, HEADS_SIZE);
- }
-
- /** {@inheritDoc} */
- @Override public boolean visit(boolean ignoreLastVisited, Visitor v) throws IgniteCheckedException {
- if (!visitGuard.compareAndSet(false, true))
- return false;
-
- for (long meta = nextMeta(heads, 0); meta != 0L; meta = nextMeta(meta, 0)) {
- long valPtr = value(meta);
-
- long lastVisited = ignoreLastVisited ? 0 : lastVisitedValue(meta);
-
- if (valPtr != lastVisited) {
- long k = key(meta);
-
- v.onKey(k + 4, keySize(k));
-
- lastVisitedValue(meta, valPtr); // Set it to the first value in chain.
-
- do {
- v.onValue(valPtr + 12, valueSize(valPtr));
-
- valPtr = nextValue(valPtr);
- }
- while (valPtr != lastVisited);
- }
- }
-
- visitGuard.lazySet(false);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public Adder startAdding(GridHadoopTaskContext ctx) throws IgniteCheckedException {
- return new AdderImpl(ctx);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopTaskInput input(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- Input in = new Input(taskCtx);
-
- Comparator<Object> grpCmp = taskCtx.groupComparator();
-
- if (grpCmp != null)
- return new GroupedInput(grpCmp, in);
-
- return in;
- }
-
- /**
- * @param meta Meta pointer.
- * @return Key pointer.
- */
- private long key(long meta) {
- return mem.readLong(meta);
- }
-
- /**
- * @param meta Meta pointer.
- * @param key Key pointer.
- */
- private void key(long meta, long key) {
- mem.writeLong(meta, key);
- }
-
- /**
- * @param meta Meta pointer.
- * @return Value pointer.
- */
- private long value(long meta) {
- return mem.readLongVolatile(meta + 8);
- }
-
- /**
- * @param meta Meta pointer.
- * @param valPtr Value pointer.
- */
- private void value(long meta, long valPtr) {
- mem.writeLongVolatile(meta + 8, valPtr);
- }
-
- /**
- * @param meta Meta pointer.
- * @param oldValPtr Old first value pointer.
- * @param newValPtr New first value pointer.
- * @return {@code true} If operation succeeded.
- */
- private boolean casValue(long meta, long oldValPtr, long newValPtr) {
- return mem.casLong(meta + 8, oldValPtr, newValPtr);
- }
-
- /**
- * @param meta Meta pointer.
- * @return Last visited value pointer.
- */
- private long lastVisitedValue(long meta) {
- return mem.readLong(meta + 16);
- }
-
- /**
- * @param meta Meta pointer.
- * @param valPtr Last visited value pointer.
- */
- private void lastVisitedValue(long meta, long valPtr) {
- mem.writeLong(meta + 16, valPtr);
- }
-
- /**
- * @param meta Meta pointer.
- * @param level Level.
- * @return Next meta pointer.
- */
- private long nextMeta(long meta, int level) {
- assert meta > 0 : meta;
-
- return mem.readLongVolatile(meta + 24 + 8 * level);
- }
-
- /**
- * @param meta Meta pointer.
- * @param level Level.
- * @param oldNext Old next meta pointer.
- * @param newNext New next meta pointer.
- * @return {@code true} If operation succeeded.
- */
- private boolean casNextMeta(long meta, int level, long oldNext, long newNext) {
- assert meta > 0 : meta;
-
- return mem.casLong(meta + 24 + 8 * level, oldNext, newNext);
- }
-
- /**
- * @param meta Meta pointer.
- * @param level Level.
- * @param nextMeta Next meta.
- */
- private void nextMeta(long meta, int level, long nextMeta) {
- assert meta != 0;
-
- mem.writeLong(meta + 24 + 8 * level, nextMeta);
- }
-
- /**
- * @param keyPtr Key pointer.
- * @return Key size.
- */
- private int keySize(long keyPtr) {
- return mem.readInt(keyPtr);
- }
-
- /**
- * @param keyPtr Key pointer.
- * @param keySize Key size.
- */
- private void keySize(long keyPtr, int keySize) {
- mem.writeInt(keyPtr, keySize);
- }
-
- /**
- * @param rnd Random.
- * @return Next level.
- */
- public static int randomLevel(Random rnd) {
- int x = rnd.nextInt();
-
- int level = 0;
-
- while ((x & 1) != 0) { // Count sequential 1 bits.
- level++;
-
- x >>>= 1;
- }
-
- return level;
- }
-
- /**
- * Reader.
- */
- private class Reader extends ReaderBase {
- /**
- * @param ser Serialization.
- */
- protected Reader(GridHadoopSerialization ser) {
- super(ser);
- }
-
- /**
- * @param meta Meta pointer.
- * @return Key.
- */
- public Object readKey(long meta) {
- assert meta > 0 : meta;
-
- long k = key(meta);
-
- try {
- return read(k + 4, keySize(k));
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
- }
-
- /**
- * Adder.
- */
- private class AdderImpl extends AdderBase {
- /** */
- private final Comparator<Object> cmp;
-
- /** */
- private final Random rnd = new GridRandom();
-
- /** */
- private final GridLongList stack = new GridLongList(16);
-
- /** */
- private final Reader keyReader;
-
- /**
- * @param ctx Task context.
- * @throws IgniteCheckedException If failed.
- */
- protected AdderImpl(GridHadoopTaskContext ctx) throws IgniteCheckedException {
- super(ctx);
-
- keyReader = new Reader(keySer);
-
- cmp = ctx.sortComparator();
- }
-
- /** {@inheritDoc} */
- @Override public void write(Object key, Object val) throws IgniteCheckedException {
- A.notNull(val, "val");
-
- add(key, val);
- }
-
- /** {@inheritDoc} */
- @Override public Key addKey(DataInput in, @Nullable Key reuse) throws IgniteCheckedException {
- KeyImpl k = reuse == null ? new KeyImpl() : (KeyImpl)reuse;
-
- k.tmpKey = keySer.read(in, k.tmpKey);
-
- k.meta = add(k.tmpKey, null);
-
- return k;
- }
-
- /**
- * @param key Key.
- * @param val Value.
- * @param level Level.
- * @return Meta pointer.
- */
- private long createMeta(long key, long val, int level) {
- int size = 32 + 8 * level;
-
- long meta = allocate(size);
-
- key(meta, key);
- value(meta, val);
- lastVisitedValue(meta, 0L);
-
- for (int i = 32; i < size; i += 8) // Fill with 0.
- mem.writeLong(meta + i, 0L);
-
- return meta;
- }
-
- /**
- * @param key Key.
- * @return Pointer.
- * @throws IgniteCheckedException If failed.
- */
- private long writeKey(Object key) throws IgniteCheckedException {
- long keyPtr = write(4, key, keySer);
- int keySize = writtenSize() - 4;
-
- keySize(keyPtr, keySize);
-
- return keyPtr;
- }
-
- /**
- * @param prevMeta Previous meta.
- * @param meta Next meta.
- */
- private void stackPush(long prevMeta, long meta) {
- stack.add(prevMeta);
- stack.add(meta);
- }
-
- /**
- * Drops last remembered frame from the stack.
- */
- private void stackPop() {
- stack.pop(2);
- }
-
- /**
- * @param key Key.
- * @param val Value.
- * @return Meta pointer.
- * @throws IgniteCheckedException If failed.
- */
- private long add(Object key, @Nullable Object val) throws IgniteCheckedException {
- assert key != null;
-
- stack.clear();
-
- long valPtr = 0;
-
- if (val != null) { // Write value.
- valPtr = write(12, val, valSer);
- int valSize = writtenSize() - 12;
-
- nextValue(valPtr, 0);
- valueSize(valPtr, valSize);
- }
-
- long keyPtr = 0;
- long newMeta = 0;
- int newMetaLevel = -1;
-
- long prevMeta = heads;
- int level = topLevel.get();
- long meta = level < 0 ? 0 : nextMeta(heads, level);
-
- for (;;) {
- if (level < 0) { // We did not find our key, trying to add new meta.
- if (keyPtr == 0) { // Write key and create meta only once.
- keyPtr = writeKey(key);
-
- newMetaLevel = randomLevel(rnd);
- newMeta = createMeta(keyPtr, valPtr, newMetaLevel);
- }
-
- nextMeta(newMeta, 0, meta); // Set next to new meta before publishing.
-
- if (casNextMeta(prevMeta, 0, meta, newMeta)) { // New key was added successfully.
- laceUp(key, newMeta, newMetaLevel);
-
- return newMeta;
- }
- else { // Add failed, need to check out what was added by another thread.
- meta = nextMeta(prevMeta, level = 0);
-
- stackPop();
- }
- }
-
- int cmpRes = cmp(key, meta);
-
- if (cmpRes == 0) { // Key found.
- if (newMeta != 0) // Deallocate if we've allocated something.
- localDeallocate(keyPtr);
-
- if (valPtr == 0) // Only key needs to be added.
- return meta;
-
- for (;;) { // Add value for the key found.
- long nextVal = value(meta);
-
- nextValue(valPtr, nextVal);
-
- if (casValue(meta, nextVal, valPtr))
- return meta;
- }
- }
-
- assert cmpRes != 0;
-
- if (cmpRes > 0) { // Go right.
- prevMeta = meta;
- meta = nextMeta(meta, level);
-
- if (meta != 0) // If nothing to the right then go down.
- continue;
- }
-
- while (--level >= 0) { // Go down.
- stackPush(prevMeta, meta); // Remember the path.
-
- long nextMeta = nextMeta(prevMeta, level);
-
- if (nextMeta != meta) { // If the meta is the same as on upper level go deeper.
- meta = nextMeta;
-
- assert meta != 0;
-
- break;
- }
- }
- }
- }
-
- /**
- * @param key Key.
- * @param meta Meta pointer.
- * @return Comparison result.
- */
- @SuppressWarnings("unchecked")
- private int cmp(Object key, long meta) {
- assert meta != 0;
-
- return cmp.compare(key, keyReader.readKey(meta));
- }
-
- /**
- * Adds appropriate index links between metas.
- *
- * @param newMeta Just added meta.
- * @param newMetaLevel New level.
- */
- private void laceUp(Object key, long newMeta, int newMetaLevel) {
- for (int level = 1; level <= newMetaLevel; level++) { // Go from the bottom up.
- long prevMeta = heads;
- long meta = 0;
-
- if (!stack.isEmpty()) { // Get the path back.
- meta = stack.remove();
- prevMeta = stack.remove();
- }
-
- for (;;) {
- nextMeta(newMeta, level, meta);
-
- if (casNextMeta(prevMeta, level, meta, newMeta))
- break;
-
- long oldMeta = meta;
-
- meta = nextMeta(prevMeta, level); // Reread meta.
-
- for (;;) {
- int cmpRes = cmp(key, meta);
-
- if (cmpRes > 0) { // Go right.
- prevMeta = meta;
- meta = nextMeta(prevMeta, level);
-
- if (meta != oldMeta) // Old meta already known to be greater than ours or is 0.
- continue;
- }
-
- assert cmpRes != 0; // Two different metas with equal keys must be impossible.
-
- break; // Retry cas.
- }
- }
- }
-
- if (!stack.isEmpty())
- return; // Our level already lower than top.
-
- for (;;) { // Raise top level.
- int top = topLevel.get();
-
- if (newMetaLevel <= top || topLevel.compareAndSet(top, newMetaLevel))
- break;
- }
- }
-
- /**
- * Key.
- */
- private class KeyImpl implements Key {
- /** */
- private long meta;
-
- /** */
- private Object tmpKey;
-
- /**
- * @return Meta pointer for the key.
- */
- public long address() {
- return meta;
- }
-
- /**
- * @param val Value.
- */
- @Override public void add(Value val) {
- int size = val.size();
-
- long valPtr = allocate(size + 12);
-
- val.copyTo(valPtr + 12);
-
- valueSize(valPtr, size);
-
- long nextVal;
-
- do {
- nextVal = value(meta);
-
- nextValue(valPtr, nextVal);
- }
- while(!casValue(meta, nextVal, valPtr));
- }
- }
- }
-
- /**
- * Task input.
- */
- private class Input implements GridHadoopTaskInput {
- /** */
- private long metaPtr = heads;
-
- /** */
- private final Reader keyReader;
-
- /** */
- private final Reader valReader;
-
- /**
- * @param taskCtx Task context.
- * @throws IgniteCheckedException If failed.
- */
- private Input(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- keyReader = new Reader(taskCtx.keySerialization());
- valReader = new Reader(taskCtx.valueSerialization());
- }
-
- /** {@inheritDoc} */
- @Override public boolean next() {
- metaPtr = nextMeta(metaPtr, 0);
-
- return metaPtr != 0;
- }
-
- /** {@inheritDoc} */
- @Override public Object key() {
- return keyReader.readKey(metaPtr);
- }
-
- /** {@inheritDoc} */
- @Override public Iterator<?> values() {
- return new ValueIterator(value(metaPtr), valReader);
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteCheckedException {
- keyReader.close();
- valReader.close();
- }
- }
-
- /**
- * Grouped input using grouping comparator.
- */
- private class GroupedInput implements GridHadoopTaskInput {
- /** */
- private final Comparator<Object> grpCmp;
-
- /** */
- private final Input in;
-
- /** */
- private Object prevKey;
-
- /** */
- private Object nextKey;
-
- /** */
- private final GridLongList vals = new GridLongList();
-
- /**
- * @param grpCmp Grouping comparator.
- * @param in Input.
- */
- private GroupedInput(Comparator<Object> grpCmp, Input in) {
- this.grpCmp = grpCmp;
- this.in = in;
- }
-
- /** {@inheritDoc} */
- @Override public boolean next() {
- if (prevKey == null) { // First call.
- if (!in.next())
- return false;
-
- prevKey = in.key();
-
- assert prevKey != null;
-
- in.keyReader.resetReusedObject(null); // We need 2 instances of key object for comparison.
-
- vals.add(value(in.metaPtr));
- }
- else {
- if (in.metaPtr == 0) // We reached the end of the input.
- return false;
-
- vals.clear();
-
- vals.add(value(in.metaPtr));
-
- in.keyReader.resetReusedObject(prevKey); // Switch key instances.
-
- prevKey = nextKey;
- }
-
- while (in.next()) { // Fill with head value pointers with equal keys.
- if (grpCmp.compare(prevKey, nextKey = in.key()) == 0)
- vals.add(value(in.metaPtr));
- else
- break;
- }
-
- assert !vals.isEmpty();
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public Object key() {
- return prevKey;
- }
-
- /** {@inheritDoc} */
- @Override public Iterator<?> values() {
- assert !vals.isEmpty();
-
- final ValueIterator valIter = new ValueIterator(vals.get(0), in.valReader);
-
- return new Iterator<Object>() {
- /** */
- private int idx;
-
- @Override public boolean hasNext() {
- if (!valIter.hasNext()) {
- if (++idx == vals.size())
- return false;
-
- valIter.head(vals.get(idx));
-
- assert valIter.hasNext();
- }
-
- return true;
- }
-
- @Override public Object next() {
- return valIter.next();
- }
-
- @Override public void remove() {
- valIter.remove();
- }
- };
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteCheckedException {
- in.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopConcurrentHashMultimap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopConcurrentHashMultimap.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopConcurrentHashMultimap.java
new file mode 100644
index 0000000..65d9268
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopConcurrentHashMultimap.java
@@ -0,0 +1,611 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+
+/**
+ * Multimap for map reduce intermediate results.
+ */
+public class HadoopConcurrentHashMultimap extends HadoopHashMultimapBase {
+ /** */
+ private final AtomicReference<State> state = new AtomicReference<>(State.READING_WRITING);
+
+ /** */
+ private volatile AtomicLongArray oldTbl;
+
+ /** */
+ private volatile AtomicLongArray newTbl;
+
+ /** */
+ private final AtomicInteger keys = new AtomicInteger();
+
+ /** */
+ private final CopyOnWriteArrayList<AdderImpl> adders = new CopyOnWriteArrayList<>();
+
+ /** */
+ private final AtomicInteger inputs = new AtomicInteger();
+
+ /**
+ * @param jobInfo Job info.
+ * @param mem Memory.
+ * @param cap Initial capacity.
+ */
+ public HadoopConcurrentHashMultimap(HadoopJobInfo jobInfo, GridUnsafeMemory mem, int cap) {
+ super(jobInfo, mem);
+
+ assert U.isPow2(cap);
+
+ newTbl = oldTbl = new AtomicLongArray(cap);
+ }
+
+ /**
+ * @return Number of keys.
+ */
+ public long keys() {
+ int res = keys.get();
+
+ for (AdderImpl adder : adders)
+ res += adder.locKeys.get();
+
+ return res;
+ }
+
+ /**
+ * @return Current table capacity.
+ */
+ @Override public int capacity() {
+ return oldTbl.length();
+ }
+
+ /**
+ * @return Adder object.
+ * @param ctx Task context.
+ */
+ @Override public Adder startAdding(HadoopTaskContext ctx) throws IgniteCheckedException {
+ if (inputs.get() != 0)
+ throw new IllegalStateException("Active inputs.");
+
+ if (state.get() == State.CLOSING)
+ throw new IllegalStateException("Closed.");
+
+ return new AdderImpl(ctx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() {
+ assert inputs.get() == 0 : inputs.get();
+ assert adders.isEmpty() : adders.size();
+
+ state(State.READING_WRITING, State.CLOSING);
+
+ if (keys() == 0)
+ return;
+
+ super.close();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long meta(int idx) {
+ return oldTbl.get(idx);
+ }
+
+ /**
+ * Incrementally visits all the keys and values in the map.
+ *
+ * @param ignoreLastVisited Flag indicating that visiting must be started from the beginning.
+ * @param v Visitor.
+ * @return {@code false} If visiting was impossible due to rehashing.
+ */
+ @Override public boolean visit(boolean ignoreLastVisited, Visitor v) throws IgniteCheckedException {
+ if (!state.compareAndSet(State.READING_WRITING, State.VISITING)) {
+ assert state.get() != State.CLOSING;
+
+ return false; // Can not visit while rehashing happens.
+ }
+
+ AtomicLongArray tbl0 = oldTbl;
+
+ for (int i = 0; i < tbl0.length(); i++) {
+ long meta = tbl0.get(i);
+
+ while (meta != 0) {
+ long valPtr = value(meta);
+
+ long lastVisited = ignoreLastVisited ? 0 : lastVisitedValue(meta);
+
+ if (valPtr != lastVisited) {
+ v.onKey(key(meta), keySize(meta));
+
+ lastVisitedValue(meta, valPtr); // Set it to the first value in chain.
+
+ do {
+ v.onValue(valPtr + 12, valueSize(valPtr));
+
+ valPtr = nextValue(valPtr);
+ }
+ while (valPtr != lastVisited);
+ }
+
+ meta = collision(meta);
+ }
+ }
+
+ state(State.VISITING, State.READING_WRITING);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopTaskInput input(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ inputs.incrementAndGet();
+
+ if (!adders.isEmpty())
+ throw new IllegalStateException("Active adders.");
+
+ State s = state.get();
+
+ if (s == State.CLOSING)
+ throw new IllegalStateException("Closed.");
+
+ assert s != State.REHASHING;
+
+ return new Input(taskCtx) {
+ @Override public void close() throws IgniteCheckedException {
+ if (inputs.decrementAndGet() < 0)
+ throw new IllegalStateException();
+
+ super.close();
+ }
+ };
+ }
+
+ /**
+ * @param fromTbl Table.
+ */
+ private void rehashIfNeeded(AtomicLongArray fromTbl) {
+ if (fromTbl.length() == Integer.MAX_VALUE)
+ return;
+
+ long keys0 = keys();
+
+ if (keys0 < 3 * (fromTbl.length() >>> 2)) // New size has to be >= than 3/4 of capacity to rehash.
+ return;
+
+ if (fromTbl != newTbl) // Check if someone else have done the job.
+ return;
+
+ if (!state.compareAndSet(State.READING_WRITING, State.REHASHING)) {
+ assert state.get() != State.CLOSING; // Visiting is allowed, but we will not rehash.
+
+ return;
+ }
+
+ if (fromTbl != newTbl) { // Double check.
+ state(State.REHASHING, State.READING_WRITING); // Switch back.
+
+ return;
+ }
+
+ // Calculate new table capacity.
+ int newLen = fromTbl.length();
+
+ do {
+ newLen <<= 1;
+ }
+ while (newLen < keys0);
+
+ if (keys0 >= 3 * (newLen >>> 2)) // Still more than 3/4.
+ newLen <<= 1;
+
+ // This is our target table for rehashing.
+ AtomicLongArray toTbl = new AtomicLongArray(newLen);
+
+ // Make the new table visible before rehashing.
+ newTbl = toTbl;
+
+ // Rehash.
+ int newMask = newLen - 1;
+
+ long failedMeta = 0;
+
+ GridLongList collisions = new GridLongList(16);
+
+ for (int i = 0; i < fromTbl.length(); i++) { // Scan source table.
+ long meta = fromTbl.get(i);
+
+ assert meta != -1;
+
+ if (meta == 0) { // No entry.
+ failedMeta = 0;
+
+ if (!fromTbl.compareAndSet(i, 0, -1)) // Mark as moved.
+ i--; // Retry.
+
+ continue;
+ }
+
+ do { // Collect all the collisions before the last one failed to nullify or 0.
+ collisions.add(meta);
+
+ meta = collision(meta);
+ }
+ while (meta != failedMeta);
+
+ do { // Go from the last to the first to avoid 'in-flight' state for meta entries.
+ meta = collisions.remove();
+
+ int addr = keyHash(meta) & newMask;
+
+ for (;;) { // Move meta entry to the new table.
+ long toCollision = toTbl.get(addr);
+
+ collision(meta, toCollision);
+
+ if (toTbl.compareAndSet(addr, toCollision, meta))
+ break;
+ }
+ }
+ while (!collisions.isEmpty());
+
+ // Here 'meta' will be a root pointer in old table.
+ if (!fromTbl.compareAndSet(i, meta, -1)) { // Try to mark as moved.
+ failedMeta = meta;
+
+ i--; // Retry the same address in table because new keys were added.
+ }
+ else
+ failedMeta = 0;
+ }
+
+ // Now old and new tables will be the same again.
+ oldTbl = toTbl;
+
+ state(State.REHASHING, State.READING_WRITING);
+ }
+
+ /**
+ * Switch state.
+ *
+ * @param oldState Expected state.
+ * @param newState New state.
+ */
+ private void state(State oldState, State newState) {
+ if (!state.compareAndSet(oldState, newState))
+ throw new IllegalStateException();
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @return Value pointer.
+ */
+ @Override protected long value(long meta) {
+ return mem.readLongVolatile(meta + 16);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @param oldValPtr Old value.
+ * @param newValPtr New value.
+ * @return {@code true} If succeeded.
+ */
+ private boolean casValue(long meta, long oldValPtr, long newValPtr) {
+ return mem.casLong(meta + 16, oldValPtr, newValPtr);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @return Collision pointer.
+ */
+ @Override protected long collision(long meta) {
+ return mem.readLongVolatile(meta + 24);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @param collision Collision pointer.
+ */
+ @Override protected void collision(long meta, long collision) {
+ assert meta != collision : meta;
+
+ mem.writeLongVolatile(meta + 24, collision);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @return Last visited value pointer.
+ */
+ private long lastVisitedValue(long meta) {
+ return mem.readLong(meta + 32);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @param valPtr Last visited value pointer.
+ */
+ private void lastVisitedValue(long meta, long valPtr) {
+ mem.writeLong(meta + 32, valPtr);
+ }
+
+ /**
+ * Adder. Must not be shared between threads.
+ */
+ private class AdderImpl extends AdderBase {
+ /** */
+ private final Reader keyReader;
+
+ /** */
+ private final AtomicInteger locKeys = new AtomicInteger();
+
+ /** */
+ private final Random rnd = new GridRandom();
+
+ /**
+ * @param ctx Task context.
+ * @throws IgniteCheckedException If failed.
+ */
+ private AdderImpl(HadoopTaskContext ctx) throws IgniteCheckedException {
+ super(ctx);
+
+ keyReader = new Reader(keySer);
+
+ rehashIfNeeded(oldTbl);
+
+ adders.add(this);
+ }
+
+ /**
+ * @param in Data input.
+ * @param reuse Reusable key.
+ * @return Key.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Override public Key addKey(DataInput in, @Nullable Key reuse) throws IgniteCheckedException {
+ KeyImpl k = reuse == null ? new KeyImpl() : (KeyImpl)reuse;
+
+ k.tmpKey = keySer.read(in, k.tmpKey);
+
+ k.meta = add(k.tmpKey, null);
+
+ return k;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(Object key, Object val) throws IgniteCheckedException {
+ A.notNull(val, "val");
+
+ add(key, val);
+ }
+
+ /**
+ * @param tbl Table.
+ */
+ private void incrementKeys(AtomicLongArray tbl) {
+ locKeys.lazySet(locKeys.get() + 1);
+
+ if (rnd.nextInt(tbl.length()) < 512)
+ rehashIfNeeded(tbl);
+ }
+
+ /**
+ * @param keyHash Key hash.
+ * @param keySize Key size.
+ * @param keyPtr Key pointer.
+ * @param valPtr Value page pointer.
+ * @param collisionPtr Pointer to meta with hash collision.
+ * @param lastVisitedVal Last visited value pointer.
+ * @return Created meta page pointer.
+ */
+ private long createMeta(int keyHash, int keySize, long keyPtr, long valPtr, long collisionPtr, long lastVisitedVal) {
+ long meta = allocate(40);
+
+ mem.writeInt(meta, keyHash);
+ mem.writeInt(meta + 4, keySize);
+ mem.writeLong(meta + 8, keyPtr);
+ mem.writeLong(meta + 16, valPtr);
+ mem.writeLong(meta + 24, collisionPtr);
+ mem.writeLong(meta + 32, lastVisitedVal);
+
+ return meta;
+ }
+
+ /**
+ * @param key Key.
+ * @param val Value.
+ * @return Updated or created meta page pointer.
+ * @throws IgniteCheckedException If failed.
+ */
+ private long add(Object key, @Nullable Object val) throws IgniteCheckedException {
+ AtomicLongArray tbl = oldTbl;
+
+ int keyHash = U.hash(key.hashCode());
+
+ long newMetaPtr = 0;
+
+ long valPtr = 0;
+
+ if (val != null) {
+ valPtr = write(12, val, valSer);
+ int valSize = writtenSize() - 12;
+
+ valueSize(valPtr, valSize);
+ }
+
+ for (AtomicLongArray old = null;;) {
+ int addr = keyHash & (tbl.length() - 1);
+
+ long metaPtrRoot = tbl.get(addr); // Read root meta pointer at this address.
+
+ if (metaPtrRoot == -1) { // The cell was already moved by rehashing.
+ AtomicLongArray n = newTbl; // Need to read newTbl first here.
+ AtomicLongArray o = oldTbl;
+
+ tbl = tbl == o ? n : o; // Trying to get the oldest table but newer than ours.
+
+ old = null;
+
+ continue;
+ }
+
+ if (metaPtrRoot != 0) { // Not empty slot.
+ long metaPtr = metaPtrRoot;
+
+ do { // Scan all the collisions.
+ if (keyHash(metaPtr) == keyHash && key.equals(keyReader.readKey(metaPtr))) { // Found key.
+ if (newMetaPtr != 0) // Deallocate new meta if one was allocated.
+ localDeallocate(key(newMetaPtr)); // Key was allocated first, so rewind to it's pointer.
+
+ if (valPtr != 0) { // Add value if it exists.
+ long nextValPtr;
+
+ // Values are linked to each other to a stack like structure.
+ // Replace the last value in meta with ours and link it as next.
+ do {
+ nextValPtr = value(metaPtr);
+
+ nextValue(valPtr, nextValPtr);
+ }
+ while (!casValue(metaPtr, nextValPtr, valPtr));
+ }
+
+ return metaPtr;
+ }
+
+ metaPtr = collision(metaPtr);
+ }
+ while (metaPtr != 0);
+
+ // Here we did not find our key, need to check if it was moved by rehashing to the new table.
+ if (old == null) { // If the old table already set, then we will just try to update it.
+ AtomicLongArray n = newTbl;
+
+ if (n != tbl) { // Rehashing happens, try to find the key in new table but preserve the old one.
+ old = tbl;
+ tbl = n;
+
+ continue;
+ }
+ }
+ }
+
+ if (old != null) { // We just checked new table but did not find our key as well as in the old one.
+ tbl = old; // Try to add new key to the old table.
+
+ addr = keyHash & (tbl.length() - 1);
+
+ old = null;
+ }
+
+ if (newMetaPtr == 0) { // Allocate new meta page.
+ long keyPtr = write(0, key, keySer);
+ int keySize = writtenSize();
+
+ if (valPtr != 0)
+ nextValue(valPtr, 0);
+
+ newMetaPtr = createMeta(keyHash, keySize, keyPtr, valPtr, metaPtrRoot, 0);
+ }
+ else // Update new meta with root pointer collision.
+ collision(newMetaPtr, metaPtrRoot);
+
+ if (tbl.compareAndSet(addr, metaPtrRoot, newMetaPtr)) { // Try to replace root pointer with new one.
+ incrementKeys(tbl);
+
+ return newMetaPtr;
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ if (!adders.remove(this))
+ throw new IllegalStateException();
+
+ keys.addAndGet(locKeys.get()); // Here we have race and #keys() method can return wrong result but it is ok.
+
+ super.close();
+ }
+
+ /**
+ * Key.
+ */
+ private class KeyImpl implements Key {
+ /** */
+ private long meta;
+
+ /** */
+ private Object tmpKey;
+
+ /**
+ * @return Meta pointer for the key.
+ */
+ public long address() {
+ return meta;
+ }
+
+ /**
+ * @param val Value.
+ */
+ @Override public void add(Value val) {
+ int size = val.size();
+
+ long valPtr = allocate(size + 12);
+
+ val.copyTo(valPtr + 12);
+
+ valueSize(valPtr, size);
+
+ long nextVal;
+
+ do {
+ nextVal = value(meta);
+
+ nextValue(valPtr, nextVal);
+ }
+ while(!casValue(meta, nextVal, valPtr));
+ }
+ }
+ }
+
+ /**
+ * Current map state.
+ */
+ private enum State {
+ /** */
+ REHASHING,
+
+ /** */
+ VISITING,
+
+ /** */
+ READING_WRITING,
+
+ /** */
+ CLOSING
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopHashMultimap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopHashMultimap.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopHashMultimap.java
new file mode 100644
index 0000000..f524bdc
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopHashMultimap.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+/**
+ * Hash multimap.
+ */
+public class HadoopHashMultimap extends HadoopHashMultimapBase {
+ /** */
+ private long[] tbl;
+
+ /** */
+ private int keys;
+
+ /**
+ * @param jobInfo Job info.
+ * @param mem Memory.
+ * @param cap Initial capacity.
+ */
+ public HadoopHashMultimap(HadoopJobInfo jobInfo, GridUnsafeMemory mem, int cap) {
+ super(jobInfo, mem);
+
+ assert U.isPow2(cap) : cap;
+
+ tbl = new long[cap];
+ }
+
+ /** {@inheritDoc} */
+ @Override public Adder startAdding(HadoopTaskContext ctx) throws IgniteCheckedException {
+ return new AdderImpl(ctx);
+ }
+
+ /**
+ * Rehash.
+ */
+ private void rehash() {
+ long[] newTbl = new long[tbl.length << 1];
+
+ int newMask = newTbl.length - 1;
+
+ for (long meta : tbl) {
+ while (meta != 0) {
+ long collision = collision(meta);
+
+ int idx = keyHash(meta) & newMask;
+
+ collision(meta, newTbl[idx]);
+
+ newTbl[idx] = meta;
+
+ meta = collision;
+ }
+ }
+
+ tbl = newTbl;
+ }
+
+ /**
+ * @return Keys count.
+ */
+ public int keys() {
+ return keys;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int capacity() {
+ return tbl.length;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long meta(int idx) {
+ return tbl[idx];
+ }
+
+ /**
+ * Adder.
+ */
+ private class AdderImpl extends AdderBase {
+ /** */
+ private final Reader keyReader;
+
+ /**
+ * @param ctx Task context.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected AdderImpl(HadoopTaskContext ctx) throws IgniteCheckedException {
+ super(ctx);
+
+ keyReader = new Reader(keySer);
+ }
+
+ /**
+ * @param keyHash Key hash.
+ * @param keySize Key size.
+ * @param keyPtr Key pointer.
+ * @param valPtr Value page pointer.
+ * @param collisionPtr Pointer to meta with hash collision.
+ * @return Created meta page pointer.
+ */
+ private long createMeta(int keyHash, int keySize, long keyPtr, long valPtr, long collisionPtr) {
+ long meta = allocate(32);
+
+ mem.writeInt(meta, keyHash);
+ mem.writeInt(meta + 4, keySize);
+ mem.writeLong(meta + 8, keyPtr);
+ mem.writeLong(meta + 16, valPtr);
+ mem.writeLong(meta + 24, collisionPtr);
+
+ return meta;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(Object key, Object val) throws IgniteCheckedException {
+ A.notNull(val, "val");
+
+ int keyHash = U.hash(key.hashCode());
+
+ // Write value.
+ long valPtr = write(12, val, valSer);
+ int valSize = writtenSize() - 12;
+
+ valueSize(valPtr, valSize);
+
+ // Find position in table.
+ int idx = keyHash & (tbl.length - 1);
+
+ long meta = tbl[idx];
+
+ // Search for our key in collisions.
+ while (meta != 0) {
+ if (keyHash(meta) == keyHash && key.equals(keyReader.readKey(meta))) { // Found key.
+ nextValue(valPtr, value(meta));
+
+ value(meta, valPtr);
+
+ return;
+ }
+
+ meta = collision(meta);
+ }
+
+ // Write key.
+ long keyPtr = write(0, key, keySer);
+ int keySize = writtenSize();
+
+ nextValue(valPtr, 0);
+
+ tbl[idx] = createMeta(keyHash, keySize, keyPtr, valPtr, tbl[idx]);
+
+ if (++keys > (tbl.length >>> 2) * 3)
+ rehash();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopHashMultimapBase.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopHashMultimapBase.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopHashMultimapBase.java
new file mode 100644
index 0000000..16aa673
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopHashMultimapBase.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+
+import java.util.*;
+
+/**
+ * Base class for hash multimaps.
+ */
+public abstract class HadoopHashMultimapBase extends HadoopMultimapBase {
+ /**
+ * @param jobInfo Job info.
+ * @param mem Memory.
+ */
+ protected HadoopHashMultimapBase(HadoopJobInfo jobInfo, GridUnsafeMemory mem) {
+ super(jobInfo, mem);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean visit(boolean ignoreLastVisited, Visitor v) throws IgniteCheckedException {
+ throw new UnsupportedOperationException("visit");
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopTaskInput input(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ return new Input(taskCtx);
+ }
+
+ /**
+ * @return Hash table capacity.
+ */
+ public abstract int capacity();
+
+ /**
+ * @param idx Index in hash table.
+ * @return Meta page pointer.
+ */
+ protected abstract long meta(int idx);
+
+ /**
+ * @param meta Meta pointer.
+ * @return Key hash.
+ */
+ protected int keyHash(long meta) {
+ return mem.readInt(meta);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @return Key size.
+ */
+ protected int keySize(long meta) {
+ return mem.readInt(meta + 4);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @return Key pointer.
+ */
+ protected long key(long meta) {
+ return mem.readLong(meta + 8);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @return Value pointer.
+ */
+ protected long value(long meta) {
+ return mem.readLong(meta + 16);
+ }
+ /**
+ * @param meta Meta pointer.
+ * @param val Value pointer.
+ */
+ protected void value(long meta, long val) {
+ mem.writeLong(meta + 16, val);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @return Collision pointer.
+ */
+ protected long collision(long meta) {
+ return mem.readLong(meta + 24);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @param collision Collision pointer.
+ */
+ protected void collision(long meta, long collision) {
+ assert meta != collision : meta;
+
+ mem.writeLong(meta + 24, collision);
+ }
+
+ /**
+ * Reader for key and value.
+ */
+ protected class Reader extends ReaderBase {
+ /**
+ * @param ser Serialization.
+ */
+ protected Reader(HadoopSerialization ser) {
+ super(ser);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @return Key.
+ */
+ public Object readKey(long meta) {
+ assert meta > 0 : meta;
+
+ try {
+ return read(key(meta), keySize(meta));
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+ }
+
+ /**
+ * Task input.
+ */
+ protected class Input implements HadoopTaskInput {
+ /** */
+ private int idx = -1;
+
+ /** */
+ private long metaPtr;
+
+ /** */
+ private final int cap;
+
+ /** */
+ private final Reader keyReader;
+
+ /** */
+ private final Reader valReader;
+
+ /**
+ * @param taskCtx Task context.
+ * @throws IgniteCheckedException If failed.
+ */
+ public Input(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ cap = capacity();
+
+ keyReader = new Reader(taskCtx.keySerialization());
+ valReader = new Reader(taskCtx.valueSerialization());
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean next() {
+ if (metaPtr != 0) {
+ metaPtr = collision(metaPtr);
+
+ if (metaPtr != 0)
+ return true;
+ }
+
+ while (++idx < cap) { // Scan table.
+ metaPtr = meta(idx);
+
+ if (metaPtr != 0)
+ return true;
+ }
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object key() {
+ return keyReader.readKey(metaPtr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Iterator<?> values() {
+ return new ValueIterator(value(metaPtr), valReader);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ keyReader.close();
+ valReader.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimap.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimap.java
new file mode 100644
index 0000000..5def6d3
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimap.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * Multimap for hadoop intermediate results.
+ */
+@SuppressWarnings("PublicInnerClass")
+public interface HadoopMultimap extends AutoCloseable {
+ /**
+ * Incrementally visits all the keys and values in the map.
+ *
+ * @param ignoreLastVisited Flag indicating that visiting must be started from the beginning.
+ * @param v Visitor.
+ * @return {@code false} If visiting was impossible.
+ */
+ public boolean visit(boolean ignoreLastVisited, Visitor v) throws IgniteCheckedException;
+
+ /**
+ * @param ctx Task context.
+ * @return Adder.
+ * @throws IgniteCheckedException If failed.
+ */
+ public Adder startAdding(HadoopTaskContext ctx) throws IgniteCheckedException;
+
+ /**
+ * @param taskCtx Task context.
+ * @return Task input.
+ * @throws IgniteCheckedException If failed.
+ */
+ public HadoopTaskInput input(HadoopTaskContext taskCtx)
+ throws IgniteCheckedException;
+
+ /** {@inheritDoc} */
+ @Override public void close();
+
+ /**
+ * Adder.
+ */
+ public interface Adder extends HadoopTaskOutput {
+ /**
+ * @param in Data input.
+ * @param reuse Reusable key.
+ * @return Key.
+ * @throws IgniteCheckedException If failed.
+ */
+ public Key addKey(DataInput in, @Nullable Key reuse) throws IgniteCheckedException;
+ }
+
+ /**
+ * Key add values to.
+ */
+ public interface Key {
+ /**
+ * @param val Value.
+ */
+ public void add(Value val);
+ }
+
+ /**
+ * Value.
+ */
+ public interface Value {
+ /**
+ * @return Size in bytes.
+ */
+ public int size();
+
+ /**
+ * @param ptr Pointer.
+ */
+ public void copyTo(long ptr);
+ }
+
+ /**
+ * Key and values visitor.
+ */
+ public interface Visitor {
+ /**
+ * @param keyPtr Key pointer.
+ * @param keySize Key size.
+ */
+ public void onKey(long keyPtr, int keySize) throws IgniteCheckedException;
+
+ /**
+ * @param valPtr Value pointer.
+ * @param valSize Value size.
+ */
+ public void onValue(long valPtr, int valSize) throws IgniteCheckedException;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
new file mode 100644
index 0000000..7f332aa
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.shuffle.streams.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.*;
+
+/**
+ * Base class for all multimaps.
+ */
+public abstract class HadoopMultimapBase implements HadoopMultimap {
+ /** */
+ protected final GridUnsafeMemory mem;
+
+ /** */
+ protected final int pageSize;
+
+ /** */
+ private final Collection<GridLongList> allPages = new ConcurrentLinkedQueue<>();
+
+ /**
+ * @param jobInfo Job info.
+ * @param mem Memory.
+ */
+ protected HadoopMultimapBase(HadoopJobInfo jobInfo, GridUnsafeMemory mem) {
+ assert jobInfo != null;
+ assert mem != null;
+
+ this.mem = mem;
+
+ pageSize = get(jobInfo, SHUFFLE_OFFHEAP_PAGE_SIZE, 32 * 1024);
+ }
+
+ /**
+ * @param ptrs Page pointers.
+ */
+ private void deallocate(GridLongList ptrs) {
+ while (!ptrs.isEmpty())
+ mem.release(ptrs.remove(), ptrs.remove());
+ }
+
+ /**
+ * @param valPtr Value page pointer.
+ * @param nextValPtr Next value page pointer.
+ */
+ protected void nextValue(long valPtr, long nextValPtr) {
+ mem.writeLong(valPtr, nextValPtr);
+ }
+
+ /**
+ * @param valPtr Value page pointer.
+ * @return Next value page pointer.
+ */
+ protected long nextValue(long valPtr) {
+ return mem.readLong(valPtr);
+ }
+
+ /**
+ * @param valPtr Value page pointer.
+ * @param size Size.
+ */
+ protected void valueSize(long valPtr, int size) {
+ mem.writeInt(valPtr + 8, size);
+ }
+
+ /**
+ * @param valPtr Value page pointer.
+ * @return Value size.
+ */
+ protected int valueSize(long valPtr) {
+ return mem.readInt(valPtr + 8);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() {
+ for (GridLongList list : allPages)
+ deallocate(list);
+ }
+
+ /**
+ * Reader for key and value.
+ */
+ protected class ReaderBase implements AutoCloseable {
+ /** */
+ private Object tmp;
+
+ /** */
+ private final HadoopSerialization ser;
+
+ /** */
+ private final HadoopDataInStream in = new HadoopDataInStream(mem);
+
+ /**
+ * @param ser Serialization.
+ */
+ protected ReaderBase(HadoopSerialization ser) {
+ assert ser != null;
+
+ this.ser = ser;
+ }
+
+ /**
+ * @param valPtr Value page pointer.
+ * @return Value.
+ */
+ public Object readValue(long valPtr) {
+ assert valPtr > 0 : valPtr;
+
+ try {
+ return read(valPtr + 12, valueSize(valPtr));
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /**
+ * Resets temporary object to the given one.
+ *
+ * @param tmp Temporary object for reuse.
+ */
+ public void resetReusedObject(Object tmp) {
+ this.tmp = tmp;
+ }
+
+ /**
+ * @param ptr Pointer.
+ * @param size Object size.
+ * @return Object.
+ */
+ protected Object read(long ptr, long size) throws IgniteCheckedException {
+ in.buffer().set(ptr, size);
+
+ tmp = ser.read(in, tmp);
+
+ return tmp;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ ser.close();
+ }
+ }
+
+ /**
+ * Base class for adders.
+ */
+ protected abstract class AdderBase implements Adder {
+ /** */
+ protected final HadoopSerialization keySer;
+
+ /** */
+ protected final HadoopSerialization valSer;
+
+ /** */
+ private final HadoopDataOutStream out;
+
+ /** */
+ private long writeStart;
+
+ /** Size and pointer pairs list. */
+ private final GridLongList pages = new GridLongList(16);
+
+ /**
+ * @param ctx Task context.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected AdderBase(HadoopTaskContext ctx) throws IgniteCheckedException {
+ valSer = ctx.valueSerialization();
+ keySer = ctx.keySerialization();
+
+ out = new HadoopDataOutStream(mem) {
+ @Override public long move(long size) {
+ long ptr = super.move(size);
+
+ if (ptr == 0) // Was not able to move - not enough free space.
+ ptr = allocateNextPage(size);
+
+ assert ptr != 0;
+
+ return ptr;
+ }
+ };
+ }
+
+ /**
+ * @param requestedSize Requested size.
+ * @return Next write pointer.
+ */
+ private long allocateNextPage(long requestedSize) {
+ int writtenSize = writtenSize();
+
+ long newPageSize = Math.max(writtenSize + requestedSize, pageSize);
+ long newPagePtr = mem.allocate(newPageSize);
+
+ pages.add(newPageSize);
+ pages.add(newPagePtr);
+
+ HadoopOffheapBuffer b = out.buffer();
+
+ b.set(newPagePtr, newPageSize);
+
+ if (writtenSize != 0) {
+ mem.copyMemory(writeStart, newPagePtr, writtenSize);
+
+ b.move(writtenSize);
+ }
+
+ writeStart = newPagePtr;
+
+ return b.move(requestedSize);
+ }
+
+ /**
+ * @return Fixed pointer.
+ */
+ private long fixAlignment() {
+ HadoopOffheapBuffer b = out.buffer();
+
+ long ptr = b.pointer();
+
+ if ((ptr & 7L) != 0) { // Address is not aligned by octet.
+ ptr = (ptr + 8L) & ~7L;
+
+ b.pointer(ptr);
+ }
+
+ return ptr;
+ }
+
+ /**
+ * @param off Offset.
+ * @param o Object.
+ * @return Page pointer.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected long write(int off, Object o, HadoopSerialization ser) throws IgniteCheckedException {
+ writeStart = fixAlignment();
+
+ if (off != 0)
+ out.move(off);
+
+ ser.write(out, o);
+
+ return writeStart;
+ }
+
+ /**
+ * @param size Size.
+ * @return Pointer.
+ */
+ protected long allocate(int size) {
+ writeStart = fixAlignment();
+
+ out.move(size);
+
+ return writeStart;
+ }
+
+ /**
+ * Rewinds local allocation pointer to the given pointer if possible.
+ *
+ * @param ptr Pointer.
+ */
+ protected void localDeallocate(long ptr) {
+ HadoopOffheapBuffer b = out.buffer();
+
+ if (b.isInside(ptr))
+ b.pointer(ptr);
+ else
+ b.reset();
+ }
+
+ /**
+ * @return Written size.
+ */
+ protected int writtenSize() {
+ return (int)(out.buffer().pointer() - writeStart);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Key addKey(DataInput in, @Nullable Key reuse) throws IgniteCheckedException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ allPages.add(pages);
+
+ keySer.close();
+ valSer.close();
+ }
+ }
+
+ /**
+ * Iterator over values.
+ */
+ protected class ValueIterator implements Iterator<Object> {
+ /** */
+ private long valPtr;
+
+ /** */
+ private final ReaderBase valReader;
+
+ /**
+ * @param valPtr Value page pointer.
+ * @param valReader Value reader.
+ */
+ protected ValueIterator(long valPtr, ReaderBase valReader) {
+ this.valPtr = valPtr;
+ this.valReader = valReader;
+ }
+
+ /**
+ * @param valPtr Head value pointer.
+ */
+ public void head(long valPtr) {
+ this.valPtr = valPtr;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean hasNext() {
+ return valPtr != 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object next() {
+ if (!hasNext())
+ throw new NoSuchElementException();
+
+ Object res = valReader.readValue(valPtr);
+
+ valPtr = nextValue(valPtr);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java
new file mode 100644
index 0000000..69aa7a7
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java
@@ -0,0 +1,726 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.atomic.*;
+
+/**
+ * Skip list.
+ */
+public class HadoopSkipList extends HadoopMultimapBase {
+ /** */
+ private static final int HEADS_SIZE = 24 + 33 * 8; // Offset + max level is from 0 to 32 inclusive.
+
+ /** Top level. */
+ private final AtomicInteger topLevel = new AtomicInteger(-1);
+
+ /** Heads for all the lists. */
+ private final long heads;
+
+ /** */
+ private final AtomicBoolean visitGuard = new AtomicBoolean();
+
+ /**
+ * @param jobInfo Job info.
+ * @param mem Memory.
+ */
+ public HadoopSkipList(HadoopJobInfo jobInfo, GridUnsafeMemory mem) {
+ super(jobInfo, mem);
+
+ heads = mem.allocate(HEADS_SIZE, true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() {
+ super.close();
+
+ mem.release(heads, HEADS_SIZE);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean visit(boolean ignoreLastVisited, Visitor v) throws IgniteCheckedException {
+ if (!visitGuard.compareAndSet(false, true))
+ return false;
+
+ for (long meta = nextMeta(heads, 0); meta != 0L; meta = nextMeta(meta, 0)) {
+ long valPtr = value(meta);
+
+ long lastVisited = ignoreLastVisited ? 0 : lastVisitedValue(meta);
+
+ if (valPtr != lastVisited) {
+ long k = key(meta);
+
+ v.onKey(k + 4, keySize(k));
+
+ lastVisitedValue(meta, valPtr); // Set it to the first value in chain.
+
+ do {
+ v.onValue(valPtr + 12, valueSize(valPtr));
+
+ valPtr = nextValue(valPtr);
+ }
+ while (valPtr != lastVisited);
+ }
+ }
+
+ visitGuard.lazySet(false);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Adder startAdding(HadoopTaskContext ctx) throws IgniteCheckedException {
+ return new AdderImpl(ctx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopTaskInput input(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ Input in = new Input(taskCtx);
+
+ Comparator<Object> grpCmp = taskCtx.groupComparator();
+
+ if (grpCmp != null)
+ return new GroupedInput(grpCmp, in);
+
+ return in;
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @return Key pointer.
+ */
+ private long key(long meta) {
+ return mem.readLong(meta);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @param key Key pointer.
+ */
+ private void key(long meta, long key) {
+ mem.writeLong(meta, key);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @return Value pointer.
+ */
+ private long value(long meta) {
+ return mem.readLongVolatile(meta + 8);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @param valPtr Value pointer.
+ */
+ private void value(long meta, long valPtr) {
+ mem.writeLongVolatile(meta + 8, valPtr);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @param oldValPtr Old first value pointer.
+ * @param newValPtr New first value pointer.
+ * @return {@code true} If operation succeeded.
+ */
+ private boolean casValue(long meta, long oldValPtr, long newValPtr) {
+ return mem.casLong(meta + 8, oldValPtr, newValPtr);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @return Last visited value pointer.
+ */
+ private long lastVisitedValue(long meta) {
+ return mem.readLong(meta + 16);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @param valPtr Last visited value pointer.
+ */
+ private void lastVisitedValue(long meta, long valPtr) {
+ mem.writeLong(meta + 16, valPtr);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @param level Level.
+ * @return Next meta pointer.
+ */
+ private long nextMeta(long meta, int level) {
+ assert meta > 0 : meta;
+
+ return mem.readLongVolatile(meta + 24 + 8 * level);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @param level Level.
+ * @param oldNext Old next meta pointer.
+ * @param newNext New next meta pointer.
+ * @return {@code true} If operation succeeded.
+ */
+ private boolean casNextMeta(long meta, int level, long oldNext, long newNext) {
+ assert meta > 0 : meta;
+
+ return mem.casLong(meta + 24 + 8 * level, oldNext, newNext);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @param level Level.
+ * @param nextMeta Next meta.
+ */
+ private void nextMeta(long meta, int level, long nextMeta) {
+ assert meta != 0;
+
+ mem.writeLong(meta + 24 + 8 * level, nextMeta);
+ }
+
+ /**
+ * @param keyPtr Key pointer.
+ * @return Key size.
+ */
+ private int keySize(long keyPtr) {
+ return mem.readInt(keyPtr);
+ }
+
+ /**
+ * @param keyPtr Key pointer.
+ * @param keySize Key size.
+ */
+ private void keySize(long keyPtr, int keySize) {
+ mem.writeInt(keyPtr, keySize);
+ }
+
+ /**
+ * @param rnd Random.
+ * @return Next level.
+ */
+ public static int randomLevel(Random rnd) {
+ int x = rnd.nextInt();
+
+ int level = 0;
+
+ while ((x & 1) != 0) { // Count sequential 1 bits.
+ level++;
+
+ x >>>= 1;
+ }
+
+ return level;
+ }
+
+ /**
+ * Reader.
+ */
+ private class Reader extends ReaderBase {
+ /**
+ * @param ser Serialization.
+ */
+ protected Reader(HadoopSerialization ser) {
+ super(ser);
+ }
+
+ /**
+ * @param meta Meta pointer.
+ * @return Key.
+ */
+ public Object readKey(long meta) {
+ assert meta > 0 : meta;
+
+ long k = key(meta);
+
+ try {
+ return read(k + 4, keySize(k));
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+ }
+
+ /**
+ * Adder.
+ */
+ private class AdderImpl extends AdderBase {
+ /** */
+ private final Comparator<Object> cmp;
+
+ /** */
+ private final Random rnd = new GridRandom();
+
+ /** */
+ private final GridLongList stack = new GridLongList(16);
+
+ /** */
+ private final Reader keyReader;
+
+ /**
+ * @param ctx Task context.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected AdderImpl(HadoopTaskContext ctx) throws IgniteCheckedException {
+ super(ctx);
+
+ keyReader = new Reader(keySer);
+
+ cmp = ctx.sortComparator();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(Object key, Object val) throws IgniteCheckedException {
+ A.notNull(val, "val");
+
+ add(key, val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Key addKey(DataInput in, @Nullable Key reuse) throws IgniteCheckedException {
+ KeyImpl k = reuse == null ? new KeyImpl() : (KeyImpl)reuse;
+
+ k.tmpKey = keySer.read(in, k.tmpKey);
+
+ k.meta = add(k.tmpKey, null);
+
+ return k;
+ }
+
+ /**
+ * @param key Key.
+ * @param val Value.
+ * @param level Level.
+ * @return Meta pointer.
+ */
+ private long createMeta(long key, long val, int level) {
+ int size = 32 + 8 * level;
+
+ long meta = allocate(size);
+
+ key(meta, key);
+ value(meta, val);
+ lastVisitedValue(meta, 0L);
+
+ for (int i = 32; i < size; i += 8) // Fill with 0.
+ mem.writeLong(meta + i, 0L);
+
+ return meta;
+ }
+
+ /**
+ * @param key Key.
+ * @return Pointer.
+ * @throws IgniteCheckedException If failed.
+ */
+ private long writeKey(Object key) throws IgniteCheckedException {
+ long keyPtr = write(4, key, keySer);
+ int keySize = writtenSize() - 4;
+
+ keySize(keyPtr, keySize);
+
+ return keyPtr;
+ }
+
+ /**
+ * @param prevMeta Previous meta.
+ * @param meta Next meta.
+ */
+ private void stackPush(long prevMeta, long meta) {
+ stack.add(prevMeta);
+ stack.add(meta);
+ }
+
+ /**
+ * Drops last remembered frame from the stack.
+ */
+ private void stackPop() {
+ stack.pop(2);
+ }
+
+ /**
+ * @param key Key.
+ * @param val Value.
+ * @return Meta pointer.
+ * @throws IgniteCheckedException If failed.
+ */
+ private long add(Object key, @Nullable Object val) throws IgniteCheckedException {
+ assert key != null;
+
+ stack.clear();
+
+ long valPtr = 0;
+
+ if (val != null) { // Write value.
+ valPtr = write(12, val, valSer);
+ int valSize = writtenSize() - 12;
+
+ nextValue(valPtr, 0);
+ valueSize(valPtr, valSize);
+ }
+
+ long keyPtr = 0;
+ long newMeta = 0;
+ int newMetaLevel = -1;
+
+ long prevMeta = heads;
+ int level = topLevel.get();
+ long meta = level < 0 ? 0 : nextMeta(heads, level);
+
+ for (;;) {
+ if (level < 0) { // We did not find our key, trying to add new meta.
+ if (keyPtr == 0) { // Write key and create meta only once.
+ keyPtr = writeKey(key);
+
+ newMetaLevel = randomLevel(rnd);
+ newMeta = createMeta(keyPtr, valPtr, newMetaLevel);
+ }
+
+ nextMeta(newMeta, 0, meta); // Set next to new meta before publishing.
+
+ if (casNextMeta(prevMeta, 0, meta, newMeta)) { // New key was added successfully.
+ laceUp(key, newMeta, newMetaLevel);
+
+ return newMeta;
+ }
+ else { // Add failed, need to check out what was added by another thread.
+ meta = nextMeta(prevMeta, level = 0);
+
+ stackPop();
+ }
+ }
+
+ int cmpRes = cmp(key, meta);
+
+ if (cmpRes == 0) { // Key found.
+ if (newMeta != 0) // Deallocate if we've allocated something.
+ localDeallocate(keyPtr);
+
+ if (valPtr == 0) // Only key needs to be added.
+ return meta;
+
+ for (;;) { // Add value for the key found.
+ long nextVal = value(meta);
+
+ nextValue(valPtr, nextVal);
+
+ if (casValue(meta, nextVal, valPtr))
+ return meta;
+ }
+ }
+
+ assert cmpRes != 0;
+
+ if (cmpRes > 0) { // Go right.
+ prevMeta = meta;
+ meta = nextMeta(meta, level);
+
+ if (meta != 0) // If nothing to the right then go down.
+ continue;
+ }
+
+ while (--level >= 0) { // Go down.
+ stackPush(prevMeta, meta); // Remember the path.
+
+ long nextMeta = nextMeta(prevMeta, level);
+
+ if (nextMeta != meta) { // If the meta is the same as on upper level go deeper.
+ meta = nextMeta;
+
+ assert meta != 0;
+
+ break;
+ }
+ }
+ }
+ }
+
+ /**
+ * @param key Key.
+ * @param meta Meta pointer.
+ * @return Comparison result.
+ */
+ @SuppressWarnings("unchecked")
+ private int cmp(Object key, long meta) {
+ assert meta != 0;
+
+ return cmp.compare(key, keyReader.readKey(meta));
+ }
+
+ /**
+ * Adds appropriate index links between metas.
+ *
+ * @param newMeta Just added meta.
+ * @param newMetaLevel New level.
+ */
+ private void laceUp(Object key, long newMeta, int newMetaLevel) {
+ for (int level = 1; level <= newMetaLevel; level++) { // Go from the bottom up.
+ long prevMeta = heads;
+ long meta = 0;
+
+ if (!stack.isEmpty()) { // Get the path back.
+ meta = stack.remove();
+ prevMeta = stack.remove();
+ }
+
+ for (;;) {
+ nextMeta(newMeta, level, meta);
+
+ if (casNextMeta(prevMeta, level, meta, newMeta))
+ break;
+
+ long oldMeta = meta;
+
+ meta = nextMeta(prevMeta, level); // Reread meta.
+
+ for (;;) {
+ int cmpRes = cmp(key, meta);
+
+ if (cmpRes > 0) { // Go right.
+ prevMeta = meta;
+ meta = nextMeta(prevMeta, level);
+
+ if (meta != oldMeta) // Old meta already known to be greater than ours or is 0.
+ continue;
+ }
+
+ assert cmpRes != 0; // Two different metas with equal keys must be impossible.
+
+ break; // Retry cas.
+ }
+ }
+ }
+
+ if (!stack.isEmpty())
+ return; // Our level already lower than top.
+
+ for (;;) { // Raise top level.
+ int top = topLevel.get();
+
+ if (newMetaLevel <= top || topLevel.compareAndSet(top, newMetaLevel))
+ break;
+ }
+ }
+
+ /**
+ * Key.
+ */
+ private class KeyImpl implements Key {
+ /** */
+ private long meta;
+
+ /** */
+ private Object tmpKey;
+
+ /**
+ * @return Meta pointer for the key.
+ */
+ public long address() {
+ return meta;
+ }
+
+ /**
+ * @param val Value.
+ */
+ @Override public void add(Value val) {
+ int size = val.size();
+
+ long valPtr = allocate(size + 12);
+
+ val.copyTo(valPtr + 12);
+
+ valueSize(valPtr, size);
+
+ long nextVal;
+
+ do {
+ nextVal = value(meta);
+
+ nextValue(valPtr, nextVal);
+ }
+ while(!casValue(meta, nextVal, valPtr));
+ }
+ }
+ }
+
+ /**
+ * Task input.
+ */
+ private class Input implements HadoopTaskInput {
+ /** */
+ private long metaPtr = heads;
+
+ /** */
+ private final Reader keyReader;
+
+ /** */
+ private final Reader valReader;
+
+ /**
+ * @param taskCtx Task context.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Input(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ keyReader = new Reader(taskCtx.keySerialization());
+ valReader = new Reader(taskCtx.valueSerialization());
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean next() {
+ metaPtr = nextMeta(metaPtr, 0);
+
+ return metaPtr != 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object key() {
+ return keyReader.readKey(metaPtr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Iterator<?> values() {
+ return new ValueIterator(value(metaPtr), valReader);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ keyReader.close();
+ valReader.close();
+ }
+ }
+
+ /**
+ * Grouped input using grouping comparator.
+ */
+ private class GroupedInput implements HadoopTaskInput {
+ /** */
+ private final Comparator<Object> grpCmp;
+
+ /** */
+ private final Input in;
+
+ /** */
+ private Object prevKey;
+
+ /** */
+ private Object nextKey;
+
+ /** */
+ private final GridLongList vals = new GridLongList();
+
+ /**
+ * @param grpCmp Grouping comparator.
+ * @param in Input.
+ */
+ private GroupedInput(Comparator<Object> grpCmp, Input in) {
+ this.grpCmp = grpCmp;
+ this.in = in;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean next() {
+ if (prevKey == null) { // First call.
+ if (!in.next())
+ return false;
+
+ prevKey = in.key();
+
+ assert prevKey != null;
+
+ in.keyReader.resetReusedObject(null); // We need 2 instances of key object for comparison.
+
+ vals.add(value(in.metaPtr));
+ }
+ else {
+ if (in.metaPtr == 0) // We reached the end of the input.
+ return false;
+
+ vals.clear();
+
+ vals.add(value(in.metaPtr));
+
+ in.keyReader.resetReusedObject(prevKey); // Switch key instances.
+
+ prevKey = nextKey;
+ }
+
+ while (in.next()) { // Fill with head value pointers with equal keys.
+ if (grpCmp.compare(prevKey, nextKey = in.key()) == 0)
+ vals.add(value(in.metaPtr));
+ else
+ break;
+ }
+
+ assert !vals.isEmpty();
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object key() {
+ return prevKey;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Iterator<?> values() {
+ assert !vals.isEmpty();
+
+ final ValueIterator valIter = new ValueIterator(vals.get(0), in.valReader);
+
+ return new Iterator<Object>() {
+ /** */
+ private int idx;
+
+ @Override public boolean hasNext() {
+ if (!valIter.hasNext()) {
+ if (++idx == vals.size())
+ return false;
+
+ valIter.head(vals.get(idx));
+
+ assert valIter.hasNext();
+ }
+
+ return true;
+ }
+
+ @Override public Object next() {
+ return valIter.next();
+ }
+
+ @Override public void remove() {
+ valIter.remove();
+ }
+ };
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ in.close();
+ }
+ }
+}
[17/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopShutdownHookManager.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopShutdownHookManager.java
deleted file mode 100644
index 48558fc..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopShutdownHookManager.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import java.util.*;
-import java.util.concurrent.atomic.*;
-
-/**
- * Fake manager for shutdown hooks.
- */
-public class GridHadoopShutdownHookManager {
- /** */
- private static final GridHadoopShutdownHookManager MGR = new GridHadoopShutdownHookManager();
-
- /**
- * Return <code>ShutdownHookManager</code> singleton.
- *
- * @return <code>ShutdownHookManager</code> singleton.
- */
- public static GridHadoopShutdownHookManager get() {
- return MGR;
- }
-
- /** */
- private Set<Runnable> hooks = Collections.synchronizedSet(new HashSet<Runnable>());
-
- /** */
- private AtomicBoolean shutdownInProgress = new AtomicBoolean(false);
-
- /**
- * Singleton.
- */
- private GridHadoopShutdownHookManager() {
- // No-op.
- }
-
- /**
- * Adds a shutdownHook with a priority, the higher the priority
- * the earlier will run. ShutdownHooks with same priority run
- * in a non-deterministic order.
- *
- * @param shutdownHook shutdownHook <code>Runnable</code>
- * @param priority priority of the shutdownHook.
- */
- public void addShutdownHook(Runnable shutdownHook, int priority) {
- if (shutdownHook == null)
- throw new IllegalArgumentException("shutdownHook cannot be NULL");
-
- hooks.add(shutdownHook);
- }
-
- /**
- * Removes a shutdownHook.
- *
- * @param shutdownHook shutdownHook to remove.
- * @return TRUE if the shutdownHook was registered and removed,
- * FALSE otherwise.
- */
- public boolean removeShutdownHook(Runnable shutdownHook) {
- return hooks.remove(shutdownHook);
- }
-
- /**
- * Indicates if a shutdownHook is registered or not.
- *
- * @param shutdownHook shutdownHook to check if registered.
- * @return TRUE/FALSE depending if the shutdownHook is is registered.
- */
- public boolean hasShutdownHook(Runnable shutdownHook) {
- return hooks.contains(shutdownHook);
- }
-
- /**
- * Indicates if shutdown is in progress or not.
- *
- * @return TRUE if the shutdown is in progress, otherwise FALSE.
- */
- public boolean isShutdownInProgress() {
- return shutdownInProgress.get();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopSplitWrapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopSplitWrapper.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopSplitWrapper.java
deleted file mode 100644
index 57edfa9..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopSplitWrapper.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-
-/**
- * The wrapper for native hadoop input splits.
- *
- * Warning!! This class must not depend on any Hadoop classes directly or indirectly.
- */
-public class GridHadoopSplitWrapper extends GridHadoopInputSplit {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Native hadoop input split. */
- private byte[] bytes;
-
- /** */
- private String clsName;
-
- /** Internal ID */
- private int id;
-
- /**
- * Creates new split wrapper.
- */
- public GridHadoopSplitWrapper() {
- // No-op.
- }
-
- /**
- * Creates new split wrapper.
- *
- * @param id Split ID.
- * @param clsName Class name.
- * @param bytes Serialized class.
- * @param hosts Hosts where split is located.
- */
- public GridHadoopSplitWrapper(int id, String clsName, byte[] bytes, String[] hosts) {
- assert hosts != null;
- assert clsName != null;
- assert bytes != null;
-
- this.hosts = hosts;
- this.id = id;
-
- this.clsName = clsName;
- this.bytes = bytes;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeInt(id);
-
- out.writeUTF(clsName);
- U.writeByteArray(out, bytes);
- }
-
- /**
- * @return Class name.
- */
- public String className() {
- return clsName;
- }
-
- /**
- * @return Class bytes.
- */
- public byte[] bytes() {
- return bytes;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- id = in.readInt();
-
- clsName = in.readUTF();
- bytes = U.readByteArray(in);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- GridHadoopSplitWrapper that = (GridHadoopSplitWrapper)o;
-
- return id == that.id;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return id;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2CleanupTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2CleanupTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2CleanupTask.java
deleted file mode 100644
index 38be3da..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2CleanupTask.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapreduce.JobStatus;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.*;
-
-import java.io.*;
-
-/**
- * Hadoop cleanup task (commits or aborts job).
- */
-public class GridHadoopV2CleanupTask extends GridHadoopV2Task {
- /** Abort flag. */
- private final boolean abort;
-
- /**
- * @param taskInfo Task info.
- * @param abort Abort flag.
- */
- public GridHadoopV2CleanupTask(GridHadoopTaskInfo taskInfo, boolean abort) {
- super(taskInfo);
-
- this.abort = abort;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("ConstantConditions")
- @Override public void run0(GridHadoopV2TaskContext taskCtx) throws IgniteCheckedException {
- JobContextImpl jobCtx = taskCtx.jobContext();
-
- try {
- OutputFormat outputFormat = getOutputFormat(jobCtx);
-
- OutputCommitter committer = outputFormat.getOutputCommitter(hadoopContext());
-
- if (committer != null) {
- if (abort)
- committer.abortJob(jobCtx, JobStatus.State.FAILED);
- else
- committer.commitJob(jobCtx);
- }
- }
- catch (ClassNotFoundException | IOException e) {
- throw new IgniteCheckedException(e);
- }
- catch (InterruptedException e) {
- Thread.currentThread().interrupt();
-
- throw new IgniteInterruptedCheckedException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Context.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Context.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Context.java
deleted file mode 100644
index 287b10f..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Context.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.hadoop.mapreduce.task.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.counter.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Hadoop context implementation for v2 API. It provides IO operations for hadoop tasks.
- */
-public class GridHadoopV2Context extends JobContextImpl implements MapContext, ReduceContext {
- /** Input reader to overriding of GridHadoopTaskContext input. */
- private RecordReader reader;
-
- /** Output writer to overriding of GridHadoopTaskContext output. */
- private RecordWriter writer;
-
- /** Output is provided by executor environment. */
- private final GridHadoopTaskOutput output;
-
- /** Input is provided by executor environment. */
- private final GridHadoopTaskInput input;
-
- /** Unique identifier for a task attempt. */
- private final TaskAttemptID taskAttemptID;
-
- /** Indicates that this task is to be cancelled. */
- private volatile boolean cancelled;
-
- /** Input split. */
- private InputSplit inputSplit;
-
- /** */
- private final GridHadoopTaskContext ctx;
-
- /** */
- private String status;
-
- /**
- * @param ctx Context for IO operations.
- */
- public GridHadoopV2Context(GridHadoopV2TaskContext ctx) {
- super(ctx.jobConf(), ctx.jobContext().getJobID());
-
- taskAttemptID = ctx.attemptId();
-
- conf.set("mapreduce.job.id", taskAttemptID.getJobID().toString());
- conf.set("mapreduce.task.id", taskAttemptID.getTaskID().toString());
-
- output = ctx.output();
- input = ctx.input();
-
- this.ctx = ctx;
- }
-
- /** {@inheritDoc} */
- @Override public InputSplit getInputSplit() {
- if (inputSplit == null) {
- GridHadoopInputSplit split = ctx.taskInfo().inputSplit();
-
- if (split == null)
- return null;
-
- if (split instanceof GridHadoopFileBlock) {
- GridHadoopFileBlock fileBlock = (GridHadoopFileBlock)split;
-
- inputSplit = new FileSplit(new Path(fileBlock.file()), fileBlock.start(), fileBlock.length(), null);
- }
- else if (split instanceof GridHadoopExternalSplit)
- throw new UnsupportedOperationException(); // TODO
- else if (split instanceof GridHadoopSplitWrapper)
- inputSplit = (InputSplit)GridHadoopUtils.unwrapSplit((GridHadoopSplitWrapper)split);
- else
- throw new IllegalStateException();
- }
-
- return inputSplit;
- }
-
- /** {@inheritDoc} */
- @Override public boolean nextKeyValue() throws IOException, InterruptedException {
- if (cancelled)
- throw new GridHadoopTaskCancelledException("Task cancelled.");
-
- return reader.nextKeyValue();
- }
-
- /** {@inheritDoc} */
- @Override public Object getCurrentKey() throws IOException, InterruptedException {
- if (reader != null)
- return reader.getCurrentKey();
-
- return input.key();
- }
-
- /** {@inheritDoc} */
- @Override public Object getCurrentValue() throws IOException, InterruptedException {
- return reader.getCurrentValue();
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public void write(Object key, Object val) throws IOException, InterruptedException {
- if (cancelled)
- throw new GridHadoopTaskCancelledException("Task cancelled.");
-
- if (writer != null)
- writer.write(key, val);
- else {
- try {
- output.write(key, val);
- }
- catch (IgniteCheckedException e) {
- throw new IOException(e);
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public OutputCommitter getOutputCommitter() {
- throw new UnsupportedOperationException();
- }
-
- /** {@inheritDoc} */
- @Override public TaskAttemptID getTaskAttemptID() {
- return taskAttemptID;
- }
-
- /** {@inheritDoc} */
- @Override public void setStatus(String msg) {
- status = msg;
- }
-
- /** {@inheritDoc} */
- @Override public String getStatus() {
- return status;
- }
-
- /** {@inheritDoc} */
- @Override public float getProgress() {
- return 0.5f; // TODO
- }
-
- /** {@inheritDoc} */
- @Override public Counter getCounter(Enum<?> cntrName) {
- return getCounter(cntrName.getDeclaringClass().getName(), cntrName.name());
- }
-
- /** {@inheritDoc} */
- @Override public Counter getCounter(String grpName, String cntrName) {
- return new GridHadoopV2Counter(ctx.counter(grpName, cntrName, GridHadoopLongCounter.class));
- }
-
- /** {@inheritDoc} */
- @Override public void progress() {
- // No-op.
- }
-
- /**
- * Overrides default input data reader.
- *
- * @param reader New reader.
- */
- public void reader(RecordReader reader) {
- this.reader = reader;
- }
-
- /** {@inheritDoc} */
- @Override public boolean nextKey() throws IOException, InterruptedException {
- if (cancelled)
- throw new GridHadoopTaskCancelledException("Task cancelled.");
-
- return input.next();
- }
-
- /** {@inheritDoc} */
- @Override public Iterable getValues() throws IOException, InterruptedException {
- return new Iterable() {
- @Override public Iterator iterator() {
- return input.values();
- }
- };
- }
-
- /**
- * @return Overridden output data writer.
- */
- public RecordWriter writer() {
- return writer;
- }
-
- /**
- * Overrides default output data writer.
- *
- * @param writer New writer.
- */
- public void writer(RecordWriter writer) {
- this.writer = writer;
- }
-
- /**
- * Cancels the task by stop the IO.
- */
- public void cancel() {
- cancelled = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Counter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Counter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Counter.java
deleted file mode 100644
index 6bf8a44..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Counter.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.mapreduce.*;
-import org.apache.ignite.internal.processors.hadoop.counter.*;
-
-import java.io.*;
-
-/**
- * Adapter from own counter implementation into Hadoop API Counter od version 2.0.
- */
-public class GridHadoopV2Counter implements Counter {
- /** Delegate. */
- private final GridHadoopLongCounter cntr;
-
- /**
- * Creates new instance with given delegate.
- *
- * @param cntr Internal counter.
- */
- public GridHadoopV2Counter(GridHadoopLongCounter cntr) {
- assert cntr != null : "counter must be non-null";
-
- this.cntr = cntr;
- }
-
- /** {@inheritDoc} */
- @Override public void setDisplayName(String displayName) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public String getName() {
- return cntr.name();
- }
-
- /** {@inheritDoc} */
- @Override public String getDisplayName() {
- return getName();
- }
-
- /** {@inheritDoc} */
- @Override public long getValue() {
- return cntr.value();
- }
-
- /** {@inheritDoc} */
- @Override public void setValue(long val) {
- cntr.value(val);
- }
-
- /** {@inheritDoc} */
- @Override public void increment(long incr) {
- cntr.increment(incr);
- }
-
- /** {@inheritDoc} */
- @Override public Counter getUnderlyingCounter() {
- return this;
- }
-
- /** {@inheritDoc} */
- @Override public void write(DataOutput out) throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-
- /** {@inheritDoc} */
- @Override public void readFields(DataInput in) throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Job.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Job.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Job.java
deleted file mode 100644
index 7c36948..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Job.java
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapred.JobID;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.split.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.fs.*;
-import org.apache.ignite.internal.processors.hadoop.v1.*;
-import org.apache.ignite.internal.util.future.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jdk8.backport.*;
-
-import java.io.*;
-import java.lang.reflect.*;
-import java.util.*;
-import java.util.Queue;
-import java.util.concurrent.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Hadoop job implementation for v2 API.
- */
-public class GridHadoopV2Job implements GridHadoopJob {
- /** */
- private final JobConf jobConf;
-
- /** */
- private final JobContextImpl jobCtx;
-
- /** Hadoop job ID. */
- private final GridHadoopJobId jobId;
-
- /** Job info. */
- protected GridHadoopJobInfo jobInfo;
-
- /** */
- private final JobID hadoopJobID;
-
- /** */
- private final GridHadoopV2JobResourceManager rsrcMgr;
-
- /** */
- private final ConcurrentMap<T2<GridHadoopTaskType, Integer>, GridFutureAdapter<GridHadoopTaskContext>> ctxs =
- new ConcurrentHashMap8<>();
-
- /** Pooling task context class and thus class loading environment. */
- private final Queue<Class<?>> taskCtxClsPool = new ConcurrentLinkedQueue<>();
-
- /** Local node ID */
- private UUID locNodeId;
-
- /** Serialized JobConf. */
- private volatile byte[] jobConfData;
-
- /**
- * @param jobId Job ID.
- * @param jobInfo Job info.
- * @param log Logger.
- */
- public GridHadoopV2Job(GridHadoopJobId jobId, final GridHadoopDefaultJobInfo jobInfo, IgniteLogger log) {
- assert jobId != null;
- assert jobInfo != null;
-
- this.jobId = jobId;
- this.jobInfo = jobInfo;
-
- hadoopJobID = new JobID(jobId.globalId().toString(), jobId.localId());
-
- GridHadoopClassLoader clsLdr = (GridHadoopClassLoader)getClass().getClassLoader();
-
- // Before create JobConf instance we should set new context class loader.
- Thread.currentThread().setContextClassLoader(clsLdr);
-
- jobConf = new JobConf();
-
- GridHadoopFileSystemsUtils.setupFileSystems(jobConf);
-
- Thread.currentThread().setContextClassLoader(null);
-
- for (Map.Entry<String,String> e : jobInfo.properties().entrySet())
- jobConf.set(e.getKey(), e.getValue());
-
- jobCtx = new JobContextImpl(jobConf, hadoopJobID);
-
- rsrcMgr = new GridHadoopV2JobResourceManager(jobId, jobCtx, log);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopJobId id() {
- return jobId;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopJobInfo info() {
- return jobInfo;
- }
-
- /** {@inheritDoc} */
- @Override public Collection<GridHadoopInputSplit> input() throws IgniteCheckedException {
- Thread.currentThread().setContextClassLoader(jobConf.getClassLoader());
-
- try {
- String jobDirPath = jobConf.get(MRJobConfig.MAPREDUCE_JOB_DIR);
-
- if (jobDirPath == null) { // Probably job was submitted not by hadoop client.
- // Assume that we have needed classes and try to generate input splits ourself.
- if (jobConf.getUseNewMapper())
- return GridHadoopV2Splitter.splitJob(jobCtx);
- else
- return GridHadoopV1Splitter.splitJob(jobConf);
- }
-
- Path jobDir = new Path(jobDirPath);
-
- try (FileSystem fs = FileSystem.get(jobDir.toUri(), jobConf)) {
- JobSplit.TaskSplitMetaInfo[] metaInfos = SplitMetaInfoReader.readSplitMetaInfo(hadoopJobID, fs, jobConf,
- jobDir);
-
- if (F.isEmpty(metaInfos))
- throw new IgniteCheckedException("No input splits found.");
-
- Path splitsFile = JobSubmissionFiles.getJobSplitFile(jobDir);
-
- try (FSDataInputStream in = fs.open(splitsFile)) {
- Collection<GridHadoopInputSplit> res = new ArrayList<>(metaInfos.length);
-
- for (JobSplit.TaskSplitMetaInfo metaInfo : metaInfos) {
- long off = metaInfo.getStartOffset();
-
- String[] hosts = metaInfo.getLocations();
-
- in.seek(off);
-
- String clsName = Text.readString(in);
-
- GridHadoopFileBlock block = GridHadoopV1Splitter.readFileBlock(clsName, in, hosts);
-
- if (block == null)
- block = GridHadoopV2Splitter.readFileBlock(clsName, in, hosts);
-
- res.add(block != null ? block : new GridHadoopExternalSplit(hosts, off));
- }
-
- return res;
- }
- }
- catch (Throwable e) {
- throw transformException(e);
- }
- }
- finally {
- Thread.currentThread().setContextClassLoader(null);
- }
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopTaskContext getTaskContext(GridHadoopTaskInfo info) throws IgniteCheckedException {
- T2<GridHadoopTaskType, Integer> locTaskId = new T2<>(info.type(), info.taskNumber());
-
- GridFutureAdapter<GridHadoopTaskContext> fut = ctxs.get(locTaskId);
-
- if (fut != null)
- return fut.get();
-
- GridFutureAdapter<GridHadoopTaskContext> old = ctxs.putIfAbsent(locTaskId, fut = new GridFutureAdapter<>());
-
- if (old != null)
- return old.get();
-
- Class<?> cls = taskCtxClsPool.poll();
-
- try {
- if (cls == null) {
- // If there is no pooled class, then load new one.
- GridHadoopClassLoader ldr = new GridHadoopClassLoader(rsrcMgr.classPath());
-
- cls = ldr.loadClass(GridHadoopV2TaskContext.class.getName());
- }
-
- Constructor<?> ctr = cls.getConstructor(GridHadoopTaskInfo.class, GridHadoopJob.class,
- GridHadoopJobId.class, UUID.class, DataInput.class);
-
- if (jobConfData == null)
- synchronized(jobConf) {
- if (jobConfData == null) {
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
-
- jobConf.write(new DataOutputStream(buf));
-
- jobConfData = buf.toByteArray();
- }
- }
-
- GridHadoopTaskContext res = (GridHadoopTaskContext)ctr.newInstance(info, this, jobId, locNodeId,
- new DataInputStream(new ByteArrayInputStream(jobConfData)));
-
- fut.onDone(res);
-
- return res;
- }
- catch (Throwable e) {
- IgniteCheckedException te = transformException(e);
-
- fut.onDone(te);
-
- throw te;
- }
- }
-
- /** {@inheritDoc} */
- @Override public void initialize(boolean external, UUID locNodeId) throws IgniteCheckedException {
- this.locNodeId = locNodeId;
-
- Thread.currentThread().setContextClassLoader(jobConf.getClassLoader());
-
- try {
- rsrcMgr.prepareJobEnvironment(!external, jobLocalDir(locNodeId, jobId));
- }
- finally {
- Thread.currentThread().setContextClassLoader(null);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void dispose(boolean external) throws IgniteCheckedException {
- if (rsrcMgr != null && !external) {
- File jobLocDir = jobLocalDir(locNodeId, jobId);
-
- if (jobLocDir.exists())
- U.delete(jobLocDir);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void prepareTaskEnvironment(GridHadoopTaskInfo info) throws IgniteCheckedException {
- rsrcMgr.prepareTaskWorkDir(taskLocalDir(locNodeId, info));
- }
-
- /** {@inheritDoc} */
- @Override public void cleanupTaskEnvironment(GridHadoopTaskInfo info) throws IgniteCheckedException {
- GridHadoopTaskContext ctx = ctxs.remove(new T2<>(info.type(), info.taskNumber())).get();
-
- taskCtxClsPool.offer(ctx.getClass());
-
- File locDir = taskLocalDir(locNodeId, info);
-
- if (locDir.exists())
- U.delete(locDir);
- }
-
- /** {@inheritDoc} */
- @Override public void cleanupStagingDirectory() {
- if (rsrcMgr != null)
- rsrcMgr.cleanupStagingDirectory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2JobResourceManager.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2JobResourceManager.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2JobResourceManager.java
deleted file mode 100644
index be619c7..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2JobResourceManager.java
+++ /dev/null
@@ -1,305 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.util.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.fs.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.net.*;
-import java.nio.file.*;
-import java.util.*;
-
-/**
- * Provides all resources are needed to the job execution. Downloads the main jar, the configuration and additional
- * files are needed to be placed on local files system.
- */
-public class GridHadoopV2JobResourceManager {
- /** Hadoop job context. */
- private final JobContextImpl ctx;
-
- /** Logger. */
- private final IgniteLogger log;
-
- /** Job ID. */
- private final GridHadoopJobId jobId;
-
- /** Class path list. */
- private URL[] clsPath;
-
- /** Set of local resources. */
- private final Collection<File> rsrcSet = new HashSet<>();
-
- /** Staging directory to delivery job jar and config to the work nodes. */
- private Path stagingDir;
-
- /**
- * Creates new instance.
- * @param jobId Job ID.
- * @param ctx Hadoop job context.
- * @param log Logger.
- */
- public GridHadoopV2JobResourceManager(GridHadoopJobId jobId, JobContextImpl ctx, IgniteLogger log) {
- this.jobId = jobId;
- this.ctx = ctx;
- this.log = log.getLogger(GridHadoopV2JobResourceManager.class);
- }
-
- /**
- * Set working directory in local file system.
- *
- * @param dir Working directory.
- * @throws IOException If fails.
- */
- private void setLocalFSWorkingDirectory(File dir) throws IOException {
- JobConf cfg = ctx.getJobConf();
-
- Thread.currentThread().setContextClassLoader(cfg.getClassLoader());
-
- try {
- cfg.set(GridHadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP, dir.getAbsolutePath());
-
- if(!cfg.getBoolean("fs.file.impl.disable.cache", false))
- FileSystem.getLocal(cfg).setWorkingDirectory(new Path(dir.getAbsolutePath()));
- }
- finally {
- Thread.currentThread().setContextClassLoader(null);
- }
- }
-
- /**
- * Prepare job resources. Resolve the classpath list and download it if needed.
- *
- * @param download {@code true} If need to download resources.
- * @param jobLocDir Work directory for the job.
- * @throws IgniteCheckedException If failed.
- */
- public void prepareJobEnvironment(boolean download, File jobLocDir) throws IgniteCheckedException {
- try {
- if (jobLocDir.exists())
- throw new IgniteCheckedException("Local job directory already exists: " + jobLocDir.getAbsolutePath());
-
- JobConf cfg = ctx.getJobConf();
-
- String mrDir = cfg.get("mapreduce.job.dir");
-
- if (mrDir != null) {
- stagingDir = new Path(new URI(mrDir));
-
- if (download) {
- FileSystem fs = FileSystem.get(stagingDir.toUri(), cfg);
-
- if (!fs.exists(stagingDir))
- throw new IgniteCheckedException("Failed to find map-reduce submission directory (does not exist): " +
- stagingDir);
-
- if (!FileUtil.copy(fs, stagingDir, jobLocDir, false, cfg))
- throw new IgniteCheckedException("Failed to copy job submission directory contents to local file system " +
- "[path=" + stagingDir + ", locDir=" + jobLocDir.getAbsolutePath() + ", jobId=" + jobId + ']');
- }
-
- File jarJobFile = new File(jobLocDir, "job.jar");
-
- Collection<URL> clsPathUrls = new ArrayList<>();
-
- clsPathUrls.add(jarJobFile.toURI().toURL());
-
- rsrcSet.add(jarJobFile);
- rsrcSet.add(new File(jobLocDir, "job.xml"));
-
- processFiles(jobLocDir, ctx.getCacheFiles(), download, false, null, MRJobConfig.CACHE_LOCALFILES);
- processFiles(jobLocDir, ctx.getCacheArchives(), download, true, null, MRJobConfig.CACHE_LOCALARCHIVES);
- processFiles(jobLocDir, ctx.getFileClassPaths(), download, false, clsPathUrls, null);
- processFiles(jobLocDir, ctx.getArchiveClassPaths(), download, true, clsPathUrls, null);
-
- if (!clsPathUrls.isEmpty()) {
- clsPath = new URL[clsPathUrls.size()];
-
- clsPathUrls.toArray(clsPath);
- }
- }
- else if (!jobLocDir.mkdirs())
- throw new IgniteCheckedException("Failed to create local job directory: " + jobLocDir.getAbsolutePath());
-
- setLocalFSWorkingDirectory(jobLocDir);
- }
- catch (URISyntaxException | IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /**
- * Process list of resources.
- *
- * @param jobLocDir Job working directory.
- * @param files Array of {@link java.net.URI} or {@link org.apache.hadoop.fs.Path} to process resources.
- * @param download {@code true}, if need to download. Process class path only else.
- * @param extract {@code true}, if need to extract archive.
- * @param clsPathUrls Collection to add resource as classpath resource.
- * @param rsrcNameProp Property for resource name array setting.
- * @throws IOException If failed.
- */
- private void processFiles(File jobLocDir, @Nullable Object[] files, boolean download, boolean extract,
- @Nullable Collection<URL> clsPathUrls, @Nullable String rsrcNameProp) throws IOException {
- if (F.isEmptyOrNulls(files))
- return;
-
- Collection<String> res = new ArrayList<>();
-
- for (Object pathObj : files) {
- String locName = null;
- Path srcPath;
-
- if (pathObj instanceof URI) {
- URI uri = (URI)pathObj;
-
- locName = uri.getFragment();
-
- srcPath = new Path(uri);
- }
- else
- srcPath = (Path)pathObj;
-
- if (locName == null)
- locName = srcPath.getName();
-
- File dstPath = new File(jobLocDir.getAbsolutePath(), locName);
-
- res.add(locName);
-
- rsrcSet.add(dstPath);
-
- if (clsPathUrls != null)
- clsPathUrls.add(dstPath.toURI().toURL());
-
- if (!download)
- continue;
-
- JobConf cfg = ctx.getJobConf();
-
- FileSystem dstFs = FileSystem.getLocal(cfg);
-
- FileSystem srcFs = srcPath.getFileSystem(cfg);
-
- if (extract) {
- File archivesPath = new File(jobLocDir.getAbsolutePath(), ".cached-archives");
-
- if (!archivesPath.exists() && !archivesPath.mkdir())
- throw new IOException("Failed to create directory " +
- "[path=" + archivesPath + ", jobId=" + jobId + ']');
-
- File archiveFile = new File(archivesPath, locName);
-
- FileUtil.copy(srcFs, srcPath, dstFs, new Path(archiveFile.toString()), false, cfg);
-
- String archiveNameLC = archiveFile.getName().toLowerCase();
-
- if (archiveNameLC.endsWith(".jar"))
- RunJar.unJar(archiveFile, dstPath);
- else if (archiveNameLC.endsWith(".zip"))
- FileUtil.unZip(archiveFile, dstPath);
- else if (archiveNameLC.endsWith(".tar.gz") ||
- archiveNameLC.endsWith(".tgz") ||
- archiveNameLC.endsWith(".tar"))
- FileUtil.unTar(archiveFile, dstPath);
- else
- throw new IOException("Cannot unpack archive [path=" + srcPath + ", jobId=" + jobId + ']');
- }
- else
- FileUtil.copy(srcFs, srcPath, dstFs, new Path(dstPath.toString()), false, cfg);
- }
-
- if (!res.isEmpty() && rsrcNameProp != null)
- ctx.getJobConf().setStrings(rsrcNameProp, res.toArray(new String[res.size()]));
- }
-
- /**
- * Prepares working directory for the task.
- *
- * <ul>
- * <li>Creates working directory.</li>
- * <li>Creates symbolic links to all job resources in working directory.</li>
- * </ul>
- *
- * @param path Path to working directory of the task.
- * @throws IgniteCheckedException If fails.
- */
- public void prepareTaskWorkDir(File path) throws IgniteCheckedException {
- try {
- if (path.exists())
- throw new IOException("Task local directory already exists: " + path);
-
- if (!path.mkdir())
- throw new IOException("Failed to create directory: " + path);
-
- for (File resource : rsrcSet) {
- File symLink = new File(path, resource.getName());
-
- try {
- Files.createSymbolicLink(symLink.toPath(), resource.toPath());
- }
- catch (IOException e) {
- String msg = "Unable to create symlink \"" + symLink + "\" to \"" + resource + "\".";
-
- if (U.isWindows() && e instanceof FileSystemException)
- msg += "\n\nAbility to create symbolic links is required!\n" +
- "On Windows platform you have to grant permission 'Create symbolic links'\n" +
- "to your user or run the Accelerator as Administrator.\n";
-
- throw new IOException(msg, e);
- }
- }
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Unable to prepare local working directory for the task " +
- "[jobId=" + jobId + ", path=" + path+ ']', e);
- }
- }
-
- /**
- * Cleans up job staging directory.
- */
- public void cleanupStagingDirectory() {
- try {
- if (stagingDir != null)
- stagingDir.getFileSystem(ctx.getJobConf()).delete(stagingDir, true);
- }
- catch (Exception e) {
- log.error("Failed to remove job staging directory [path=" + stagingDir + ", jobId=" + jobId + ']' , e);
- }
- }
-
- /**
- * Returns array of class path for current job.
- *
- * @return Class path collection.
- */
- @Nullable public URL[] classPath() {
- return clsPath;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2MapTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2MapTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2MapTask.java
deleted file mode 100644
index be0bea2..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2MapTask.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.lib.map.*;
-import org.apache.hadoop.util.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.*;
-
-/**
- * Hadoop map task implementation for v2 API.
- */
-public class GridHadoopV2MapTask extends GridHadoopV2Task {
- /**
- * @param taskInfo Task info.
- */
- public GridHadoopV2MapTask(GridHadoopTaskInfo taskInfo) {
- super(taskInfo);
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings({"ConstantConditions", "unchecked"})
- @Override public void run0(GridHadoopV2TaskContext taskCtx) throws IgniteCheckedException {
- GridHadoopInputSplit split = info().inputSplit();
-
- InputSplit nativeSplit;
-
- if (split instanceof GridHadoopFileBlock) {
- GridHadoopFileBlock block = (GridHadoopFileBlock)split;
-
- nativeSplit = new FileSplit(new Path(block.file().toString()), block.start(), block.length(), null);
- }
- else
- nativeSplit = (InputSplit)taskCtx.getNativeSplit(split);
-
- assert nativeSplit != null;
-
- OutputFormat outputFormat = null;
- Exception err = null;
-
- JobContextImpl jobCtx = taskCtx.jobContext();
-
- try {
- InputFormat inFormat = ReflectionUtils.newInstance(jobCtx.getInputFormatClass(),
- hadoopContext().getConfiguration());
-
- RecordReader reader = inFormat.createRecordReader(nativeSplit, hadoopContext());
-
- reader.initialize(nativeSplit, hadoopContext());
-
- hadoopContext().reader(reader);
-
- GridHadoopJobInfo jobInfo = taskCtx.job().info();
-
- outputFormat = jobInfo.hasCombiner() || jobInfo.hasReducer() ? null : prepareWriter(jobCtx);
-
- Mapper mapper = ReflectionUtils.newInstance(jobCtx.getMapperClass(), hadoopContext().getConfiguration());
-
- try {
- mapper.run(new WrappedMapper().getMapContext(hadoopContext()));
- }
- finally {
- closeWriter();
- }
-
- commit(outputFormat);
- }
- catch (InterruptedException e) {
- err = e;
-
- Thread.currentThread().interrupt();
-
- throw new IgniteInterruptedCheckedException(e);
- }
- catch (Exception e) {
- err = e;
-
- throw new IgniteCheckedException(e);
- }
- finally {
- if (err != null)
- abort(outputFormat);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Partitioner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Partitioner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Partitioner.java
deleted file mode 100644
index 0883520..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Partitioner.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.util.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-
-/**
- * Hadoop partitioner adapter for v2 API.
- */
-public class GridHadoopV2Partitioner implements GridHadoopPartitioner {
- /** Partitioner instance. */
- private Partitioner<Object, Object> part;
-
- /**
- * @param cls Hadoop partitioner class.
- * @param conf Job configuration.
- */
- public GridHadoopV2Partitioner(Class<? extends Partitioner<?, ?>> cls, Configuration conf) {
- part = (Partitioner<Object, Object>) ReflectionUtils.newInstance(cls, conf);
- }
-
- /** {@inheritDoc} */
- @Override public int partition(Object key, Object val, int parts) {
- return part.getPartition(key, val, parts);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2ReduceTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2ReduceTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2ReduceTask.java
deleted file mode 100644
index 146e05c..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2ReduceTask.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.reduce.*;
-import org.apache.hadoop.util.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.*;
-
-/**
- * Hadoop reduce task implementation for v2 API.
- */
-public class GridHadoopV2ReduceTask extends GridHadoopV2Task {
- /** {@code True} if reduce, {@code false} if combine. */
- private final boolean reduce;
-
- /**
- * Constructor.
- *
- * @param taskInfo Task info.
- * @param reduce {@code True} if reduce, {@code false} if combine.
- */
- public GridHadoopV2ReduceTask(GridHadoopTaskInfo taskInfo, boolean reduce) {
- super(taskInfo);
-
- this.reduce = reduce;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings({"ConstantConditions", "unchecked"})
- @Override public void run0(GridHadoopV2TaskContext taskCtx) throws IgniteCheckedException {
- OutputFormat outputFormat = null;
- Exception err = null;
-
- JobContextImpl jobCtx = taskCtx.jobContext();
-
- try {
- outputFormat = reduce || !taskCtx.job().info().hasReducer() ? prepareWriter(jobCtx) : null;
-
- Reducer reducer = ReflectionUtils.newInstance(reduce ? jobCtx.getReducerClass() : jobCtx.getCombinerClass(),
- jobCtx.getConfiguration());
-
- try {
- reducer.run(new WrappedReducer().getReducerContext(hadoopContext()));
- }
- finally {
- closeWriter();
- }
-
- commit(outputFormat);
- }
- catch (InterruptedException e) {
- err = e;
-
- Thread.currentThread().interrupt();
-
- throw new IgniteInterruptedCheckedException(e);
- }
- catch (Exception e) {
- err = e;
-
- throw new IgniteCheckedException(e);
- }
- finally {
- if (err != null)
- abort(outputFormat);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2SetupTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2SetupTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2SetupTask.java
deleted file mode 100644
index 54eda25..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2SetupTask.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.*;
-
-import java.io.*;
-
-/**
- * Hadoop setup task (prepares job).
- */
-public class GridHadoopV2SetupTask extends GridHadoopV2Task {
- /**
- * Constructor.
- *
- * @param taskInfo task info.
- */
- public GridHadoopV2SetupTask(GridHadoopTaskInfo taskInfo) {
- super(taskInfo);
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("ConstantConditions")
- @Override protected void run0(GridHadoopV2TaskContext taskCtx) throws IgniteCheckedException {
- try {
- JobContextImpl jobCtx = taskCtx.jobContext();
-
- OutputFormat outputFormat = getOutputFormat(jobCtx);
-
- outputFormat.checkOutputSpecs(jobCtx);
-
- OutputCommitter committer = outputFormat.getOutputCommitter(hadoopContext());
-
- if (committer != null)
- committer.setupJob(jobCtx);
- }
- catch (ClassNotFoundException | IOException e) {
- throw new IgniteCheckedException(e);
- }
- catch (InterruptedException e) {
- Thread.currentThread().interrupt();
-
- throw new IgniteInterruptedCheckedException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Splitter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Splitter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Splitter.java
deleted file mode 100644
index e8ce70b..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Splitter.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.hadoop.util.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Hadoop API v2 splitter.
- */
-public class GridHadoopV2Splitter {
- /** */
- private static final String[] EMPTY_HOSTS = {};
-
- /**
- * @param ctx Job context.
- * @return Collection of mapped splits.
- * @throws IgniteCheckedException If mapping failed.
- */
- public static Collection<GridHadoopInputSplit> splitJob(JobContext ctx) throws IgniteCheckedException {
- try {
- InputFormat<?, ?> format = ReflectionUtils.newInstance(ctx.getInputFormatClass(), ctx.getConfiguration());
-
- assert format != null;
-
- List<InputSplit> splits = format.getSplits(ctx);
-
- Collection<GridHadoopInputSplit> res = new ArrayList<>(splits.size());
-
- int id = 0;
-
- for (InputSplit nativeSplit : splits) {
- if (nativeSplit instanceof FileSplit) {
- FileSplit s = (FileSplit)nativeSplit;
-
- res.add(new GridHadoopFileBlock(s.getLocations(), s.getPath().toUri(), s.getStart(), s.getLength()));
- }
- else
- res.add(GridHadoopUtils.wrapSplit(id, nativeSplit, nativeSplit.getLocations()));
-
- id++;
- }
-
- return res;
- }
- catch (IOException | ClassNotFoundException e) {
- throw new IgniteCheckedException(e);
- }
- catch (InterruptedException e) {
- Thread.currentThread().interrupt();
-
- throw new IgniteInterruptedCheckedException(e);
- }
- }
-
- /**
- * @param clsName Input split class name.
- * @param in Input stream.
- * @param hosts Optional hosts.
- * @return File block or {@code null} if it is not a {@link FileSplit} instance.
- * @throws IgniteCheckedException If failed.
- */
- public static GridHadoopFileBlock readFileBlock(String clsName, DataInput in, @Nullable String[] hosts)
- throws IgniteCheckedException {
- if (!FileSplit.class.getName().equals(clsName))
- return null;
-
- FileSplit split = new FileSplit();
-
- try {
- split.readFields(in);
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
-
- if (hosts == null)
- hosts = EMPTY_HOSTS;
-
- return new GridHadoopFileBlock(hosts, split.getPath().toUri(), split.getStart(), split.getLength());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Task.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Task.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Task.java
deleted file mode 100644
index 37697c6..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2Task.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.util.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * Extended Hadoop v2 task.
- */
-public abstract class GridHadoopV2Task extends GridHadoopTask {
- /** Hadoop context. */
- private GridHadoopV2Context hadoopCtx;
-
- /**
- * Constructor.
- *
- * @param taskInfo Task info.
- */
- protected GridHadoopV2Task(GridHadoopTaskInfo taskInfo) {
- super(taskInfo);
- }
-
- /** {@inheritDoc} */
- @Override public void run(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- GridHadoopV2TaskContext ctx = (GridHadoopV2TaskContext)taskCtx;
-
- hadoopCtx = new GridHadoopV2Context(ctx);
-
- run0(ctx);
- }
-
- /**
- * Internal task routine.
- *
- * @param taskCtx Task context.
- * @throws IgniteCheckedException
- */
- protected abstract void run0(GridHadoopV2TaskContext taskCtx) throws IgniteCheckedException;
-
- /**
- * @return hadoop context.
- */
- protected GridHadoopV2Context hadoopContext() {
- return hadoopCtx;
- }
-
- /**
- * Create and configure an OutputFormat instance.
- *
- * @param jobCtx Job context.
- * @return Instance of OutputFormat is specified in job configuration.
- * @throws ClassNotFoundException If specified class not found.
- */
- protected OutputFormat getOutputFormat(JobContext jobCtx) throws ClassNotFoundException {
- return ReflectionUtils.newInstance(jobCtx.getOutputFormatClass(), hadoopContext().getConfiguration());
- }
-
- /**
- * Put write into Hadoop context and return associated output format instance.
- *
- * @param jobCtx Job context.
- * @return Output format.
- * @throws IgniteCheckedException In case of Grid exception.
- * @throws InterruptedException In case of interrupt.
- */
- protected OutputFormat prepareWriter(JobContext jobCtx)
- throws IgniteCheckedException, InterruptedException {
- try {
- OutputFormat outputFormat = getOutputFormat(jobCtx);
-
- assert outputFormat != null;
-
- OutputCommitter outCommitter = outputFormat.getOutputCommitter(hadoopCtx);
-
- if (outCommitter != null)
- outCommitter.setupTask(hadoopCtx);
-
- RecordWriter writer = outputFormat.getRecordWriter(hadoopCtx);
-
- hadoopCtx.writer(writer);
-
- return outputFormat;
- }
- catch (IOException | ClassNotFoundException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /**
- * Closes writer.
- *
- * @throws Exception If fails and logger hasn't been specified.
- */
- protected void closeWriter() throws Exception {
- RecordWriter writer = hadoopCtx.writer();
-
- if (writer != null)
- writer.close(hadoopCtx);
- }
-
- /**
- * Setup task.
- *
- * @param outputFormat Output format.
- * @throws IOException In case of IO exception.
- * @throws InterruptedException In case of interrupt.
- */
- protected void setup(@Nullable OutputFormat outputFormat) throws IOException, InterruptedException {
- if (hadoopCtx.writer() != null) {
- assert outputFormat != null;
-
- outputFormat.getOutputCommitter(hadoopCtx).setupTask(hadoopCtx);
- }
- }
-
- /**
- * Commit task.
- *
- * @param outputFormat Output format.
- * @throws IgniteCheckedException In case of Grid exception.
- * @throws IOException In case of IO exception.
- * @throws InterruptedException In case of interrupt.
- */
- protected void commit(@Nullable OutputFormat outputFormat) throws IgniteCheckedException, IOException, InterruptedException {
- if (hadoopCtx.writer() != null) {
- assert outputFormat != null;
-
- OutputCommitter outputCommitter = outputFormat.getOutputCommitter(hadoopCtx);
-
- if (outputCommitter.needsTaskCommit(hadoopCtx))
- outputCommitter.commitTask(hadoopCtx);
- }
- }
-
- /**
- * Abort task.
- *
- * @param outputFormat Output format.
- */
- protected void abort(@Nullable OutputFormat outputFormat) {
- if (hadoopCtx.writer() != null) {
- assert outputFormat != null;
-
- try {
- outputFormat.getOutputCommitter(hadoopCtx).abortTask(hadoopCtx);
- }
- catch (IOException ignore) {
- // Ignore.
- }
- catch (InterruptedException ignore) {
- Thread.currentThread().interrupt();
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public void cancel() {
- hadoopCtx.cancel();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2TaskContext.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2TaskContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2TaskContext.java
deleted file mode 100644
index 41bd24a..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopV2TaskContext.java
+++ /dev/null
@@ -1,443 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.io.serializer.*;
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapred.JobID;
-import org.apache.hadoop.mapred.TaskAttemptID;
-import org.apache.hadoop.mapred.TaskID;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.counter.*;
-import org.apache.ignite.internal.processors.hadoop.fs.*;
-import org.apache.ignite.internal.processors.hadoop.v1.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-
-import static org.apache.ignite.igfs.hadoop.IgfsHadoopParameters.*;
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Context for task execution.
- */
-public class GridHadoopV2TaskContext extends GridHadoopTaskContext {
- /** */
- private static final boolean COMBINE_KEY_GROUPING_SUPPORTED;
-
- /**
- * Check for combiner grouping support (available since Hadoop 2.3).
- */
- static {
- boolean ok;
-
- try {
- JobContext.class.getDeclaredMethod("getCombinerKeyGroupingComparator");
-
- ok = true;
- }
- catch (NoSuchMethodException ignore) {
- ok = false;
- }
-
- COMBINE_KEY_GROUPING_SUPPORTED = ok;
- }
-
- /** Flag is set if new context-object code is used for running the mapper. */
- private final boolean useNewMapper;
-
- /** Flag is set if new context-object code is used for running the reducer. */
- private final boolean useNewReducer;
-
- /** Flag is set if new context-object code is used for running the combiner. */
- private final boolean useNewCombiner;
-
- /** */
- private final JobContextImpl jobCtx;
-
- /** Set if task is to cancelling. */
- private volatile boolean cancelled;
-
- /** Current task. */
- private volatile GridHadoopTask task;
-
- /** Local node ID */
- private UUID locNodeId;
-
- /** Counters for task. */
- private final GridHadoopCounters cntrs = new GridHadoopCountersImpl();
-
- /**
- * @param taskInfo Task info.
- * @param job Job.
- * @param jobId Job ID.
- * @param locNodeId Local node ID.
- * @param jobConfDataInput DataInput for read JobConf.
- */
- public GridHadoopV2TaskContext(GridHadoopTaskInfo taskInfo, GridHadoopJob job, GridHadoopJobId jobId,
- @Nullable UUID locNodeId, DataInput jobConfDataInput) throws IgniteCheckedException {
- super(taskInfo, job);
- this.locNodeId = locNodeId;
-
- // Before create JobConf instance we should set new context class loader.
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
-
- try {
- JobConf jobConf = new JobConf();
-
- try {
- jobConf.readFields(jobConfDataInput);
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
-
- // For map-reduce jobs prefer local writes.
- jobConf.setBooleanIfUnset(PARAM_IGFS_PREFER_LOCAL_WRITES, true);
-
- jobCtx = new JobContextImpl(jobConf, new JobID(jobId.globalId().toString(), jobId.localId()));
-
- useNewMapper = jobConf.getUseNewMapper();
- useNewReducer = jobConf.getUseNewReducer();
- useNewCombiner = jobConf.getCombinerClass() == null;
- }
- finally {
- Thread.currentThread().setContextClassLoader(null);
- }
- }
-
- /** {@inheritDoc} */
- @Override public <T extends GridHadoopCounter> T counter(String grp, String name, Class<T> cls) {
- return cntrs.counter(grp, name, cls);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopCounters counters() {
- return cntrs;
- }
-
- /**
- * Creates appropriate task from current task info.
- *
- * @return Task.
- */
- private GridHadoopTask createTask() {
- boolean isAbort = taskInfo().type() == GridHadoopTaskType.ABORT;
-
- switch (taskInfo().type()) {
- case SETUP:
- return useNewMapper ? new GridHadoopV2SetupTask(taskInfo()) : new GridHadoopV1SetupTask(taskInfo());
-
- case MAP:
- return useNewMapper ? new GridHadoopV2MapTask(taskInfo()) : new GridHadoopV1MapTask(taskInfo());
-
- case REDUCE:
- return useNewReducer ? new GridHadoopV2ReduceTask(taskInfo(), true) :
- new GridHadoopV1ReduceTask(taskInfo(), true);
-
- case COMBINE:
- return useNewCombiner ? new GridHadoopV2ReduceTask(taskInfo(), false) :
- new GridHadoopV1ReduceTask(taskInfo(), false);
-
- case COMMIT:
- case ABORT:
- return useNewReducer ? new GridHadoopV2CleanupTask(taskInfo(), isAbort) :
- new GridHadoopV1CleanupTask(taskInfo(), isAbort);
-
- default:
- return null;
- }
- }
-
- /** {@inheritDoc} */
- @Override public void run() throws IgniteCheckedException {
- try {
- Thread.currentThread().setContextClassLoader(jobConf().getClassLoader());
-
- try {
- task = createTask();
- }
- catch (Throwable e) {
- throw transformException(e);
- }
-
- if (cancelled)
- throw new GridHadoopTaskCancelledException("Task cancelled.");
-
- try {
- task.run(this);
- }
- catch (Throwable e) {
- throw transformException(e);
- }
- }
- finally {
- task = null;
-
- Thread.currentThread().setContextClassLoader(null);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void cancel() {
- cancelled = true;
-
- GridHadoopTask t = task;
-
- if (t != null)
- t.cancel();
- }
-
- /** {@inheritDoc} */
- @Override public void prepareTaskEnvironment() throws IgniteCheckedException {
- File locDir;
-
- switch(taskInfo().type()) {
- case MAP:
- case REDUCE:
- job().prepareTaskEnvironment(taskInfo());
-
- locDir = taskLocalDir(locNodeId, taskInfo());
-
- break;
-
- default:
- locDir = jobLocalDir(locNodeId, taskInfo().jobId());
- }
-
- Thread.currentThread().setContextClassLoader(jobConf().getClassLoader());
-
- try {
- FileSystem fs = FileSystem.get(jobConf());
-
- GridHadoopFileSystemsUtils.setUser(fs, jobConf().getUser());
-
- LocalFileSystem locFs = FileSystem.getLocal(jobConf());
-
- locFs.setWorkingDirectory(new Path(locDir.getAbsolutePath()));
- }
- catch (Throwable e) {
- throw transformException(e);
- }
- finally {
- Thread.currentThread().setContextClassLoader(null);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void cleanupTaskEnvironment() throws IgniteCheckedException {
- job().cleanupTaskEnvironment(taskInfo());
- }
-
- /**
- * Creates Hadoop attempt ID.
- *
- * @return Attempt ID.
- */
- public TaskAttemptID attemptId() {
- TaskID tid = new TaskID(jobCtx.getJobID(), taskType(taskInfo().type()), taskInfo().taskNumber());
-
- return new TaskAttemptID(tid, taskInfo().attempt());
- }
-
- /**
- * @param type Task type.
- * @return Hadoop task type.
- */
- private TaskType taskType(GridHadoopTaskType type) {
- switch (type) {
- case SETUP:
- return TaskType.JOB_SETUP;
- case MAP:
- case COMBINE:
- return TaskType.MAP;
-
- case REDUCE:
- return TaskType.REDUCE;
-
- case COMMIT:
- case ABORT:
- return TaskType.JOB_CLEANUP;
-
- default:
- return null;
- }
- }
-
- /**
- * Gets job configuration of the task.
- *
- * @return Job configuration.
- */
- public JobConf jobConf() {
- return jobCtx.getJobConf();
- }
-
- /**
- * Gets job context of the task.
- *
- * @return Job context.
- */
- public JobContextImpl jobContext() {
- return jobCtx;
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopPartitioner partitioner() throws IgniteCheckedException {
- Class<?> partClsOld = jobConf().getClass("mapred.partitioner.class", null);
-
- if (partClsOld != null)
- return new GridHadoopV1Partitioner(jobConf().getPartitionerClass(), jobConf());
-
- try {
- return new GridHadoopV2Partitioner(jobCtx.getPartitionerClass(), jobConf());
- }
- catch (ClassNotFoundException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /**
- * Gets serializer for specified class.
- *
- * @param cls Class.
- * @param jobConf Job configuration.
- * @return Appropriate serializer.
- */
- @SuppressWarnings("unchecked")
- private GridHadoopSerialization getSerialization(Class<?> cls, Configuration jobConf) throws IgniteCheckedException {
- A.notNull(cls, "cls");
-
- SerializationFactory factory = new SerializationFactory(jobConf);
-
- Serialization<?> serialization = factory.getSerialization(cls);
-
- if (serialization == null)
- throw new IgniteCheckedException("Failed to find serialization for: " + cls.getName());
-
- if (serialization.getClass() == WritableSerialization.class)
- return new GridHadoopWritableSerialization((Class<? extends Writable>)cls);
-
- return new GridHadoopSerializationWrapper(serialization, cls);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopSerialization keySerialization() throws IgniteCheckedException {
- return getSerialization(jobCtx.getMapOutputKeyClass(), jobConf());
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopSerialization valueSerialization() throws IgniteCheckedException {
- return getSerialization(jobCtx.getMapOutputValueClass(), jobConf());
- }
-
- /** {@inheritDoc} */
- @Override public Comparator<Object> sortComparator() {
- return (Comparator<Object>)jobCtx.getSortComparator();
- }
-
- /** {@inheritDoc} */
- @Override public Comparator<Object> groupComparator() {
- Comparator<?> res;
-
- switch (taskInfo().type()) {
- case COMBINE:
- res = COMBINE_KEY_GROUPING_SUPPORTED ?
- jobContext().getCombinerKeyGroupingComparator() : jobContext().getGroupingComparator();
-
- break;
-
- case REDUCE:
- res = jobContext().getGroupingComparator();
-
- break;
-
- default:
- return null;
- }
-
- if (res != null && res.getClass() != sortComparator().getClass())
- return (Comparator<Object>)res;
-
- return null;
- }
-
- /**
- * @param split Split.
- * @return Native Hadoop split.
- * @throws IgniteCheckedException if failed.
- */
- @SuppressWarnings("unchecked")
- public Object getNativeSplit(GridHadoopInputSplit split) throws IgniteCheckedException {
- if (split instanceof GridHadoopExternalSplit)
- return readExternalSplit((GridHadoopExternalSplit)split);
-
- if (split instanceof GridHadoopSplitWrapper)
- return unwrapSplit((GridHadoopSplitWrapper)split);
-
- throw new IllegalStateException("Unknown split: " + split);
- }
-
- /**
- * @param split External split.
- * @return Native input split.
- * @throws IgniteCheckedException If failed.
- */
- @SuppressWarnings("unchecked")
- private Object readExternalSplit(GridHadoopExternalSplit split) throws IgniteCheckedException {
- Path jobDir = new Path(jobConf().get(MRJobConfig.MAPREDUCE_JOB_DIR));
-
- try (FileSystem fs = FileSystem.get(jobDir.toUri(), jobConf());
- FSDataInputStream in = fs.open(JobSubmissionFiles.getJobSplitFile(jobDir))) {
-
- in.seek(split.offset());
-
- String clsName = Text.readString(in);
-
- Class<?> cls = jobConf().getClassByName(clsName);
-
- assert cls != null;
-
- Serialization serialization = new SerializationFactory(jobConf()).getSerialization(cls);
-
- Deserializer deserializer = serialization.getDeserializer(cls);
-
- deserializer.open(in);
-
- Object res = deserializer.deserialize(null);
-
- deserializer.close();
-
- assert res != null;
-
- return res;
- }
- catch (IOException | ClassNotFoundException e) {
- throw new IgniteCheckedException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopWritableSerialization.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopWritableSerialization.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopWritableSerialization.java
deleted file mode 100644
index 4361ad4..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/GridHadoopWritableSerialization.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.v2;
-
-import org.apache.hadoop.io.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * Optimized serialization for Hadoop {@link Writable} types.
- */
-public class GridHadoopWritableSerialization implements GridHadoopSerialization {
- /** */
- private final Class<? extends Writable> cls;
-
- /**
- * @param cls Class.
- */
- public GridHadoopWritableSerialization(Class<? extends Writable> cls) {
- assert cls != null;
-
- this.cls = cls;
- }
-
- /** {@inheritDoc} */
- @Override public void write(DataOutput out, Object obj) throws IgniteCheckedException {
- assert cls.isAssignableFrom(obj.getClass()) : cls + " " + obj.getClass();
-
- try {
- ((Writable)obj).write(out);
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Object read(DataInput in, @Nullable Object obj) throws IgniteCheckedException {
- Writable w = obj == null ? U.newInstance(cls) : cls.cast(obj);
-
- try {
- w.readFields(in);
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
-
- return w;
- }
-
- /** {@inheritDoc} */
- @Override public void close() {
- // No-op.
- }
-}
[42/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReducePlanner.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReducePlanner.java
deleted file mode 100644
index 56c6913..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReducePlanner.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.cluster.*;
-import org.jetbrains.annotations.*;
-
-import java.util.*;
-
-/**
- * Map-reduce execution planner.
- */
-public interface GridHadoopMapReducePlanner {
- /**
- * Prepares map-reduce execution plan for the given job and topology.
- *
- * @param job Job.
- * @param top Topology.
- * @param oldPlan Old plan in case of partial failure.
- * @return Map reduce plan.
- */
- public GridHadoopMapReducePlan preparePlan(GridHadoopJob job, Collection<ClusterNode> top,
- @Nullable GridHadoopMapReducePlan oldPlan) throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopPartitioner.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopPartitioner.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopPartitioner.java
deleted file mode 100644
index fcde424..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopPartitioner.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-/**
- * Partitioner.
- */
-public interface GridHadoopPartitioner {
- /**
- * Gets partition which is actually a reducer index for the given key and value pair.
- *
- * @param key Key.
- * @param val Value.
- * @param parts Number of partitions.
- * @return Partition.
- */
- public int partition(Object key, Object val, int parts);
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSerialization.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSerialization.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSerialization.java
deleted file mode 100644
index 5bc8806..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSerialization.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * Hadoop serialization. Not thread safe object, must be created for each thread or correctly synchronized.
- */
-public interface GridHadoopSerialization extends AutoCloseable {
- /**
- * Writes the given object to output.
- *
- * @param out Output.
- * @param obj Object to serialize.
- * @throws IgniteCheckedException If failed.
- */
- public void write(DataOutput out, Object obj) throws IgniteCheckedException;
-
- /**
- * Reads object from the given input optionally reusing given instance.
- *
- * @param in Input.
- * @param obj Object.
- * @return New object or reused instance.
- * @throws IgniteCheckedException If failed.
- */
- public Object read(DataInput in, @Nullable Object obj) throws IgniteCheckedException;
-
- /**
- * Finalise the internal objects.
- *
- * @throws IgniteCheckedException If failed.
- */
- @Override public void close() throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTask.java
deleted file mode 100644
index be34f81..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTask.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-
-import java.io.*;
-
-/**
- * Hadoop task.
- */
-public abstract class GridHadoopTask {
- /** */
- private GridHadoopTaskInfo taskInfo;
-
- /**
- * Creates task.
- *
- * @param taskInfo Task info.
- */
- protected GridHadoopTask(GridHadoopTaskInfo taskInfo) {
- assert taskInfo != null;
-
- this.taskInfo = taskInfo;
- }
-
- /**
- * For {@link Externalizable}.
- */
- @SuppressWarnings("ConstructorNotProtectedInAbstractClass")
- public GridHadoopTask() {
- // No-op.
- }
-
- /**
- * Gets task info.
- *
- * @return Task info.
- */
- public GridHadoopTaskInfo info() {
- return taskInfo;
- }
-
- /**
- * Runs task.
- *
- * @param taskCtx Context.
- * @throws org.apache.ignite.internal.IgniteInterruptedCheckedException If interrupted.
- * @throws IgniteCheckedException If failed.
- */
- public abstract void run(GridHadoopTaskContext taskCtx) throws IgniteCheckedException;
-
- /**
- * Interrupts task execution.
- */
- public abstract void cancel();
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskContext.java
deleted file mode 100644
index bedd93b..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskContext.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-
-import java.util.*;
-
-/**
- * Task context.
- */
-public abstract class GridHadoopTaskContext {
- /** */
- private final GridHadoopJob job;
-
- /** */
- private GridHadoopTaskInput input;
-
- /** */
- private GridHadoopTaskOutput output;
-
- /** */
- private GridHadoopTaskInfo taskInfo;
-
- /**
- * @param taskInfo Task info.
- * @param job Job.
- */
- protected GridHadoopTaskContext(GridHadoopTaskInfo taskInfo, GridHadoopJob job) {
- this.taskInfo = taskInfo;
- this.job = job;
- }
-
- /**
- * Gets task info.
- *
- * @return Task info.
- */
- public GridHadoopTaskInfo taskInfo() {
- return taskInfo;
- }
-
- /**
- * Set a new task info.
- *
- * @param info Task info.
- */
- public void taskInfo(GridHadoopTaskInfo info) {
- taskInfo = info;
- }
-
- /**
- * Gets task output.
- *
- * @return Task output.
- */
- public GridHadoopTaskOutput output() {
- return output;
- }
-
- /**
- * Gets task input.
- *
- * @return Task input.
- */
- public GridHadoopTaskInput input() {
- return input;
- }
-
- /**
- * @return Job.
- */
- public GridHadoopJob job() {
- return job;
- }
-
- /**
- * Gets counter for the given name.
- *
- * @param grp Counter group's name.
- * @param name Counter name.
- * @return Counter.
- */
- public abstract <T extends GridHadoopCounter> T counter(String grp, String name, Class<T> cls);
-
- /**
- * Gets all known counters.
- *
- * @return Unmodifiable collection of counters.
- */
- public abstract GridHadoopCounters counters();
-
- /**
- * Sets input of the task.
- *
- * @param in Input.
- */
- public void input(GridHadoopTaskInput in) {
- input = in;
- }
-
- /**
- * Sets output of the task.
- *
- * @param out Output.
- */
- public void output(GridHadoopTaskOutput out) {
- output = out;
- }
-
- /**
- * Gets partitioner.
- *
- * @return Partitioner.
- * @throws IgniteCheckedException If failed.
- */
- public abstract GridHadoopPartitioner partitioner() throws IgniteCheckedException;
-
- /**
- * Gets serializer for values.
- *
- * @return Serializer for keys.
- * @throws IgniteCheckedException If failed.
- */
- public abstract GridHadoopSerialization keySerialization() throws IgniteCheckedException;
-
- /**
- * Gets serializer for values.
- *
- * @return Serializer for values.
- * @throws IgniteCheckedException If failed.
- */
- public abstract GridHadoopSerialization valueSerialization() throws IgniteCheckedException;
-
- /**
- * Gets sorting comparator.
- *
- * @return Comparator for sorting.
- */
- public abstract Comparator<Object> sortComparator();
-
- /**
- * Gets comparator for grouping on combine or reduce operation.
- *
- * @return Comparator.
- */
- public abstract Comparator<Object> groupComparator();
-
- /**
- * Execute current task.
- *
- * @throws IgniteCheckedException If failed.
- */
- public abstract void run() throws IgniteCheckedException;
-
- /**
- * Cancel current task execution.
- */
- public abstract void cancel();
-
- /**
- * Prepare local environment for the task.
- *
- * @throws IgniteCheckedException If failed.
- */
- public abstract void prepareTaskEnvironment() throws IgniteCheckedException;
-
- /**
- * Cleans up local environment of the task.
- *
- * @throws IgniteCheckedException If failed.
- */
- public abstract void cleanupTaskEnvironment() throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskInfo.java
deleted file mode 100644
index 75e06ca..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskInfo.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * Task info.
- */
-public class GridHadoopTaskInfo implements Externalizable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private GridHadoopTaskType type;
-
- /** */
- private GridHadoopJobId jobId;
-
- /** */
- private int taskNum;
-
- /** */
- private int attempt;
-
- /** */
- private GridHadoopInputSplit inputSplit;
-
- /**
- * For {@link Externalizable}.
- */
- public GridHadoopTaskInfo() {
- // No-op.
- }
-
- /**
- * Creates new task info.
- *
- * @param type Task type.
- * @param jobId Job id.
- * @param taskNum Task number.
- * @param attempt Attempt for this task.
- * @param inputSplit Input split.
- */
- public GridHadoopTaskInfo(GridHadoopTaskType type, GridHadoopJobId jobId, int taskNum, int attempt,
- @Nullable GridHadoopInputSplit inputSplit) {
- this.type = type;
- this.jobId = jobId;
- this.taskNum = taskNum;
- this.attempt = attempt;
- this.inputSplit = inputSplit;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeByte(type.ordinal());
- out.writeObject(jobId);
- out.writeInt(taskNum);
- out.writeInt(attempt);
- out.writeObject(inputSplit);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- type = GridHadoopTaskType.fromOrdinal(in.readByte());
- jobId = (GridHadoopJobId)in.readObject();
- taskNum = in.readInt();
- attempt = in.readInt();
- inputSplit = (GridHadoopInputSplit)in.readObject();
- }
-
- /**
- * @return Type.
- */
- public GridHadoopTaskType type() {
- return type;
- }
-
- /**
- * @return Job id.
- */
- public GridHadoopJobId jobId() {
- return jobId;
- }
-
- /**
- * @return Task number.
- */
- public int taskNumber() {
- return taskNum;
- }
-
- /**
- * @return Attempt.
- */
- public int attempt() {
- return attempt;
- }
-
- /**
- * @return Input split.
- */
- @Nullable public GridHadoopInputSplit inputSplit() {
- return inputSplit;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (!(o instanceof GridHadoopTaskInfo))
- return false;
-
- GridHadoopTaskInfo that = (GridHadoopTaskInfo)o;
-
- return attempt == that.attempt && taskNum == that.taskNum && jobId.equals(that.jobId) && type == that.type;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = type.hashCode();
-
- res = 31 * res + jobId.hashCode();
- res = 31 * res + taskNum;
- res = 31 * res + attempt;
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopTaskInfo.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskInput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskInput.java
deleted file mode 100644
index 479cf6d..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskInput.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-
-import java.util.*;
-
-/**
- * Task input.
- */
-public interface GridHadoopTaskInput extends AutoCloseable {
- /**
- * Moves cursor to the next element.
- *
- * @return {@code false} If input is exceeded.
- */
- boolean next();
-
- /**
- * Gets current key.
- *
- * @return Key.
- */
- Object key();
-
- /**
- * Gets values for current key.
- *
- * @return Values.
- */
- Iterator<?> values();
-
- /**
- * Closes input.
- *
- * @throws IgniteCheckedException If failed.
- */
- @Override public void close() throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskOutput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskOutput.java
deleted file mode 100644
index 6480d8d..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskOutput.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.apache.ignite.*;
-
-/**
- * Task output.
- */
-public interface GridHadoopTaskOutput extends AutoCloseable {
- /**
- * Writes key and value to the output.
- *
- * @param key Key.
- * @param val Value.
- */
- public void write(Object key, Object val) throws IgniteCheckedException;
-
- /**
- * Closes output.
- *
- * @throws IgniteCheckedException If failed.
- */
- @Override public void close() throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskType.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskType.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskType.java
deleted file mode 100644
index 404d6b8..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopTaskType.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-import org.jetbrains.annotations.*;
-
-/**
-* Task type.
-*/
-public enum GridHadoopTaskType {
- /** Setup task. */
- SETUP,
-
- /** Map task. */
- MAP,
-
- /** Reduce task. */
- REDUCE,
-
- /** Combine task. */
- COMBINE,
-
- /** Commit task. */
- COMMIT,
-
- /** Abort task. */
- ABORT;
-
- /** Enumerated values. */
- private static final GridHadoopTaskType[] VALS = values();
-
- /**
- * Efficiently gets enumerated value from its ordinal.
- *
- * @param ord Ordinal value.
- * @return Enumerated value.
- */
- @Nullable public static GridHadoopTaskType fromOrdinal(byte ord) {
- return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/Hadoop.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/Hadoop.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/Hadoop.java
new file mode 100644
index 0000000..9efc4a9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/Hadoop.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.jetbrains.annotations.*;
+
+/**
+ * Hadoop facade providing access to Ignite Hadoop features.
+ */
+public interface Hadoop {
+ /**
+ * Gets Hadoop module configuration.
+ *
+ * @return Hadoop module configuration.
+ */
+ public HadoopConfiguration configuration();
+
+ /**
+ * Generate next job ID.
+ *
+ * @return Next job ID.
+ */
+ public HadoopJobId nextJobId();
+
+ /**
+ * Submits job to job tracker.
+ *
+ * @param jobId Job ID to submit.
+ * @param jobInfo Job info to submit.
+ * @return Execution future.
+ */
+ public IgniteInternalFuture<?> submit(HadoopJobId jobId, HadoopJobInfo jobInfo);
+
+ /**
+ * Gets Hadoop job execution status.
+ *
+ * @param jobId Job ID to get status for.
+ * @return Job execution status or {@code null} in case job with the given ID is not found.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable public HadoopJobStatus status(HadoopJobId jobId) throws IgniteCheckedException;
+
+ /**
+ * Returns job counters.
+ *
+ * @param jobId Job ID to get counters for.
+ * @return Job counters object.
+ * @throws IgniteCheckedException If failed.
+ */
+ public HadoopCounters counters(HadoopJobId jobId) throws IgniteCheckedException;
+
+ /**
+ * Gets Hadoop finish future for particular job.
+ *
+ * @param jobId Job ID.
+ * @return Job finish future or {@code null} in case job with the given ID is not found.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable public IgniteInternalFuture<?> finishFuture(HadoopJobId jobId) throws IgniteCheckedException;
+
+ /**
+ * Kills job.
+ *
+ * @param jobId Job ID.
+ * @return {@code True} if job was killed.
+ * @throws IgniteCheckedException If failed.
+ */
+ public boolean kill(HadoopJobId jobId) throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopFileBlock.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopFileBlock.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopFileBlock.java
new file mode 100644
index 0000000..223e572
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopFileBlock.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.internal.util.tostring.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+/**
+ * Hadoop file block.
+ */
+public class HadoopFileBlock extends HadoopInputSplit {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ @GridToStringInclude
+ protected URI file;
+
+ /** */
+ @GridToStringInclude
+ protected long start;
+
+ /** */
+ @GridToStringInclude
+ protected long len;
+
+ /**
+ * Creates new file block.
+ */
+ public HadoopFileBlock() {
+ // No-op.
+ }
+
+ /**
+ * Creates new file block.
+ *
+ * @param hosts List of hosts where the block resides.
+ * @param file File URI.
+ * @param start Start position of the block in the file.
+ * @param len Length of the block.
+ */
+ public HadoopFileBlock(String[] hosts, URI file, long start, long len) {
+ A.notNull(hosts, "hosts", file, "file");
+
+ this.hosts = hosts;
+ this.file = file;
+ this.start = start;
+ this.len = len;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeObject(file());
+ out.writeLong(start());
+ out.writeLong(length());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ file = (URI)in.readObject();
+ start = in.readLong();
+ len = in.readLong();
+ }
+
+ /**
+ * @return Length.
+ */
+ public long length() {
+ return len;
+ }
+
+ /**
+ * @param len New length.
+ */
+ public void length(long len) {
+ this.len = len;
+ }
+
+ /**
+ * @return Start.
+ */
+ public long start() {
+ return start;
+ }
+
+ /**
+ * @param start New start.
+ */
+ public void start(long start) {
+ this.start = start;
+ }
+
+ /**
+ * @return File.
+ */
+ public URI file() {
+ return file;
+ }
+
+ /**
+ * @param file New file.
+ */
+ public void file(URI file) {
+ this.file = file;
+ }
+
+ /**
+ * @param hosts New hosts.
+ */
+ public void hosts(String[] hosts) {
+ A.notNull(hosts, "hosts");
+
+ this.hosts = hosts;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof HadoopFileBlock))
+ return false;
+
+ HadoopFileBlock that = (HadoopFileBlock)o;
+
+ return len == that.len && start == that.start && file.equals(that.file);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = file.hashCode();
+
+ res = 31 * res + (int)(start ^ (start >>> 32));
+ res = 31 * res + (int)(len ^ (len >>> 32));
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ public String toString() {
+ return S.toString(HadoopFileBlock.class, this, "hosts", Arrays.toString(hosts));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopInputSplit.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopInputSplit.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopInputSplit.java
new file mode 100644
index 0000000..0c94012
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopInputSplit.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import java.io.*;
+
+/**
+ * Abstract fragment of an input data source.
+ */
+public abstract class HadoopInputSplit implements Externalizable {
+ /** */
+ protected String[] hosts;
+
+ /**
+ * Array of hosts where this input split resides.
+ *
+ * @return Hosts.
+ */
+ public String[] hosts() {
+ assert hosts != null;
+
+ return hosts;
+ }
+
+ /**
+ * This method must be implemented for purpose of internal implementation.
+ *
+ * @param obj Another object.
+ * @return {@code true} If objects are equal.
+ */
+ @Override public abstract boolean equals(Object obj);
+
+ /**
+ * This method must be implemented for purpose of internal implementation.
+ *
+ * @return Hash code of the object.
+ */
+ @Override public abstract int hashCode();
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJob.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJob.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJob.java
new file mode 100644
index 0000000..65cb48d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJob.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+
+import java.util.*;
+
+/**
+ * Hadoop job.
+ */
+public interface HadoopJob {
+ /**
+ * Gets job ID.
+ *
+ * @return Job ID.
+ */
+ public HadoopJobId id();
+
+ /**
+ * Gets job information.
+ *
+ * @return Job information.
+ */
+ public HadoopJobInfo info();
+
+ /**
+ * Gets collection of input splits for this job.
+ *
+ * @return Input splits.
+ */
+ public Collection<HadoopInputSplit> input() throws IgniteCheckedException;
+
+ /**
+ * Returns context for task execution.
+ *
+ * @param info Task info.
+ * @return Task Context.
+ * @throws IgniteCheckedException If failed.
+ */
+ public HadoopTaskContext getTaskContext(HadoopTaskInfo info) throws IgniteCheckedException;
+
+ /**
+ * Does all the needed initialization for the job. Will be called on each node where tasks for this job must
+ * be executed.
+ * <p>
+ * If job is running in external mode this method will be called on instance in Ignite node with parameter
+ * {@code false} and on instance in external process with parameter {@code true}.
+ *
+ * @param external If {@code true} then this job instance resides in external process.
+ * @param locNodeId Local node ID.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void initialize(boolean external, UUID locNodeId) throws IgniteCheckedException;
+
+ /**
+ * Release all the resources.
+ * <p>
+ * If job is running in external mode this method will be called on instance in Ignite node with parameter
+ * {@code false} and on instance in external process with parameter {@code true}.
+ *
+ * @param external If {@code true} then this job instance resides in external process.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void dispose(boolean external) throws IgniteCheckedException;
+
+ /**
+ * Prepare local environment for the task.
+ *
+ * @param info Task info.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void prepareTaskEnvironment(HadoopTaskInfo info) throws IgniteCheckedException;
+
+ /**
+ * Cleans up local environment of the task.
+ *
+ * @param info Task info.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void cleanupTaskEnvironment(HadoopTaskInfo info) throws IgniteCheckedException;
+
+ /**
+ * Cleans up the job staging directory.
+ */
+ void cleanupStagingDirectory();
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
new file mode 100644
index 0000000..b0593a8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.internal.processors.cache.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Job ID.
+ */
+public class HadoopJobId implements GridCacheInternal, Externalizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private UUID nodeId;
+
+ /** */
+ private int jobId;
+
+ /**
+ * For {@link Externalizable}.
+ */
+ public HadoopJobId() {
+ // No-op.
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @param jobId Job ID.
+ */
+ public HadoopJobId(UUID nodeId, int jobId) {
+ this.nodeId = nodeId;
+ this.jobId = jobId;
+ }
+
+ public UUID globalId() {
+ return nodeId;
+ }
+
+ public int localId() {
+ return jobId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ U.writeUuid(out, nodeId);
+ out.writeInt(jobId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ nodeId = U.readUuid(in);
+ jobId = in.readInt();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ HadoopJobId that = (HadoopJobId) o;
+
+ if (jobId != that.jobId)
+ return false;
+
+ if (!nodeId.equals(that.nodeId))
+ return false;
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return 31 * nodeId.hashCode() + jobId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return nodeId + "_" + jobId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java
new file mode 100644
index 0000000..51faf5d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * Compact job description.
+ */
+public interface HadoopJobInfo extends Serializable {
+ /**
+ * Gets optional configuration property for the job.
+ *
+ * @param name Property name.
+ * @return Value or {@code null} if none.
+ */
+ @Nullable public String property(String name);
+
+ /**
+ * Checks whether job has combiner.
+ *
+ * @return {@code true} If job has combiner.
+ */
+ public boolean hasCombiner();
+
+ /**
+ * Checks whether job has reducer.
+ * Actual number of reducers will be in {@link HadoopMapReducePlan#reducers()}.
+ *
+ * @return Number of reducer.
+ */
+ public boolean hasReducer();
+
+ /**
+ * Creates new job instance for the given ID.
+ * {@link HadoopJobInfo} is reusable for multiple jobs while {@link HadoopJob} is for one job execution.
+ * This method will be called once for the same ID on one node, though it can be called on the same host
+ * multiple times from different processes (in case of multiple nodes on the same host or external execution).
+ *
+ * @param jobId Job ID.
+ * @param log Logger.
+ * @return Job.
+ * @throws IgniteCheckedException If failed.
+ */
+ HadoopJob createJob(HadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException;
+
+ /**
+ * @return Number of reducers configured for job.
+ */
+ public int reducers();
+
+ /**
+ * Gets job name.
+ *
+ * @return Job name.
+ */
+ public String jobName();
+
+ /**
+ * Gets user name.
+ *
+ * @return User name.
+ */
+ public String user();
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobPhase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobPhase.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobPhase.java
new file mode 100644
index 0000000..8c932bb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobPhase.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ * Job run phase.
+ */
+public enum HadoopJobPhase {
+ /** Job is running setup task. */
+ PHASE_SETUP,
+
+ /** Job is running map and combine tasks. */
+ PHASE_MAP,
+
+ /** Job has finished all map tasks and running reduce tasks. */
+ PHASE_REDUCE,
+
+ /** Job is stopping due to exception during any of the phases. */
+ PHASE_CANCELLING,
+
+ /** Job has finished execution. */
+ PHASE_COMPLETE
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
new file mode 100644
index 0000000..1a58624
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.jetbrains.annotations.*;
+
+/**
+ * Enumeration of optional properties supported by Ignite for Apache Hadoop.
+ */
+public enum HadoopJobProperty {
+ /**
+ * Initial size for hashmap which stores output of mapper and will be used as input of combiner.
+ * <p>
+ * Setting it right allows to avoid rehashing.
+ */
+ COMBINER_HASHMAP_SIZE,
+
+ /**
+ * Initial size for hashmap which stores output of mapper or combiner and will be used as input of reducer.
+ * <p>
+ * Setting it right allows to avoid rehashing.
+ */
+ PARTITION_HASHMAP_SIZE,
+
+ /**
+ * Specifies number of concurrently running mappers for external execution mode.
+ * <p>
+ * If not specified, defaults to {@code Runtime.getRuntime().availableProcessors()}.
+ */
+ EXTERNAL_CONCURRENT_MAPPERS,
+
+ /**
+ * Specifies number of concurrently running reducers for external execution mode.
+ * <p>
+ * If not specified, defaults to {@code Runtime.getRuntime().availableProcessors()}.
+ */
+ EXTERNAL_CONCURRENT_REDUCERS,
+
+ /**
+ * Delay in milliseconds after which Ignite server will reply job status.
+ */
+ JOB_STATUS_POLL_DELAY,
+
+ /**
+ * Size in bytes of single memory page which will be allocated for data structures in shuffle.
+ * <p>
+ * By default is {@code 32 * 1024}.
+ */
+ SHUFFLE_OFFHEAP_PAGE_SIZE,
+
+ /**
+ * If set to {@code true} then input for combiner will not be sorted by key.
+ * Internally hash-map will be used instead of sorted one, so {@link Object#equals(Object)}
+ * and {@link Object#hashCode()} methods of key must be implemented consistently with
+ * comparator for that type. Grouping comparator is not supported if this setting is {@code true}.
+ * <p>
+ * By default is {@code false}.
+ */
+ SHUFFLE_COMBINER_NO_SORTING,
+
+ /**
+ * If set to {@code true} then input for reducer will not be sorted by key.
+ * Internally hash-map will be used instead of sorted one, so {@link Object#equals(Object)}
+ * and {@link Object#hashCode()} methods of key must be implemented consistently with
+ * comparator for that type. Grouping comparator is not supported if this setting is {@code true}.
+ * <p>
+ * By default is {@code false}.
+ */
+ SHUFFLE_REDUCER_NO_SORTING;
+
+ /** */
+ private final String ptyName;
+
+ /**
+ *
+ */
+ HadoopJobProperty() {
+ ptyName = "ignite." + name().toLowerCase().replace('_', '.');
+ }
+
+ /**
+ * @return Property name.
+ */
+ public String propertyName() {
+ return ptyName;
+ }
+
+ /**
+ * @param jobInfo Job info.
+ * @param pty Property.
+ * @param dflt Default value.
+ * @return Property value.
+ */
+ public static String get(HadoopJobInfo jobInfo, HadoopJobProperty pty, @Nullable String dflt) {
+ String res = jobInfo.property(pty.propertyName());
+
+ return res == null ? dflt : res;
+ }
+
+ /**
+ * @param jobInfo Job info.
+ * @param pty Property.
+ * @param dflt Default value.
+ * @return Property value.
+ */
+ public static int get(HadoopJobInfo jobInfo, HadoopJobProperty pty, int dflt) {
+ String res = jobInfo.property(pty.propertyName());
+
+ return res == null ? dflt : Integer.parseInt(res);
+ }
+
+ /**
+ * @param jobInfo Job info.
+ * @param pty Property.
+ * @param dflt Default value.
+ * @return Property value.
+ */
+ public static boolean get(HadoopJobInfo jobInfo, HadoopJobProperty pty, boolean dflt) {
+ String res = jobInfo.property(pty.propertyName());
+
+ return res == null ? dflt : Boolean.parseBoolean(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobStatus.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobStatus.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobStatus.java
new file mode 100644
index 0000000..752556d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobStatus.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+
+/**
+ * Hadoop job status.
+ */
+public class HadoopJobStatus implements Externalizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Job ID. */
+ private HadoopJobId jobId;
+
+ /** Job name. */
+ private String jobName;
+
+ /** User. */
+ private String usr;
+
+ /** Pending mappers count. */
+ private int pendingMapperCnt;
+
+ /** Pending reducers count. */
+ private int pendingReducerCnt;
+
+ /** Total mappers count. */
+ private int totalMapperCnt;
+
+ /** Total reducers count. */
+ private int totalReducerCnt;
+ /** Phase. */
+ private HadoopJobPhase jobPhase;
+
+ /** */
+ private boolean failed;
+
+ /** Version. */
+ private long ver;
+
+ /**
+ * {@link Externalizable} support.
+ */
+ public HadoopJobStatus() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param jobId Job ID.
+ * @param jobName Job name.
+ * @param usr User.
+ * @param pendingMapperCnt Pending mappers count.
+ * @param pendingReducerCnt Pending reducers count.
+ * @param totalMapperCnt Total mappers count.
+ * @param totalReducerCnt Total reducers count.
+ * @param jobPhase Job phase.
+ * @param failed Failed.
+ * @param ver Version.
+ */
+ public HadoopJobStatus(
+ HadoopJobId jobId,
+ String jobName,
+ String usr,
+ int pendingMapperCnt,
+ int pendingReducerCnt,
+ int totalMapperCnt,
+ int totalReducerCnt,
+ HadoopJobPhase jobPhase,
+ boolean failed,
+ long ver
+ ) {
+ this.jobId = jobId;
+ this.jobName = jobName;
+ this.usr = usr;
+ this.pendingMapperCnt = pendingMapperCnt;
+ this.pendingReducerCnt = pendingReducerCnt;
+ this.totalMapperCnt = totalMapperCnt;
+ this.totalReducerCnt = totalReducerCnt;
+ this.jobPhase = jobPhase;
+ this.failed = failed;
+ this.ver = ver;
+ }
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /**
+ * @return Job name.
+ */
+ public String jobName() {
+ return jobName;
+ }
+
+ /**
+ * @return User.
+ */
+ public String user() {
+ return usr;
+ }
+
+ /**
+ * @return Pending mappers count.
+ */
+ public int pendingMapperCnt() {
+ return pendingMapperCnt;
+ }
+
+ /**
+ * @return Pending reducers count.
+ */
+ public int pendingReducerCnt() {
+ return pendingReducerCnt;
+ }
+
+ /**
+ * @return Total mappers count.
+ */
+ public int totalMapperCnt() {
+ return totalMapperCnt;
+ }
+
+ /**
+ * @return Total reducers count.
+ */
+ public int totalReducerCnt() {
+ return totalReducerCnt;
+ }
+
+ /**
+ * @return Version.
+ */
+ public long version() {
+ return ver;
+ }
+
+ /**
+ * @return Job phase.
+ */
+ public HadoopJobPhase jobPhase() {
+ return jobPhase;
+ }
+
+ /**
+ * @return {@code true} If the job failed.
+ */
+ public boolean isFailed() {
+ return failed;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopJobStatus.class, this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeObject(jobId);
+ U.writeString(out, jobName);
+ U.writeString(out, usr);
+ out.writeInt(pendingMapperCnt);
+ out.writeInt(pendingReducerCnt);
+ out.writeInt(totalMapperCnt);
+ out.writeInt(totalReducerCnt);
+ out.writeObject(jobPhase);
+ out.writeBoolean(failed);
+ out.writeLong(ver);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobId = (HadoopJobId)in.readObject();
+ jobName = U.readString(in);
+ usr = U.readString(in);
+ pendingMapperCnt = in.readInt();
+ pendingReducerCnt = in.readInt();
+ totalMapperCnt = in.readInt();
+ totalReducerCnt = in.readInt();
+ jobPhase = (HadoopJobPhase)in.readObject();
+ failed = in.readBoolean();
+ ver = in.readLong();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReducePlan.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReducePlan.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReducePlan.java
new file mode 100644
index 0000000..3da2fb1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReducePlan.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Map-reduce job execution plan.
+ */
+public interface HadoopMapReducePlan extends Serializable {
+ /**
+ * Gets collection of file blocks for which mappers should be executed.
+ *
+ * @param nodeId Node ID to check.
+ * @return Collection of file blocks or {@code null} if no mappers should be executed on given node.
+ */
+ @Nullable public Collection<HadoopInputSplit> mappers(UUID nodeId);
+
+ /**
+ * Gets reducer IDs that should be started on given node.
+ *
+ * @param nodeId Node ID to check.
+ * @return Array of reducer IDs.
+ */
+ @Nullable public int[] reducers(UUID nodeId);
+
+ /**
+ * Gets collection of all node IDs involved in map part of job execution.
+ *
+ * @return Collection of node IDs.
+ */
+ public Collection<UUID> mapperNodeIds();
+
+ /**
+ * Gets collection of all node IDs involved in reduce part of job execution.
+ *
+ * @return Collection of node IDs.
+ */
+ public Collection<UUID> reducerNodeIds();
+
+ /**
+ * Gets overall number of mappers for the job.
+ *
+ * @return Number of mappers.
+ */
+ public int mappers();
+
+ /**
+ * Gets overall number of reducers for the job.
+ *
+ * @return Number of reducers.
+ */
+ public int reducers();
+
+ /**
+ * Gets node ID for reducer.
+ *
+ * @param reducer Reducer.
+ * @return Node ID.
+ */
+ public UUID nodeForReducer(int reducer);
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReducePlanner.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReducePlanner.java
new file mode 100644
index 0000000..ab885fe
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReducePlanner.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.apache.ignite.cluster.*;
+import org.jetbrains.annotations.*;
+
+import java.util.*;
+
+/**
+ * Map-reduce execution planner.
+ */
+public interface HadoopMapReducePlanner {
+ /**
+ * Prepares map-reduce execution plan for the given job and topology.
+ *
+ * @param job Job.
+ * @param top Topology.
+ * @param oldPlan Old plan in case of partial failure.
+ * @return Map reduce plan.
+ */
+ public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> top,
+ @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
new file mode 100644
index 0000000..eb84d00
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.apache.ignite.internal.util.future.*;
+
+/**
+ * Hadoop processor.
+ */
+public class HadoopNoopProcessor extends HadoopProcessorAdapter {
+ /**
+ * @param ctx Kernal context.
+ */
+ public HadoopNoopProcessor(GridKernalContext ctx) {
+ super(ctx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Hadoop hadoop() {
+ throw new IllegalStateException("Hadoop module is not found in class path.");
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopConfiguration config() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopJobId nextJobId() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteInternalFuture<?> submit(HadoopJobId jobId, HadoopJobInfo jobInfo) {
+ return new GridFinishedFutureEx<>(new IgniteCheckedException("Hadoop is not available."));
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopJobStatus status(HadoopJobId jobId) throws IgniteCheckedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public HadoopCounters counters(HadoopJobId jobId) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteInternalFuture<?> finishFuture(HadoopJobId jobId) throws IgniteCheckedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean kill(HadoopJobId jobId) throws IgniteCheckedException {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPartitioner.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPartitioner.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPartitioner.java
new file mode 100644
index 0000000..ec94f81
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPartitioner.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ * Partitioner.
+ */
+public interface HadoopPartitioner {
+ /**
+ * Gets partition which is actually a reducer index for the given key and value pair.
+ *
+ * @param key Key.
+ * @param val Value.
+ * @param parts Number of partitions.
+ * @return Partition.
+ */
+ public int partition(Object key, Object val, int parts);
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
new file mode 100644
index 0000000..44ff8be
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+
+/**
+ * Hadoop processor.
+ */
+public abstract class HadoopProcessorAdapter extends GridProcessorAdapter {
+ /**
+ * @param ctx Kernal context.
+ */
+ protected HadoopProcessorAdapter(GridKernalContext ctx) {
+ super(ctx);
+ }
+
+ /**
+ * @return Hadoop facade.
+ */
+ public abstract Hadoop hadoop();
+
+ /**
+ * @return Hadoop configuration.
+ */
+ public abstract HadoopConfiguration config();
+
+ /**
+ * @return Collection of generated IDs.
+ */
+ public abstract HadoopJobId nextJobId();
+
+ /**
+ * Submits job to job tracker.
+ *
+ * @param jobId Job ID to submit.
+ * @param jobInfo Job info to submit.
+ * @return Execution future.
+ */
+ public abstract IgniteInternalFuture<?> submit(HadoopJobId jobId, HadoopJobInfo jobInfo);
+
+ /**
+ * Gets Hadoop job execution status.
+ *
+ * @param jobId Job ID to get status for.
+ * @return Job execution status.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract HadoopJobStatus status(HadoopJobId jobId) throws IgniteCheckedException;
+
+ /**
+ * Returns Hadoop job counters.
+ *
+ * @param jobId Job ID to get counters for.
+ * @return Job counters.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract HadoopCounters counters(HadoopJobId jobId) throws IgniteCheckedException;
+
+ /**
+ * Gets Hadoop job finish future.
+ *
+ * @param jobId Job ID.
+ * @return Job finish future or {@code null}.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract IgniteInternalFuture<?> finishFuture(HadoopJobId jobId) throws IgniteCheckedException;
+
+ /**
+ * Kills job.
+ *
+ * @param jobId Job ID.
+ * @return {@code True} if job was killed.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract boolean kill(HadoopJobId jobId) throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopSerialization.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopSerialization.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopSerialization.java
new file mode 100644
index 0000000..aab803b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopSerialization.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * Hadoop serialization. Not thread safe object, must be created for each thread or correctly synchronized.
+ */
+public interface HadoopSerialization extends AutoCloseable {
+ /**
+ * Writes the given object to output.
+ *
+ * @param out Output.
+ * @param obj Object to serialize.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void write(DataOutput out, Object obj) throws IgniteCheckedException;
+
+ /**
+ * Reads object from the given input optionally reusing given instance.
+ *
+ * @param in Input.
+ * @param obj Object.
+ * @return New object or reused instance.
+ * @throws IgniteCheckedException If failed.
+ */
+ public Object read(DataInput in, @Nullable Object obj) throws IgniteCheckedException;
+
+ /**
+ * Finalise the internal objects.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ @Override public void close() throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTask.java
new file mode 100644
index 0000000..3ce83ae
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTask.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+
+import java.io.*;
+
+/**
+ * Hadoop task.
+ */
+public abstract class HadoopTask {
+ /** */
+ private HadoopTaskInfo taskInfo;
+
+ /**
+ * Creates task.
+ *
+ * @param taskInfo Task info.
+ */
+ protected HadoopTask(HadoopTaskInfo taskInfo) {
+ assert taskInfo != null;
+
+ this.taskInfo = taskInfo;
+ }
+
+ /**
+ * For {@link Externalizable}.
+ */
+ @SuppressWarnings("ConstructorNotProtectedInAbstractClass")
+ public HadoopTask() {
+ // No-op.
+ }
+
+ /**
+ * Gets task info.
+ *
+ * @return Task info.
+ */
+ public HadoopTaskInfo info() {
+ return taskInfo;
+ }
+
+ /**
+ * Runs task.
+ *
+ * @param taskCtx Context.
+ * @throws org.apache.ignite.internal.IgniteInterruptedCheckedException If interrupted.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract void run(HadoopTaskContext taskCtx) throws IgniteCheckedException;
+
+ /**
+ * Interrupts task execution.
+ */
+ public abstract void cancel();
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
new file mode 100644
index 0000000..371fd81
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+
+import java.util.*;
+
+/**
+ * Task context.
+ */
+public abstract class HadoopTaskContext {
+ /** */
+ private final HadoopJob job;
+
+ /** */
+ private HadoopTaskInput input;
+
+ /** */
+ private HadoopTaskOutput output;
+
+ /** */
+ private HadoopTaskInfo taskInfo;
+
+ /**
+ * @param taskInfo Task info.
+ * @param job Job.
+ */
+ protected HadoopTaskContext(HadoopTaskInfo taskInfo, HadoopJob job) {
+ this.taskInfo = taskInfo;
+ this.job = job;
+ }
+
+ /**
+ * Gets task info.
+ *
+ * @return Task info.
+ */
+ public HadoopTaskInfo taskInfo() {
+ return taskInfo;
+ }
+
+ /**
+ * Set a new task info.
+ *
+ * @param info Task info.
+ */
+ public void taskInfo(HadoopTaskInfo info) {
+ taskInfo = info;
+ }
+
+ /**
+ * Gets task output.
+ *
+ * @return Task output.
+ */
+ public HadoopTaskOutput output() {
+ return output;
+ }
+
+ /**
+ * Gets task input.
+ *
+ * @return Task input.
+ */
+ public HadoopTaskInput input() {
+ return input;
+ }
+
+ /**
+ * @return Job.
+ */
+ public HadoopJob job() {
+ return job;
+ }
+
+ /**
+ * Gets counter for the given name.
+ *
+ * @param grp Counter group's name.
+ * @param name Counter name.
+ * @return Counter.
+ */
+ public abstract <T extends HadoopCounter> T counter(String grp, String name, Class<T> cls);
+
+ /**
+ * Gets all known counters.
+ *
+ * @return Unmodifiable collection of counters.
+ */
+ public abstract HadoopCounters counters();
+
+ /**
+ * Sets input of the task.
+ *
+ * @param in Input.
+ */
+ public void input(HadoopTaskInput in) {
+ input = in;
+ }
+
+ /**
+ * Sets output of the task.
+ *
+ * @param out Output.
+ */
+ public void output(HadoopTaskOutput out) {
+ output = out;
+ }
+
+ /**
+ * Gets partitioner.
+ *
+ * @return Partitioner.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract HadoopPartitioner partitioner() throws IgniteCheckedException;
+
+ /**
+ * Gets serializer for values.
+ *
+ * @return Serializer for keys.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract HadoopSerialization keySerialization() throws IgniteCheckedException;
+
+ /**
+ * Gets serializer for values.
+ *
+ * @return Serializer for values.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract HadoopSerialization valueSerialization() throws IgniteCheckedException;
+
+ /**
+ * Gets sorting comparator.
+ *
+ * @return Comparator for sorting.
+ */
+ public abstract Comparator<Object> sortComparator();
+
+ /**
+ * Gets comparator for grouping on combine or reduce operation.
+ *
+ * @return Comparator.
+ */
+ public abstract Comparator<Object> groupComparator();
+
+ /**
+ * Execute current task.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract void run() throws IgniteCheckedException;
+
+ /**
+ * Cancel current task execution.
+ */
+ public abstract void cancel();
+
+ /**
+ * Prepare local environment for the task.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract void prepareTaskEnvironment() throws IgniteCheckedException;
+
+ /**
+ * Cleans up local environment of the task.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract void cleanupTaskEnvironment() throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java
new file mode 100644
index 0000000..eb82cb4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * Task info.
+ */
+public class HadoopTaskInfo implements Externalizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private HadoopTaskType type;
+
+ /** */
+ private HadoopJobId jobId;
+
+ /** */
+ private int taskNum;
+
+ /** */
+ private int attempt;
+
+ /** */
+ private HadoopInputSplit inputSplit;
+
+ /**
+ * For {@link Externalizable}.
+ */
+ public HadoopTaskInfo() {
+ // No-op.
+ }
+
+ /**
+ * Creates new task info.
+ *
+ * @param type Task type.
+ * @param jobId Job id.
+ * @param taskNum Task number.
+ * @param attempt Attempt for this task.
+ * @param inputSplit Input split.
+ */
+ public HadoopTaskInfo(HadoopTaskType type, HadoopJobId jobId, int taskNum, int attempt,
+ @Nullable HadoopInputSplit inputSplit) {
+ this.type = type;
+ this.jobId = jobId;
+ this.taskNum = taskNum;
+ this.attempt = attempt;
+ this.inputSplit = inputSplit;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeByte(type.ordinal());
+ out.writeObject(jobId);
+ out.writeInt(taskNum);
+ out.writeInt(attempt);
+ out.writeObject(inputSplit);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ type = HadoopTaskType.fromOrdinal(in.readByte());
+ jobId = (HadoopJobId)in.readObject();
+ taskNum = in.readInt();
+ attempt = in.readInt();
+ inputSplit = (HadoopInputSplit)in.readObject();
+ }
+
+ /**
+ * @return Type.
+ */
+ public HadoopTaskType type() {
+ return type;
+ }
+
+ /**
+ * @return Job id.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /**
+ * @return Task number.
+ */
+ public int taskNumber() {
+ return taskNum;
+ }
+
+ /**
+ * @return Attempt.
+ */
+ public int attempt() {
+ return attempt;
+ }
+
+ /**
+ * @return Input split.
+ */
+ @Nullable public HadoopInputSplit inputSplit() {
+ return inputSplit;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof HadoopTaskInfo))
+ return false;
+
+ HadoopTaskInfo that = (HadoopTaskInfo)o;
+
+ return attempt == that.attempt && taskNum == that.taskNum && jobId.equals(that.jobId) && type == that.type;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = type.hashCode();
+
+ res = 31 * res + jobId.hashCode();
+ res = 31 * res + taskNum;
+ res = 31 * res + attempt;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopTaskInfo.class, this);
+ }
+}
[23/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopDataInStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopDataInStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopDataInStream.java
deleted file mode 100644
index 8b4f0c4..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopDataInStream.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.streams;
-
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-
-import java.io.*;
-import java.nio.charset.*;
-
-/**
- * Data input stream.
- */
-public class GridHadoopDataInStream extends InputStream implements DataInput {
- /** */
- private final GridHadoopOffheapBuffer buf = new GridHadoopOffheapBuffer(0, 0);
-
- /** */
- private final GridUnsafeMemory mem;
-
- /**
- * @param mem Memory.
- */
- public GridHadoopDataInStream(GridUnsafeMemory mem) {
- assert mem != null;
-
- this.mem = mem;
- }
-
- /**
- * @return Buffer.
- */
- public GridHadoopOffheapBuffer buffer() {
- return buf;
- }
-
- /**
- * @param size Size.
- * @return Old pointer.
- */
- protected long move(long size) throws IOException {
- long ptr = buf.move(size);
-
- assert ptr != 0;
-
- return ptr;
- }
-
- /** {@inheritDoc} */
- @Override public int read() throws IOException {
- return readUnsignedByte();
- }
-
- /** {@inheritDoc} */
- @Override public int read(byte[] b, int off, int len) throws IOException {
- readFully(b, off, len);
-
- return len;
- }
-
- /** {@inheritDoc} */
- @Override public long skip(long n) throws IOException {
- move(n);
-
- return n;
- }
-
- /** {@inheritDoc} */
- @Override public void readFully(byte[] b) throws IOException {
- readFully(b, 0, b.length);
- }
-
- /** {@inheritDoc} */
- @Override public void readFully(byte[] b, int off, int len) throws IOException {
- mem.readBytes(move(len), b, off, len);
- }
-
- /** {@inheritDoc} */
- @Override public int skipBytes(int n) throws IOException {
- move(n);
-
- return n;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readBoolean() throws IOException {
- byte res = readByte();
-
- if (res == 1)
- return true;
-
- assert res == 0 : res;
-
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public byte readByte() throws IOException {
- return mem.readByte(move(1));
- }
-
- /** {@inheritDoc} */
- @Override public int readUnsignedByte() throws IOException {
- return readByte() & 0xff;
- }
-
- /** {@inheritDoc} */
- @Override public short readShort() throws IOException {
- return mem.readShort(move(2));
- }
-
- /** {@inheritDoc} */
- @Override public int readUnsignedShort() throws IOException {
- return readShort() & 0xffff;
- }
-
- /** {@inheritDoc} */
- @Override public char readChar() throws IOException {
- return (char)readShort();
- }
-
- /** {@inheritDoc} */
- @Override public int readInt() throws IOException {
- return mem.readInt(move(4));
- }
-
- /** {@inheritDoc} */
- @Override public long readLong() throws IOException {
- return mem.readLong(move(8));
- }
-
- /** {@inheritDoc} */
- @Override public float readFloat() throws IOException {
- return mem.readFloat(move(4));
- }
-
- /** {@inheritDoc} */
- @Override public double readDouble() throws IOException {
- return mem.readDouble(move(8));
- }
-
- /** {@inheritDoc} */
- @Override public String readLine() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- /** {@inheritDoc} */
- @Override public String readUTF() throws IOException {
- byte[] bytes = new byte[readInt()];
-
- if (bytes.length != 0)
- readFully(bytes);
-
- return new String(bytes, StandardCharsets.UTF_8);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopDataOutStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopDataOutStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopDataOutStream.java
deleted file mode 100644
index 8b837c8..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopDataOutStream.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.streams;
-
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-
-import java.io.*;
-import java.nio.charset.*;
-
-import static org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory.*;
-
-/**
- * Data output stream.
- */
-public class GridHadoopDataOutStream extends OutputStream implements DataOutput {
- /** */
- private final GridHadoopOffheapBuffer buf = new GridHadoopOffheapBuffer(0, 0);
-
- /** */
- private final GridUnsafeMemory mem;
-
- /**
- * @param mem Memory.
- */
- public GridHadoopDataOutStream(GridUnsafeMemory mem) {
- this.mem = mem;
- }
-
- /**
- * @return Buffer.
- */
- public GridHadoopOffheapBuffer buffer() {
- return buf;
- }
-
- /**
- * @param size Size.
- * @return Old pointer or {@code 0} if move was impossible.
- */
- public long move(long size) {
- return buf.move(size);
- }
-
- /** {@inheritDoc} */
- @Override public void write(int b) {
- writeByte(b);
- }
-
- /** {@inheritDoc} */
- @Override public void write(byte[] b) {
- write(b, 0, b.length);
- }
-
- /** {@inheritDoc} */
- @Override public void write(byte[] b, int off, int len) {
- UNSAFE.copyMemory(b, BYTE_ARR_OFF + off, null, move(len), len);
- }
-
- /** {@inheritDoc} */
- @Override public void writeBoolean(boolean v) {
- writeByte(v ? 1 : 0);
- }
-
- /** {@inheritDoc} */
- @Override public void writeByte(int v) {
- mem.writeByte(move(1), (byte)v);
- }
-
- /** {@inheritDoc} */
- @Override public void writeShort(int v) {
- mem.writeShort(move(2), (short)v);
- }
-
- /** {@inheritDoc} */
- @Override public void writeChar(int v) {
- writeShort(v);
- }
-
- /** {@inheritDoc} */
- @Override public void writeInt(int v) {
- mem.writeInt(move(4), v);
- }
-
- /** {@inheritDoc} */
- @Override public void writeLong(long v) {
- mem.writeLong(move(8), v);
- }
-
- /** {@inheritDoc} */
- @Override public void writeFloat(float v) {
- mem.writeFloat(move(4), v);
- }
-
- /** {@inheritDoc} */
- @Override public void writeDouble(double v) {
- mem.writeDouble(move(8), v);
- }
-
- /** {@inheritDoc} */
- @Override public void writeBytes(String s) {
- writeUTF(s);
- }
-
- /** {@inheritDoc} */
- @Override public void writeChars(String s) {
- writeUTF(s);
- }
-
- /** {@inheritDoc} */
- @Override public void writeUTF(String s) {
- byte[] b = s.getBytes(StandardCharsets.UTF_8);
-
- writeInt(b.length);
- write(b);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopOffheapBuffer.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopOffheapBuffer.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopOffheapBuffer.java
deleted file mode 100644
index f9f0e1d..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/GridHadoopOffheapBuffer.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle.streams;
-
-/**
- * Offheap buffer.
- */
-public class GridHadoopOffheapBuffer {
- /** Buffer begin address. */
- private long bufPtr;
-
- /** The first address we do not own. */
- private long bufEnd;
-
- /** Current read or write pointer. */
- private long posPtr;
-
- /**
- * @param bufPtr Pointer to buffer begin.
- * @param bufSize Size of the buffer.
- */
- public GridHadoopOffheapBuffer(long bufPtr, long bufSize) {
- set(bufPtr, bufSize);
- }
-
- /**
- * @param bufPtr Pointer to buffer begin.
- * @param bufSize Size of the buffer.
- */
- public void set(long bufPtr, long bufSize) {
- this.bufPtr = bufPtr;
-
- posPtr = bufPtr;
- bufEnd = bufPtr + bufSize;
- }
-
- /**
- * @return Pointer to internal buffer begin.
- */
- public long begin() {
- return bufPtr;
- }
-
- /**
- * @return Buffer capacity.
- */
- public long capacity() {
- return bufEnd - bufPtr;
- }
-
- /**
- * @return Remaining capacity.
- */
- public long remaining() {
- return bufEnd - posPtr;
- }
-
- /**
- * @return Absolute pointer to the current position inside of the buffer.
- */
- public long pointer() {
- return posPtr;
- }
-
- /**
- * @param ptr Absolute pointer to the current position inside of the buffer.
- */
- public void pointer(long ptr) {
- assert ptr >= bufPtr : bufPtr + " <= " + ptr;
- assert ptr <= bufEnd : bufEnd + " <= " + bufPtr;
-
- posPtr = ptr;
- }
-
- /**
- * @param size Size move on.
- * @return Old position pointer or {@code 0} if move goes beyond the end of the buffer.
- */
- public long move(long size) {
- assert size > 0 : size;
-
- long oldPos = posPtr;
- long newPos = oldPos + size;
-
- if (newPos > bufEnd)
- return 0;
-
- posPtr = newPos;
-
- return oldPos;
- }
-
- /**
- * @param ptr Pointer.
- * @return {@code true} If the given pointer is inside of this buffer.
- */
- public boolean isInside(long ptr) {
- return ptr >= bufPtr && ptr <= bufEnd;
- }
-
- /**
- * Resets position to the beginning of buffer.
- */
- public void reset() {
- posPtr = bufPtr;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataInStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataInStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataInStream.java
new file mode 100644
index 0000000..8a1ee70
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataInStream.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.streams;
+
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+
+import java.io.*;
+import java.nio.charset.*;
+
+/**
+ * Data input stream.
+ */
+public class HadoopDataInStream extends InputStream implements DataInput {
+ /** */
+ private final HadoopOffheapBuffer buf = new HadoopOffheapBuffer(0, 0);
+
+ /** */
+ private final GridUnsafeMemory mem;
+
+ /**
+ * @param mem Memory.
+ */
+ public HadoopDataInStream(GridUnsafeMemory mem) {
+ assert mem != null;
+
+ this.mem = mem;
+ }
+
+ /**
+ * @return Buffer.
+ */
+ public HadoopOffheapBuffer buffer() {
+ return buf;
+ }
+
+ /**
+ * @param size Size.
+ * @return Old pointer.
+ */
+ protected long move(long size) throws IOException {
+ long ptr = buf.move(size);
+
+ assert ptr != 0;
+
+ return ptr;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read() throws IOException {
+ return readUnsignedByte();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(byte[] b, int off, int len) throws IOException {
+ readFully(b, off, len);
+
+ return len;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long skip(long n) throws IOException {
+ move(n);
+
+ return n;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFully(byte[] b) throws IOException {
+ readFully(b, 0, b.length);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFully(byte[] b, int off, int len) throws IOException {
+ mem.readBytes(move(len), b, off, len);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int skipBytes(int n) throws IOException {
+ move(n);
+
+ return n;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readBoolean() throws IOException {
+ byte res = readByte();
+
+ if (res == 1)
+ return true;
+
+ assert res == 0 : res;
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte readByte() throws IOException {
+ return mem.readByte(move(1));
+ }
+
+ /** {@inheritDoc} */
+ @Override public int readUnsignedByte() throws IOException {
+ return readByte() & 0xff;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short readShort() throws IOException {
+ return mem.readShort(move(2));
+ }
+
+ /** {@inheritDoc} */
+ @Override public int readUnsignedShort() throws IOException {
+ return readShort() & 0xffff;
+ }
+
+ /** {@inheritDoc} */
+ @Override public char readChar() throws IOException {
+ return (char)readShort();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int readInt() throws IOException {
+ return mem.readInt(move(4));
+ }
+
+ /** {@inheritDoc} */
+ @Override public long readLong() throws IOException {
+ return mem.readLong(move(8));
+ }
+
+ /** {@inheritDoc} */
+ @Override public float readFloat() throws IOException {
+ return mem.readFloat(move(4));
+ }
+
+ /** {@inheritDoc} */
+ @Override public double readDouble() throws IOException {
+ return mem.readDouble(move(8));
+ }
+
+ /** {@inheritDoc} */
+ @Override public String readLine() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String readUTF() throws IOException {
+ byte[] bytes = new byte[readInt()];
+
+ if (bytes.length != 0)
+ readFully(bytes);
+
+ return new String(bytes, StandardCharsets.UTF_8);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java
new file mode 100644
index 0000000..51bddf9
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.streams;
+
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+
+import java.io.*;
+import java.nio.charset.*;
+
+import static org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory.*;
+
+/**
+ * Data output stream.
+ */
+public class HadoopDataOutStream extends OutputStream implements DataOutput {
+ /** */
+ private final HadoopOffheapBuffer buf = new HadoopOffheapBuffer(0, 0);
+
+ /** */
+ private final GridUnsafeMemory mem;
+
+ /**
+ * @param mem Memory.
+ */
+ public HadoopDataOutStream(GridUnsafeMemory mem) {
+ this.mem = mem;
+ }
+
+ /**
+ * @return Buffer.
+ */
+ public HadoopOffheapBuffer buffer() {
+ return buf;
+ }
+
+ /**
+ * @param size Size.
+ * @return Old pointer or {@code 0} if move was impossible.
+ */
+ public long move(long size) {
+ return buf.move(size);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(int b) {
+ writeByte(b);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(byte[] b) {
+ write(b, 0, b.length);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(byte[] b, int off, int len) {
+ UNSAFE.copyMemory(b, BYTE_ARR_OFF + off, null, move(len), len);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBoolean(boolean v) {
+ writeByte(v ? 1 : 0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeByte(int v) {
+ mem.writeByte(move(1), (byte)v);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeShort(int v) {
+ mem.writeShort(move(2), (short)v);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeChar(int v) {
+ writeShort(v);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeInt(int v) {
+ mem.writeInt(move(4), v);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeLong(long v) {
+ mem.writeLong(move(8), v);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeFloat(float v) {
+ mem.writeFloat(move(4), v);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeDouble(double v) {
+ mem.writeDouble(move(8), v);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBytes(String s) {
+ writeUTF(s);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeChars(String s) {
+ writeUTF(s);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeUTF(String s) {
+ byte[] b = s.getBytes(StandardCharsets.UTF_8);
+
+ writeInt(b.length);
+ write(b);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java
new file mode 100644
index 0000000..a8e7a33
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.streams;
+
+/**
+ * Offheap buffer.
+ */
+public class HadoopOffheapBuffer {
+ /** Buffer begin address. */
+ private long bufPtr;
+
+ /** The first address we do not own. */
+ private long bufEnd;
+
+ /** Current read or write pointer. */
+ private long posPtr;
+
+ /**
+ * @param bufPtr Pointer to buffer begin.
+ * @param bufSize Size of the buffer.
+ */
+ public HadoopOffheapBuffer(long bufPtr, long bufSize) {
+ set(bufPtr, bufSize);
+ }
+
+ /**
+ * @param bufPtr Pointer to buffer begin.
+ * @param bufSize Size of the buffer.
+ */
+ public void set(long bufPtr, long bufSize) {
+ this.bufPtr = bufPtr;
+
+ posPtr = bufPtr;
+ bufEnd = bufPtr + bufSize;
+ }
+
+ /**
+ * @return Pointer to internal buffer begin.
+ */
+ public long begin() {
+ return bufPtr;
+ }
+
+ /**
+ * @return Buffer capacity.
+ */
+ public long capacity() {
+ return bufEnd - bufPtr;
+ }
+
+ /**
+ * @return Remaining capacity.
+ */
+ public long remaining() {
+ return bufEnd - posPtr;
+ }
+
+ /**
+ * @return Absolute pointer to the current position inside of the buffer.
+ */
+ public long pointer() {
+ return posPtr;
+ }
+
+ /**
+ * @param ptr Absolute pointer to the current position inside of the buffer.
+ */
+ public void pointer(long ptr) {
+ assert ptr >= bufPtr : bufPtr + " <= " + ptr;
+ assert ptr <= bufEnd : bufEnd + " <= " + bufPtr;
+
+ posPtr = ptr;
+ }
+
+ /**
+ * @param size Size move on.
+ * @return Old position pointer or {@code 0} if move goes beyond the end of the buffer.
+ */
+ public long move(long size) {
+ assert size > 0 : size;
+
+ long oldPos = posPtr;
+ long newPos = oldPos + size;
+
+ if (newPos > bufEnd)
+ return 0;
+
+ posPtr = newPos;
+
+ return oldPos;
+ }
+
+ /**
+ * @param ptr Pointer.
+ * @return {@code true} If the given pointer is inside of this buffer.
+ */
+ public boolean isInside(long ptr) {
+ return ptr >= bufPtr && ptr <= bufEnd;
+ }
+
+ /**
+ * Resets position to the beginning of buffer.
+ */
+ public void reset() {
+ posPtr = bufPtr;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopEmbeddedTaskExecutor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopEmbeddedTaskExecutor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopEmbeddedTaskExecutor.java
deleted file mode 100644
index fde5400..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopEmbeddedTaskExecutor.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-
-/**
- * Task executor.
- */
-public class GridHadoopEmbeddedTaskExecutor extends GridHadoopTaskExecutorAdapter {
- /** Job tracker. */
- private GridHadoopJobTracker jobTracker;
-
- /** */
- private final ConcurrentMap<GridHadoopJobId, Collection<GridHadoopRunnableTask>> jobs = new ConcurrentHashMap<>();
-
- /** Executor service to run tasks. */
- private GridHadoopExecutorService exec;
-
- /** {@inheritDoc} */
- @Override public void onKernalStart() throws IgniteCheckedException {
- super.onKernalStart();
-
- jobTracker = ctx.jobTracker();
-
- exec = new GridHadoopExecutorService(log, ctx.kernalContext().gridName(),
- ctx.configuration().getMaxParallelTasks(), ctx.configuration().getMaxTaskQueueSize());
- }
-
- /** {@inheritDoc} */
- @Override public void onKernalStop(boolean cancel) {
- if (exec != null) {
- exec.shutdown(3000);
-
- if (cancel) {
- for (GridHadoopJobId jobId : jobs.keySet())
- cancelTasks(jobId);
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public void stop(boolean cancel) {
- if (exec != null && !exec.shutdown(30000))
- U.warn(log, "Failed to finish running tasks in 30 sec.");
- }
-
- /** {@inheritDoc} */
- @Override public void run(final GridHadoopJob job, Collection<GridHadoopTaskInfo> tasks) throws IgniteCheckedException {
- if (log.isDebugEnabled())
- log.debug("Submitting tasks for local execution [locNodeId=" + ctx.localNodeId() +
- ", tasksCnt=" + tasks.size() + ']');
-
- Collection<GridHadoopRunnableTask> executedTasks = jobs.get(job.id());
-
- if (executedTasks == null) {
- executedTasks = new GridConcurrentHashSet<>();
-
- Collection<GridHadoopRunnableTask> extractedCol = jobs.put(job.id(), executedTasks);
-
- assert extractedCol == null;
- }
-
- final Collection<GridHadoopRunnableTask> finalExecutedTasks = executedTasks;
-
- for (final GridHadoopTaskInfo info : tasks) {
- assert info != null;
-
- GridHadoopRunnableTask task = new GridHadoopRunnableTask(log, job, ctx.shuffle().memory(), info,
- ctx.localNodeId()) {
- @Override protected void onTaskFinished(GridHadoopTaskStatus status) {
- if (log.isDebugEnabled())
- log.debug("Finished task execution [jobId=" + job.id() + ", taskInfo=" + info + ", " +
- "waitTime=" + waitTime() + ", execTime=" + executionTime() + ']');
-
- finalExecutedTasks.remove(this);
-
- jobTracker.onTaskFinished(info, status);
- }
-
- @Override protected GridHadoopTaskInput createInput(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- return ctx.shuffle().input(taskCtx);
- }
-
- @Override protected GridHadoopTaskOutput createOutput(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
- return ctx.shuffle().output(taskCtx);
- }
- };
-
- executedTasks.add(task);
-
- exec.submit(task);
- }
- }
-
- /**
- * Cancels all currently running tasks for given job ID and cancels scheduled execution of tasks
- * for this job ID.
- * <p>
- * It is guaranteed that this method will not be called concurrently with
- * {@link #run(GridHadoopJob, Collection)} method. No more job submissions will be performed via
- * {@link #run(GridHadoopJob, Collection)} method for given job ID after this method is called.
- *
- * @param jobId Job ID to cancel.
- */
- @Override public void cancelTasks(GridHadoopJobId jobId) {
- Collection<GridHadoopRunnableTask> executedTasks = jobs.get(jobId);
-
- if (executedTasks != null) {
- for (GridHadoopRunnableTask task : executedTasks)
- task.cancel();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void onJobStateChanged(GridHadoopJobMetadata meta) throws IgniteCheckedException {
- if (meta.phase() == GridHadoopJobPhase.PHASE_COMPLETE) {
- Collection<GridHadoopRunnableTask> executedTasks = jobs.remove(meta.jobId());
-
- assert executedTasks == null || executedTasks.isEmpty();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopExecutorService.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopExecutorService.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopExecutorService.java
deleted file mode 100644
index 9ec637b..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopExecutorService.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor;
-
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.util.worker.*;
-import org.apache.ignite.thread.*;
-import org.jdk8.backport.*;
-
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-
-import static java.util.Collections.*;
-
-/**
- * Executor service without thread pooling.
- */
-public class GridHadoopExecutorService {
- /** */
- private final LinkedBlockingQueue<Callable<?>> queue;
-
- /** */
- private final Collection<GridWorker> workers = newSetFromMap(new ConcurrentHashMap8<GridWorker, Boolean>());
-
- /** */
- private final AtomicInteger active = new AtomicInteger();
-
- /** */
- private final int maxTasks;
-
- /** */
- private final String gridName;
-
- /** */
- private final IgniteLogger log;
-
- /** */
- private volatile boolean shutdown;
-
- /** */
- private final GridWorkerListener lsnr = new GridWorkerListenerAdapter() {
- @Override public void onStopped(GridWorker w) {
- workers.remove(w);
-
- if (shutdown) {
- active.decrementAndGet();
-
- return;
- }
-
- Callable<?> task = queue.poll();
-
- if (task != null)
- startThread(task);
- else {
- active.decrementAndGet();
-
- if (!queue.isEmpty())
- startFromQueue();
- }
- }
- };
-
- /**
- * @param log Logger.
- * @param gridName Grid name.
- * @param maxTasks Max number of tasks.
- * @param maxQueue Max queue length.
- */
- public GridHadoopExecutorService(IgniteLogger log, String gridName, int maxTasks, int maxQueue) {
- assert maxTasks > 0 : maxTasks;
- assert maxQueue > 0 : maxQueue;
-
- this.maxTasks = maxTasks;
- this.queue = new LinkedBlockingQueue<>(maxQueue);
- this.gridName = gridName;
- this.log = log.getLogger(GridHadoopExecutorService.class);
- }
-
- /**
- * @return Number of active workers.
- */
- public int active() {
- return workers.size();
- }
-
- /**
- * Submit task.
- *
- * @param task Task.
- */
- public void submit(Callable<?> task) {
- while (queue.isEmpty()) {
- int active0 = active.get();
-
- if (active0 == maxTasks)
- break;
-
- if (active.compareAndSet(active0, active0 + 1)) {
- startThread(task);
-
- return; // Started in new thread bypassing queue.
- }
- }
-
- try {
- while (!queue.offer(task, 100, TimeUnit.MILLISECONDS)) {
- if (shutdown)
- return; // Rejected due to shutdown.
- }
- }
- catch (InterruptedException e) {
- Thread.currentThread().interrupt();
-
- return;
- }
-
- startFromQueue();
- }
-
- /**
- * Attempts to start task from queue.
- */
- private void startFromQueue() {
- do {
- int active0 = active.get();
-
- if (active0 == maxTasks)
- break;
-
- if (active.compareAndSet(active0, active0 + 1)) {
- Callable<?> task = queue.poll();
-
- if (task == null) {
- int res = active.decrementAndGet();
-
- assert res >= 0 : res;
-
- break;
- }
-
- startThread(task);
- }
- }
- while (!queue.isEmpty());
- }
-
- /**
- * @param task Task.
- */
- private void startThread(final Callable<?> task) {
- String workerName;
-
- if (task instanceof GridHadoopRunnableTask) {
- final GridHadoopTaskInfo i = ((GridHadoopRunnableTask)task).taskInfo();
-
- workerName = "Hadoop-task-" + i.jobId() + "-" + i.type() + "-" + i.taskNumber() + "-" + i.attempt();
- }
- else
- workerName = task.toString();
-
- GridWorker w = new GridWorker(gridName, workerName, log, lsnr) {
- @Override protected void body() {
- try {
- task.call();
- }
- catch (Exception e) {
- log.error("Failed to execute task: " + task, e);
- }
- }
- };
-
- workers.add(w);
-
- if (shutdown)
- w.cancel();
-
- new IgniteThread(w).start();
- }
-
- /**
- * Shuts down this executor service.
- *
- * @param awaitTimeMillis Time in milliseconds to wait for tasks completion.
- * @return {@code true} If all tasks completed.
- */
- public boolean shutdown(long awaitTimeMillis) {
- shutdown = true;
-
- for (GridWorker w : workers)
- w.cancel();
-
- while (awaitTimeMillis > 0 && !workers.isEmpty()) {
- try {
- Thread.sleep(100);
-
- awaitTimeMillis -= 100;
- }
- catch (InterruptedException e) {
- break;
- }
- }
-
- return workers.isEmpty();
- }
-
- /**
- * @return {@code true} If method {@linkplain #shutdown(long)} was already called.
- */
- public boolean isShutdown() {
- return shutdown;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopRunnableTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopRunnableTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopRunnableTask.java
deleted file mode 100644
index fd4a030..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopRunnableTask.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.counter.*;
-import org.apache.ignite.internal.processors.hadoop.shuffle.collections.*;
-import org.apache.ignite.internal.util.offheap.unsafe.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopJobProperty.*;
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopTaskType.*;
-
-/**
- * Runnable task.
- */
-public abstract class GridHadoopRunnableTask implements Callable<Void> {
- /** */
- private final GridUnsafeMemory mem;
-
- /** */
- private final IgniteLogger log;
-
- /** */
- private final GridHadoopJob job;
-
- /** Task to run. */
- private final GridHadoopTaskInfo info;
-
- /** Submit time. */
- private final long submitTs = U.currentTimeMillis();
-
- /** Execution start timestamp. */
- private long execStartTs;
-
- /** Execution end timestamp. */
- private long execEndTs;
-
- /** */
- private GridHadoopMultimap combinerInput;
-
- /** */
- private volatile GridHadoopTaskContext ctx;
-
- /** Set if task is to cancelling. */
- private volatile boolean cancelled;
-
- /** Node id. */
- private UUID nodeId;
-
- /**
- * @param log Log.
- * @param job Job.
- * @param mem Memory.
- * @param info Task info.
- * @param nodeId Node id.
- */
- protected GridHadoopRunnableTask(IgniteLogger log, GridHadoopJob job, GridUnsafeMemory mem, GridHadoopTaskInfo info,
- UUID nodeId) {
- this.nodeId = nodeId;
- this.log = log.getLogger(GridHadoopRunnableTask.class);
- this.job = job;
- this.mem = mem;
- this.info = info;
- }
-
- /**
- * @return Wait time.
- */
- public long waitTime() {
- return execStartTs - submitTs;
- }
-
- /**
- * @return Execution time.
- */
- public long executionTime() {
- return execEndTs - execStartTs;
- }
-
- /** {@inheritDoc} */
- @Override public Void call() throws IgniteCheckedException {
- execStartTs = U.currentTimeMillis();
-
- Throwable err = null;
-
- GridHadoopTaskState state = GridHadoopTaskState.COMPLETED;
-
- GridHadoopPerformanceCounter perfCntr = null;
-
- try {
- ctx = job.getTaskContext(info);
-
- perfCntr = GridHadoopPerformanceCounter.getCounter(ctx.counters(), nodeId);
-
- perfCntr.onTaskSubmit(info, submitTs);
- perfCntr.onTaskPrepare(info, execStartTs);
-
- ctx.prepareTaskEnvironment();
-
- runTask(perfCntr);
-
- if (info.type() == MAP && job.info().hasCombiner()) {
- ctx.taskInfo(new GridHadoopTaskInfo(COMBINE, info.jobId(), info.taskNumber(), info.attempt(), null));
-
- try {
- runTask(perfCntr);
- }
- finally {
- ctx.taskInfo(info);
- }
- }
- }
- catch (GridHadoopTaskCancelledException ignored) {
- state = GridHadoopTaskState.CANCELED;
- }
- catch (Throwable e) {
- state = GridHadoopTaskState.FAILED;
- err = e;
-
- U.error(log, "Task execution failed.", e);
- }
- finally {
- execEndTs = U.currentTimeMillis();
-
- if (perfCntr != null)
- perfCntr.onTaskFinish(info, execEndTs);
-
- onTaskFinished(new GridHadoopTaskStatus(state, err, ctx==null ? null : ctx.counters()));
-
- if (combinerInput != null)
- combinerInput.close();
-
- if (ctx != null)
- ctx.cleanupTaskEnvironment();
- }
-
- return null;
- }
-
- /**
- * @param perfCntr Performance counter.
- * @throws IgniteCheckedException If failed.
- */
- private void runTask(GridHadoopPerformanceCounter perfCntr) throws IgniteCheckedException {
- if (cancelled)
- throw new GridHadoopTaskCancelledException("Task cancelled.");
-
- try (GridHadoopTaskOutput out = createOutputInternal(ctx);
- GridHadoopTaskInput in = createInputInternal(ctx)) {
-
- ctx.input(in);
- ctx.output(out);
-
- perfCntr.onTaskStart(ctx.taskInfo(), U.currentTimeMillis());
-
- ctx.run();
- }
- }
-
- /**
- * Cancel the executed task.
- */
- public void cancel() {
- cancelled = true;
-
- if (ctx != null)
- ctx.cancel();
- }
-
- /**
- * @param status Task status.
- */
- protected abstract void onTaskFinished(GridHadoopTaskStatus status);
-
- /**
- * @param ctx Task context.
- * @return Task input.
- * @throws IgniteCheckedException If failed.
- */
- @SuppressWarnings("unchecked")
- private GridHadoopTaskInput createInputInternal(GridHadoopTaskContext ctx) throws IgniteCheckedException {
- switch (ctx.taskInfo().type()) {
- case SETUP:
- case MAP:
- case COMMIT:
- case ABORT:
- return null;
-
- case COMBINE:
- assert combinerInput != null;
-
- return combinerInput.input(ctx);
-
- default:
- return createInput(ctx);
- }
- }
-
- /**
- * @param ctx Task context.
- * @return Input.
- * @throws IgniteCheckedException If failed.
- */
- protected abstract GridHadoopTaskInput createInput(GridHadoopTaskContext ctx) throws IgniteCheckedException;
-
- /**
- * @param ctx Task info.
- * @return Output.
- * @throws IgniteCheckedException If failed.
- */
- protected abstract GridHadoopTaskOutput createOutput(GridHadoopTaskContext ctx) throws IgniteCheckedException;
-
- /**
- * @param ctx Task info.
- * @return Task output.
- * @throws IgniteCheckedException If failed.
- */
- private GridHadoopTaskOutput createOutputInternal(GridHadoopTaskContext ctx) throws IgniteCheckedException {
- switch (ctx.taskInfo().type()) {
- case SETUP:
- case REDUCE:
- case COMMIT:
- case ABORT:
- return null;
-
- case MAP:
- if (job.info().hasCombiner()) {
- assert combinerInput == null;
-
- combinerInput = get(job.info(), SHUFFLE_COMBINER_NO_SORTING, false) ?
- new GridHadoopHashMultimap(job.info(), mem, get(job.info(), COMBINER_HASHMAP_SIZE, 8 * 1024)):
- new GridHadoopSkipList(job.info(), mem); // TODO replace with red-black tree
-
- return combinerInput.startAdding(ctx);
- }
-
- default:
- return createOutput(ctx);
- }
- }
-
- /**
- * @return Task info.
- */
- public GridHadoopTaskInfo taskInfo() {
- return info;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopTaskExecutorAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopTaskExecutorAdapter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopTaskExecutorAdapter.java
deleted file mode 100644
index 8f66190..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopTaskExecutorAdapter.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
-
-import java.util.*;
-
-/**
- * Common superclass for task executor.
- */
-public abstract class GridHadoopTaskExecutorAdapter extends GridHadoopComponent {
- /**
- * Runs tasks.
- *
- * @param job Job.
- * @param tasks Tasks.
- * @throws IgniteCheckedException If failed.
- */
- public abstract void run(final GridHadoopJob job, Collection<GridHadoopTaskInfo> tasks) throws IgniteCheckedException;
-
- /**
- * Cancels all currently running tasks for given job ID and cancels scheduled execution of tasks
- * for this job ID.
- * <p>
- * It is guaranteed that this method will not be called concurrently with
- * {@link #run(GridHadoopJob, Collection)} method. No more job submissions will be performed via
- * {@link #run(GridHadoopJob, Collection)} method for given job ID after this method is called.
- *
- * @param jobId Job ID to cancel.
- */
- public abstract void cancelTasks(GridHadoopJobId jobId) throws IgniteCheckedException;
-
- /**
- * On job state change callback;
- *
- * @param meta Job metadata.
- */
- public abstract void onJobStateChanged(GridHadoopJobMetadata meta) throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopTaskState.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopTaskState.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopTaskState.java
deleted file mode 100644
index d1eaa66..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopTaskState.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor;
-
-/**
-* State of the task.
-*/
-public enum GridHadoopTaskState {
- /** Running task. */
- RUNNING,
-
- /** Completed task. */
- COMPLETED,
-
- /** Failed task. */
- FAILED,
-
- /** Canceled task. */
- CANCELED,
-
- /** Process crashed. */
- CRASHED
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopTaskStatus.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopTaskStatus.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopTaskStatus.java
deleted file mode 100644
index 89ef8c1..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/GridHadoopTaskStatus.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.taskexecutor;
-
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-
-/**
- * Task status.
- */
-public class GridHadoopTaskStatus implements Externalizable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private GridHadoopTaskState state;
-
- /** */
- private Throwable failCause;
-
- /** */
- private GridHadoopCounters cntrs;
-
- /**
- * Default constructor required by {@link Externalizable}.
- */
- public GridHadoopTaskStatus() {
- // No-op.
- }
-
- /**
- * Creates new instance.
- *
- * @param state Task state.
- * @param failCause Failure cause (if any).
- */
- public GridHadoopTaskStatus(GridHadoopTaskState state, @Nullable Throwable failCause) {
- this(state, failCause, null);
- }
-
- /**
- * Creates new instance.
- *
- * @param state Task state.
- * @param failCause Failure cause (if any).
- * @param cntrs Task counters.
- */
- public GridHadoopTaskStatus(GridHadoopTaskState state, @Nullable Throwable failCause,
- @Nullable GridHadoopCounters cntrs) {
- assert state != null;
-
- this.state = state;
- this.failCause = failCause;
- this.cntrs = cntrs;
- }
-
- /**
- * @return State.
- */
- public GridHadoopTaskState state() {
- return state;
- }
-
- /**
- * @return Fail cause.
- */
- @Nullable public Throwable failCause() {
- return failCause;
- }
-
- /**
- * @return Counters.
- */
- @Nullable public GridHadoopCounters counters() {
- return cntrs;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridHadoopTaskStatus.class, this);
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeObject(state);
- out.writeObject(failCause);
- out.writeObject(cntrs);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- state = (GridHadoopTaskState)in.readObject();
- failCause = (Throwable)in.readObject();
- cntrs = (GridHadoopCounters)in.readObject();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopEmbeddedTaskExecutor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopEmbeddedTaskExecutor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopEmbeddedTaskExecutor.java
new file mode 100644
index 0000000..a3c20d8
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopEmbeddedTaskExecutor.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+
+/**
+ * Task executor.
+ */
+public class HadoopEmbeddedTaskExecutor extends HadoopTaskExecutorAdapter {
+ /** Job tracker. */
+ private HadoopJobTracker jobTracker;
+
+ /** */
+ private final ConcurrentMap<HadoopJobId, Collection<HadoopRunnableTask>> jobs = new ConcurrentHashMap<>();
+
+ /** Executor service to run tasks. */
+ private HadoopExecutorService exec;
+
+ /** {@inheritDoc} */
+ @Override public void onKernalStart() throws IgniteCheckedException {
+ super.onKernalStart();
+
+ jobTracker = ctx.jobTracker();
+
+ exec = new HadoopExecutorService(log, ctx.kernalContext().gridName(),
+ ctx.configuration().getMaxParallelTasks(), ctx.configuration().getMaxTaskQueueSize());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onKernalStop(boolean cancel) {
+ if (exec != null) {
+ exec.shutdown(3000);
+
+ if (cancel) {
+ for (HadoopJobId jobId : jobs.keySet())
+ cancelTasks(jobId);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop(boolean cancel) {
+ if (exec != null && !exec.shutdown(30000))
+ U.warn(log, "Failed to finish running tasks in 30 sec.");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run(final HadoopJob job, Collection<HadoopTaskInfo> tasks) throws IgniteCheckedException {
+ if (log.isDebugEnabled())
+ log.debug("Submitting tasks for local execution [locNodeId=" + ctx.localNodeId() +
+ ", tasksCnt=" + tasks.size() + ']');
+
+ Collection<HadoopRunnableTask> executedTasks = jobs.get(job.id());
+
+ if (executedTasks == null) {
+ executedTasks = new GridConcurrentHashSet<>();
+
+ Collection<HadoopRunnableTask> extractedCol = jobs.put(job.id(), executedTasks);
+
+ assert extractedCol == null;
+ }
+
+ final Collection<HadoopRunnableTask> finalExecutedTasks = executedTasks;
+
+ for (final HadoopTaskInfo info : tasks) {
+ assert info != null;
+
+ HadoopRunnableTask task = new HadoopRunnableTask(log, job, ctx.shuffle().memory(), info,
+ ctx.localNodeId()) {
+ @Override protected void onTaskFinished(HadoopTaskStatus status) {
+ if (log.isDebugEnabled())
+ log.debug("Finished task execution [jobId=" + job.id() + ", taskInfo=" + info + ", " +
+ "waitTime=" + waitTime() + ", execTime=" + executionTime() + ']');
+
+ finalExecutedTasks.remove(this);
+
+ jobTracker.onTaskFinished(info, status);
+ }
+
+ @Override protected HadoopTaskInput createInput(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ return ctx.shuffle().input(taskCtx);
+ }
+
+ @Override protected HadoopTaskOutput createOutput(HadoopTaskContext taskCtx) throws IgniteCheckedException {
+ return ctx.shuffle().output(taskCtx);
+ }
+ };
+
+ executedTasks.add(task);
+
+ exec.submit(task);
+ }
+ }
+
+ /**
+ * Cancels all currently running tasks for given job ID and cancels scheduled execution of tasks
+ * for this job ID.
+ * <p>
+ * It is guaranteed that this method will not be called concurrently with
+ * {@link #run(org.apache.ignite.internal.processors.hadoop.HadoopJob, Collection)} method. No more job submissions will be performed via
+ * {@link #run(org.apache.ignite.internal.processors.hadoop.HadoopJob, Collection)} method for given job ID after this method is called.
+ *
+ * @param jobId Job ID to cancel.
+ */
+ @Override public void cancelTasks(HadoopJobId jobId) {
+ Collection<HadoopRunnableTask> executedTasks = jobs.get(jobId);
+
+ if (executedTasks != null) {
+ for (HadoopRunnableTask task : executedTasks)
+ task.cancel();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onJobStateChanged(HadoopJobMetadata meta) throws IgniteCheckedException {
+ if (meta.phase() == HadoopJobPhase.PHASE_COMPLETE) {
+ Collection<HadoopRunnableTask> executedTasks = jobs.remove(meta.jobId());
+
+ assert executedTasks == null || executedTasks.isEmpty();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopExecutorService.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopExecutorService.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopExecutorService.java
new file mode 100644
index 0000000..1c318e9
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopExecutorService.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor;
+
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.util.worker.*;
+import org.apache.ignite.thread.*;
+import org.jdk8.backport.*;
+
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+
+import static java.util.Collections.*;
+
+/**
+ * Executor service without thread pooling.
+ */
+public class HadoopExecutorService {
+ /** */
+ private final LinkedBlockingQueue<Callable<?>> queue;
+
+ /** */
+ private final Collection<GridWorker> workers = newSetFromMap(new ConcurrentHashMap8<GridWorker, Boolean>());
+
+ /** */
+ private final AtomicInteger active = new AtomicInteger();
+
+ /** */
+ private final int maxTasks;
+
+ /** */
+ private final String gridName;
+
+ /** */
+ private final IgniteLogger log;
+
+ /** */
+ private volatile boolean shutdown;
+
+ /** */
+ private final GridWorkerListener lsnr = new GridWorkerListenerAdapter() {
+ @Override public void onStopped(GridWorker w) {
+ workers.remove(w);
+
+ if (shutdown) {
+ active.decrementAndGet();
+
+ return;
+ }
+
+ Callable<?> task = queue.poll();
+
+ if (task != null)
+ startThread(task);
+ else {
+ active.decrementAndGet();
+
+ if (!queue.isEmpty())
+ startFromQueue();
+ }
+ }
+ };
+
+ /**
+ * @param log Logger.
+ * @param gridName Grid name.
+ * @param maxTasks Max number of tasks.
+ * @param maxQueue Max queue length.
+ */
+ public HadoopExecutorService(IgniteLogger log, String gridName, int maxTasks, int maxQueue) {
+ assert maxTasks > 0 : maxTasks;
+ assert maxQueue > 0 : maxQueue;
+
+ this.maxTasks = maxTasks;
+ this.queue = new LinkedBlockingQueue<>(maxQueue);
+ this.gridName = gridName;
+ this.log = log.getLogger(HadoopExecutorService.class);
+ }
+
+ /**
+ * @return Number of active workers.
+ */
+ public int active() {
+ return workers.size();
+ }
+
+ /**
+ * Submit task.
+ *
+ * @param task Task.
+ */
+ public void submit(Callable<?> task) {
+ while (queue.isEmpty()) {
+ int active0 = active.get();
+
+ if (active0 == maxTasks)
+ break;
+
+ if (active.compareAndSet(active0, active0 + 1)) {
+ startThread(task);
+
+ return; // Started in new thread bypassing queue.
+ }
+ }
+
+ try {
+ while (!queue.offer(task, 100, TimeUnit.MILLISECONDS)) {
+ if (shutdown)
+ return; // Rejected due to shutdown.
+ }
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ return;
+ }
+
+ startFromQueue();
+ }
+
+ /**
+ * Attempts to start task from queue.
+ */
+ private void startFromQueue() {
+ do {
+ int active0 = active.get();
+
+ if (active0 == maxTasks)
+ break;
+
+ if (active.compareAndSet(active0, active0 + 1)) {
+ Callable<?> task = queue.poll();
+
+ if (task == null) {
+ int res = active.decrementAndGet();
+
+ assert res >= 0 : res;
+
+ break;
+ }
+
+ startThread(task);
+ }
+ }
+ while (!queue.isEmpty());
+ }
+
+ /**
+ * @param task Task.
+ */
+ private void startThread(final Callable<?> task) {
+ String workerName;
+
+ if (task instanceof HadoopRunnableTask) {
+ final HadoopTaskInfo i = ((HadoopRunnableTask)task).taskInfo();
+
+ workerName = "Hadoop-task-" + i.jobId() + "-" + i.type() + "-" + i.taskNumber() + "-" + i.attempt();
+ }
+ else
+ workerName = task.toString();
+
+ GridWorker w = new GridWorker(gridName, workerName, log, lsnr) {
+ @Override protected void body() {
+ try {
+ task.call();
+ }
+ catch (Exception e) {
+ log.error("Failed to execute task: " + task, e);
+ }
+ }
+ };
+
+ workers.add(w);
+
+ if (shutdown)
+ w.cancel();
+
+ new IgniteThread(w).start();
+ }
+
+ /**
+ * Shuts down this executor service.
+ *
+ * @param awaitTimeMillis Time in milliseconds to wait for tasks completion.
+ * @return {@code true} If all tasks completed.
+ */
+ public boolean shutdown(long awaitTimeMillis) {
+ shutdown = true;
+
+ for (GridWorker w : workers)
+ w.cancel();
+
+ while (awaitTimeMillis > 0 && !workers.isEmpty()) {
+ try {
+ Thread.sleep(100);
+
+ awaitTimeMillis -= 100;
+ }
+ catch (InterruptedException e) {
+ break;
+ }
+ }
+
+ return workers.isEmpty();
+ }
+
+ /**
+ * @return {@code true} If method {@linkplain #shutdown(long)} was already called.
+ */
+ public boolean isShutdown() {
+ return shutdown;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopRunnableTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopRunnableTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopRunnableTask.java
new file mode 100644
index 0000000..2b36267
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopRunnableTask.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.apache.ignite.internal.processors.hadoop.shuffle.collections.*;
+import org.apache.ignite.internal.util.offheap.unsafe.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.*;
+import static org.apache.ignite.internal.processors.hadoop.HadoopTaskType.*;
+
+/**
+ * Runnable task.
+ */
+public abstract class HadoopRunnableTask implements Callable<Void> {
+ /** */
+ private final GridUnsafeMemory mem;
+
+ /** */
+ private final IgniteLogger log;
+
+ /** */
+ private final HadoopJob job;
+
+ /** Task to run. */
+ private final HadoopTaskInfo info;
+
+ /** Submit time. */
+ private final long submitTs = U.currentTimeMillis();
+
+ /** Execution start timestamp. */
+ private long execStartTs;
+
+ /** Execution end timestamp. */
+ private long execEndTs;
+
+ /** */
+ private HadoopMultimap combinerInput;
+
+ /** */
+ private volatile HadoopTaskContext ctx;
+
+ /** Set if task is to cancelling. */
+ private volatile boolean cancelled;
+
+ /** Node id. */
+ private UUID nodeId;
+
+ /**
+ * @param log Log.
+ * @param job Job.
+ * @param mem Memory.
+ * @param info Task info.
+ * @param nodeId Node id.
+ */
+ protected HadoopRunnableTask(IgniteLogger log, HadoopJob job, GridUnsafeMemory mem, HadoopTaskInfo info,
+ UUID nodeId) {
+ this.nodeId = nodeId;
+ this.log = log.getLogger(HadoopRunnableTask.class);
+ this.job = job;
+ this.mem = mem;
+ this.info = info;
+ }
+
+ /**
+ * @return Wait time.
+ */
+ public long waitTime() {
+ return execStartTs - submitTs;
+ }
+
+ /**
+ * @return Execution time.
+ */
+ public long executionTime() {
+ return execEndTs - execStartTs;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Void call() throws IgniteCheckedException {
+ execStartTs = U.currentTimeMillis();
+
+ Throwable err = null;
+
+ HadoopTaskState state = HadoopTaskState.COMPLETED;
+
+ HadoopPerformanceCounter perfCntr = null;
+
+ try {
+ ctx = job.getTaskContext(info);
+
+ perfCntr = HadoopPerformanceCounter.getCounter(ctx.counters(), nodeId);
+
+ perfCntr.onTaskSubmit(info, submitTs);
+ perfCntr.onTaskPrepare(info, execStartTs);
+
+ ctx.prepareTaskEnvironment();
+
+ runTask(perfCntr);
+
+ if (info.type() == MAP && job.info().hasCombiner()) {
+ ctx.taskInfo(new HadoopTaskInfo(COMBINE, info.jobId(), info.taskNumber(), info.attempt(), null));
+
+ try {
+ runTask(perfCntr);
+ }
+ finally {
+ ctx.taskInfo(info);
+ }
+ }
+ }
+ catch (HadoopTaskCancelledException ignored) {
+ state = HadoopTaskState.CANCELED;
+ }
+ catch (Throwable e) {
+ state = HadoopTaskState.FAILED;
+ err = e;
+
+ U.error(log, "Task execution failed.", e);
+ }
+ finally {
+ execEndTs = U.currentTimeMillis();
+
+ if (perfCntr != null)
+ perfCntr.onTaskFinish(info, execEndTs);
+
+ onTaskFinished(new HadoopTaskStatus(state, err, ctx==null ? null : ctx.counters()));
+
+ if (combinerInput != null)
+ combinerInput.close();
+
+ if (ctx != null)
+ ctx.cleanupTaskEnvironment();
+ }
+
+ return null;
+ }
+
+ /**
+ * @param perfCntr Performance counter.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void runTask(HadoopPerformanceCounter perfCntr) throws IgniteCheckedException {
+ if (cancelled)
+ throw new HadoopTaskCancelledException("Task cancelled.");
+
+ try (HadoopTaskOutput out = createOutputInternal(ctx);
+ HadoopTaskInput in = createInputInternal(ctx)) {
+
+ ctx.input(in);
+ ctx.output(out);
+
+ perfCntr.onTaskStart(ctx.taskInfo(), U.currentTimeMillis());
+
+ ctx.run();
+ }
+ }
+
+ /**
+ * Cancel the executed task.
+ */
+ public void cancel() {
+ cancelled = true;
+
+ if (ctx != null)
+ ctx.cancel();
+ }
+
+ /**
+ * @param status Task status.
+ */
+ protected abstract void onTaskFinished(HadoopTaskStatus status);
+
+ /**
+ * @param ctx Task context.
+ * @return Task input.
+ * @throws IgniteCheckedException If failed.
+ */
+ @SuppressWarnings("unchecked")
+ private HadoopTaskInput createInputInternal(HadoopTaskContext ctx) throws IgniteCheckedException {
+ switch (ctx.taskInfo().type()) {
+ case SETUP:
+ case MAP:
+ case COMMIT:
+ case ABORT:
+ return null;
+
+ case COMBINE:
+ assert combinerInput != null;
+
+ return combinerInput.input(ctx);
+
+ default:
+ return createInput(ctx);
+ }
+ }
+
+ /**
+ * @param ctx Task context.
+ * @return Input.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected abstract HadoopTaskInput createInput(HadoopTaskContext ctx) throws IgniteCheckedException;
+
+ /**
+ * @param ctx Task info.
+ * @return Output.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected abstract HadoopTaskOutput createOutput(HadoopTaskContext ctx) throws IgniteCheckedException;
+
+ /**
+ * @param ctx Task info.
+ * @return Task output.
+ * @throws IgniteCheckedException If failed.
+ */
+ private HadoopTaskOutput createOutputInternal(HadoopTaskContext ctx) throws IgniteCheckedException {
+ switch (ctx.taskInfo().type()) {
+ case SETUP:
+ case REDUCE:
+ case COMMIT:
+ case ABORT:
+ return null;
+
+ case MAP:
+ if (job.info().hasCombiner()) {
+ assert combinerInput == null;
+
+ combinerInput = get(job.info(), SHUFFLE_COMBINER_NO_SORTING, false) ?
+ new HadoopHashMultimap(job.info(), mem, get(job.info(), COMBINER_HASHMAP_SIZE, 8 * 1024)):
+ new HadoopSkipList(job.info(), mem); // TODO replace with red-black tree
+
+ return combinerInput.startAdding(ctx);
+ }
+
+ default:
+ return createOutput(ctx);
+ }
+ }
+
+ /**
+ * @return Task info.
+ */
+ public HadoopTaskInfo taskInfo() {
+ return info;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopTaskExecutorAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopTaskExecutorAdapter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopTaskExecutorAdapter.java
new file mode 100644
index 0000000..39b4935
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopTaskExecutorAdapter.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
+
+import java.util.*;
+
+/**
+ * Common superclass for task executor.
+ */
+public abstract class HadoopTaskExecutorAdapter extends HadoopComponent {
+ /**
+ * Runs tasks.
+ *
+ * @param job Job.
+ * @param tasks Tasks.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract void run(final HadoopJob job, Collection<HadoopTaskInfo> tasks) throws IgniteCheckedException;
+
+ /**
+ * Cancels all currently running tasks for given job ID and cancels scheduled execution of tasks
+ * for this job ID.
+ * <p>
+ * It is guaranteed that this method will not be called concurrently with
+ * {@link #run(org.apache.ignite.internal.processors.hadoop.HadoopJob, Collection)} method. No more job submissions will be performed via
+ * {@link #run(org.apache.ignite.internal.processors.hadoop.HadoopJob, Collection)} method for given job ID after this method is called.
+ *
+ * @param jobId Job ID to cancel.
+ */
+ public abstract void cancelTasks(HadoopJobId jobId) throws IgniteCheckedException;
+
+ /**
+ * On job state change callback;
+ *
+ * @param meta Job metadata.
+ */
+ public abstract void onJobStateChanged(HadoopJobMetadata meta) throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopTaskState.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopTaskState.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopTaskState.java
new file mode 100644
index 0000000..cf2a28e
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopTaskState.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor;
+
+/**
+* State of the task.
+*/
+public enum HadoopTaskState {
+ /** Running task. */
+ RUNNING,
+
+ /** Completed task. */
+ COMPLETED,
+
+ /** Failed task. */
+ FAILED,
+
+ /** Canceled task. */
+ CANCELED,
+
+ /** Process crashed. */
+ CRASHED
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopTaskStatus.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopTaskStatus.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopTaskStatus.java
new file mode 100644
index 0000000..c5ee16c
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopTaskStatus.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.taskexecutor;
+
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+
+/**
+ * Task status.
+ */
+public class HadoopTaskStatus implements Externalizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private HadoopTaskState state;
+
+ /** */
+ private Throwable failCause;
+
+ /** */
+ private HadoopCounters cntrs;
+
+ /**
+ * Default constructor required by {@link Externalizable}.
+ */
+ public HadoopTaskStatus() {
+ // No-op.
+ }
+
+ /**
+ * Creates new instance.
+ *
+ * @param state Task state.
+ * @param failCause Failure cause (if any).
+ */
+ public HadoopTaskStatus(HadoopTaskState state, @Nullable Throwable failCause) {
+ this(state, failCause, null);
+ }
+
+ /**
+ * Creates new instance.
+ *
+ * @param state Task state.
+ * @param failCause Failure cause (if any).
+ * @param cntrs Task counters.
+ */
+ public HadoopTaskStatus(HadoopTaskState state, @Nullable Throwable failCause,
+ @Nullable HadoopCounters cntrs) {
+ assert state != null;
+
+ this.state = state;
+ this.failCause = failCause;
+ this.cntrs = cntrs;
+ }
+
+ /**
+ * @return State.
+ */
+ public HadoopTaskState state() {
+ return state;
+ }
+
+ /**
+ * @return Fail cause.
+ */
+ @Nullable public Throwable failCause() {
+ return failCause;
+ }
+
+ /**
+ * @return Counters.
+ */
+ @Nullable public HadoopCounters counters() {
+ return cntrs;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopTaskStatus.class, this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeObject(state);
+ out.writeObject(failCause);
+ out.writeObject(cntrs);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ state = (HadoopTaskState)in.readObject();
+ failCause = (Throwable)in.readObject();
+ cntrs = (HadoopCounters)in.readObject();
+ }
+}
[12/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopDualAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopDualAbstractSelfTest.java
deleted file mode 100644
index a54e264..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopDualAbstractSelfTest.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.ignite.*;
-import org.apache.ignite.cache.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.hadoop.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.spi.discovery.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
-import org.apache.ignite.testframework.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.cache.CacheMode.*;
-import static org.apache.ignite.igfs.IgfsMode.*;
-import static org.apache.ignite.igfs.hadoop.IgfsHadoopParameters.*;
-import static org.apache.ignite.internal.processors.igfs.IgfsAbstractSelfTest.*;
-
-/**
- * Tests for IGFS working in mode when remote file system exists: DUAL_SYNC, DUAL_ASYNC.
- */
-public abstract class IgfsHadoopDualAbstractSelfTest extends IgfsCommonAbstractTest {
- /** IGFS block size. */
- protected static final int IGFS_BLOCK_SIZE = 512 * 1024;
-
- /** Amount of blocks to prefetch. */
- protected static final int PREFETCH_BLOCKS = 1;
-
- /** Amount of sequential block reads before prefetch is triggered. */
- protected static final int SEQ_READS_BEFORE_PREFETCH = 2;
-
- /** Secondary file system URI. */
- protected static final String SECONDARY_URI = "igfs://igfs-secondary:grid-secondary@127.0.0.1:11500/";
-
- /** Secondary file system configuration path. */
- protected static final String SECONDARY_CFG = "modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml";
-
- /** Primary file system URI. */
- protected static final String PRIMARY_URI = "igfs://igfs:grid@/";
-
- /** Primary file system configuration path. */
- protected static final String PRIMARY_CFG = "modules/core/src/test/config/hadoop/core-site-loopback.xml";
-
- /** Primary file system REST endpoint configuration map. */
- protected static final Map<String, String> PRIMARY_REST_CFG = new HashMap<String, String>() {{
- put("type", "tcp");
- put("port", "10500");
- }};
-
- /** Secondary file system REST endpoint configuration map. */
- protected static final Map<String, String> SECONDARY_REST_CFG = new HashMap<String, String>() {{
- put("type", "tcp");
- put("port", "11500");
- }};
-
- /** Directory. */
- protected static final IgfsPath DIR = new IgfsPath("/dir");
-
- /** Sub-directory. */
- protected static final IgfsPath SUBDIR = new IgfsPath(DIR, "subdir");
-
- /** File. */
- protected static final IgfsPath FILE = new IgfsPath(SUBDIR, "file");
-
- /** Default data chunk (128 bytes). */
- protected static byte[] chunk;
-
- /** Primary IGFS. */
- protected static IgfsImpl igfs;
-
- /** Secondary IGFS. */
- protected static IgfsImpl igfsSecondary;
-
- /** IGFS mode. */
- protected final IgfsMode mode;
-
- /**
- * Constructor.
- *
- * @param mode IGFS mode.
- */
- protected IgfsHadoopDualAbstractSelfTest(IgfsMode mode) {
- this.mode = mode;
- assert mode == DUAL_SYNC || mode == DUAL_ASYNC;
- }
-
- /**
- * Start grid with IGFS.
- *
- * @param gridName Grid name.
- * @param igfsName IGFS name
- * @param mode IGFS mode.
- * @param secondaryFs Secondary file system (optional).
- * @param restCfg Rest configuration string (optional).
- * @return Started grid instance.
- * @throws Exception If failed.
- */
- protected Ignite startGridWithIgfs(String gridName, String igfsName, IgfsMode mode,
- @Nullable Igfs secondaryFs, @Nullable Map<String, String> restCfg) throws Exception {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("dataCache");
- igfsCfg.setMetaCacheName("metaCache");
- igfsCfg.setName(igfsName);
- igfsCfg.setBlockSize(IGFS_BLOCK_SIZE);
- igfsCfg.setDefaultMode(mode);
- igfsCfg.setIpcEndpointConfiguration(restCfg);
- igfsCfg.setSecondaryFileSystem(secondaryFs);
- igfsCfg.setPrefetchBlocks(PREFETCH_BLOCKS);
- igfsCfg.setSequentialReadsBeforePrefetch(SEQ_READS_BEFORE_PREFETCH);
-
- CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
-
- dataCacheCfg.setName("dataCache");
- dataCacheCfg.setCacheMode(PARTITIONED);
- dataCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(2));
- dataCacheCfg.setBackups(0);
- dataCacheCfg.setQueryIndexEnabled(false);
- dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
- dataCacheCfg.setOffHeapMaxMemory(0);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("metaCache");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- IgniteConfiguration cfg = new IgniteConfiguration();
-
- cfg.setGridName(gridName);
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
-
- cfg.setLocalHost("127.0.0.1");
- cfg.setConnectorConfiguration(null);
-
- return G.start(cfg);
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- chunk = new byte[128];
-
- for (int i = 0; i < chunk.length; i++)
- chunk[i] = (byte)i;
-
- Ignite igniteSecondary = startGridWithIgfs("grid-secondary", "igfs-secondary", PRIMARY, null, SECONDARY_REST_CFG);
-
- Igfs hadoopFs = new IgfsHadoopFileSystemWrapper(SECONDARY_URI, SECONDARY_CFG);
-
- Ignite ignite = startGridWithIgfs("grid", "igfs", mode, hadoopFs, PRIMARY_REST_CFG);
-
- igfsSecondary = (IgfsImpl) igniteSecondary.fileSystem("igfs-secondary");
- igfs = (IgfsImpl) ignite.fileSystem("igfs");
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- clear(igfs);
- clear(igfsSecondary);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- G.stopAll(true);
- }
-
- /**
- * Convenient method to group paths.
- *
- * @param paths Paths to group.
- * @return Paths as array.
- */
- protected IgfsPath[] paths(IgfsPath... paths) {
- return paths;
- }
-
- /**
- * Check how prefetch override works.
- *
- * @throws Exception IF failed.
- */
- public void testOpenPrefetchOverride() throws Exception {
- create(igfsSecondary, paths(DIR, SUBDIR), paths(FILE));
-
- // Write enough data to the secondary file system.
- final int blockSize = IGFS_BLOCK_SIZE;
-
- IgfsOutputStream out = igfsSecondary.append(FILE, false);
-
- int totalWritten = 0;
-
- while (totalWritten < blockSize * 2 + chunk.length) {
- out.write(chunk);
-
- totalWritten += chunk.length;
- }
-
- out.close();
-
- awaitFileClose(igfsSecondary, FILE);
-
- // Instantiate file system with overridden "seq reads before prefetch" property.
- Configuration cfg = new Configuration();
-
- cfg.addResource(U.resolveIgniteUrl(PRIMARY_CFG));
-
- int seqReads = SEQ_READS_BEFORE_PREFETCH + 1;
-
- cfg.setInt(String.format(PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH, "igfs:grid@"), seqReads);
-
- FileSystem fs = FileSystem.get(new URI(PRIMARY_URI), cfg);
-
- // Read the first two blocks.
- Path fsHome = new Path(PRIMARY_URI);
- Path dir = new Path(fsHome, DIR.name());
- Path subdir = new Path(dir, SUBDIR.name());
- Path file = new Path(subdir, FILE.name());
-
- FSDataInputStream fsIn = fs.open(file);
-
- final byte[] readBuf = new byte[blockSize * 2];
-
- fsIn.readFully(0, readBuf, 0, readBuf.length);
-
- // Wait for a while for prefetch to finish (if any).
- IgfsMetaManager meta = igfs.context().meta();
-
- IgfsFileInfo info = meta.info(meta.fileId(FILE));
-
- IgfsBlockKey key = new IgfsBlockKey(info.id(), info.affinityKey(), info.evictExclude(), 2);
-
- GridCache<IgfsBlockKey, byte[]> dataCache = igfs.context().kernalContext().cache().cache(
- igfs.configuration().getDataCacheName());
-
- for (int i = 0; i < 10; i++) {
- if (dataCache.containsKey(key))
- break;
- else
- U.sleep(100);
- }
-
- fsIn.close();
-
- // Remove the file from the secondary file system.
- igfsSecondary.delete(FILE, false);
-
- // Try reading the third block. Should fail.
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- IgfsInputStream in0 = igfs.open(FILE);
-
- in0.seek(blockSize * 2);
-
- try {
- in0.read(readBuf);
- }
- finally {
- U.closeQuiet(in0);
- }
-
- return null;
- }
- }, IOException.class,
- "Failed to read data due to secondary file system exception: /dir/subdir/file");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopDualAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopDualAsyncSelfTest.java
deleted file mode 100644
index c99b3c8..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopDualAsyncSelfTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * Tests for DUAL_ASYNC mode.
- */
-public class IgfsHadoopDualAsyncSelfTest extends IgfsHadoopDualAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopDualAsyncSelfTest() {
- super(DUAL_ASYNC);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopDualSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopDualSyncSelfTest.java
deleted file mode 100644
index ffcd092..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsHadoopDualSyncSelfTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * Tests for DUAL_SYNC mode.
- */
-public class IgfsHadoopDualSyncSelfTest extends IgfsHadoopDualAbstractSelfTest {
- /**
- * Constructor.
- */
- public IgfsHadoopDualSyncSelfTest() {
- super(DUAL_SYNC);
- }
-}
[39/45] incubator-ignite git commit: IGNITE-386: Squashed changes.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNodeStartup.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNodeStartup.java b/modules/core/src/test/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNodeStartup.java
index 400bbb1..8051a3e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNodeStartup.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNodeStartup.java
@@ -41,7 +41,7 @@ public class IpcSharedMemoryNodeStartup {
public static void main(String[] args) throws Exception{
IgniteConfiguration cfg = new IgniteConfiguration();
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
+ FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
@@ -60,7 +60,7 @@ public class IpcSharedMemoryNodeStartup {
igfsCfg.setMetaCacheName("partitioned");
igfsCfg.setName("igfs");
- cfg.setIgfsConfiguration(igfsCfg);
+ cfg.setFileSystemConfiguration(igfsCfg);
CacheConfiguration cacheCfg = new CacheConfiguration();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
index 58478d3..220d5d6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
@@ -176,12 +176,12 @@ public class IgniteMock implements Ignite {
}
/** {@inheritDoc} */
- @Override public IgniteFs fileSystem(String name) {
+ @Override public IgniteFileSystem fileSystem(String name) {
return null;
}
/** {@inheritDoc} */
- @Override public Collection<IgniteFs> fileSystems() {
+ @Override public Collection<IgniteFileSystem> fileSystems() {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
index 8f06117..cea510a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
@@ -48,7 +48,7 @@ public class IgniteIgfsTestSuite extends TestSuite {
if (U.isWindows())
suite.addTest(new TestSuite(IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest.class));
- suite.addTest(new TestSuite(GridCacheIgfsPerBlockLruEvictionPolicySelfTest.class));
+ suite.addTest(new TestSuite(IgfsCachePerBlockLruEvictionPolicySelfTest.class));
suite.addTest(new TestSuite(IgfsStreamsSelfTest.class));
suite.addTest(new TestSuite(IgfsModesSelfTest.class));
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocol.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocol.java b/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocol.java
deleted file mode 100644
index bd31951..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocol.java
+++ /dev/null
@@ -1,334 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.client.hadoop;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.ipc.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.protocol.*;
-import org.apache.hadoop.mapreduce.security.token.delegation.*;
-import org.apache.hadoop.mapreduce.v2.*;
-import org.apache.hadoop.mapreduce.v2.jobhistory.*;
-import org.apache.hadoop.security.*;
-import org.apache.hadoop.security.authorize.*;
-import org.apache.hadoop.security.token.*;
-import org.apache.ignite.*;
-import org.apache.ignite.client.hadoop.counter.*;
-import org.apache.ignite.internal.client.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.proto.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-
-import static org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.*;
-
-/**
- * Hadoop client protocol.
- */
-public class GridHadoopClientProtocol implements ClientProtocol {
- /** Ignite framework name property. */
- public static final String FRAMEWORK_NAME = "ignite";
-
- /** Protocol version. */
- private static final long PROTO_VER = 1L;
-
- /** Default Ignite system directory. */
- private static final String SYS_DIR = ".ignite/system";
-
- /** Configuration. */
- private final Configuration conf;
-
- /** Ignite client. */
- private volatile GridClient cli;
-
- /** Last received version. */
- private long lastVer = -1;
-
- /** Last received status. */
- private GridHadoopJobStatus lastStatus;
-
- /**
- * Constructor.
- *
- * @param conf Configuration.
- * @param cli Ignite client.
- */
- GridHadoopClientProtocol(Configuration conf, GridClient cli) {
- assert cli != null;
-
- this.conf = conf;
- this.cli = cli;
- }
-
- /** {@inheritDoc} */
- @Override public JobID getNewJobID() throws IOException, InterruptedException {
- try {
- conf.setLong(REQ_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());
-
- GridHadoopJobId jobID = cli.compute().execute(GridHadoopProtocolNextTaskIdTask.class.getName(), null);
-
- conf.setLong(RESPONSE_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());
-
- return new JobID(jobID.globalId().toString(), jobID.localId());
- }
- catch (GridClientException e) {
- throw new IOException("Failed to get new job ID.", e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) throws IOException,
- InterruptedException {
- try {
- conf.setLong(JOB_SUBMISSION_START_TS_PROPERTY, U.currentTimeMillis());
-
- GridHadoopJobStatus status = cli.compute().execute(GridHadoopProtocolSubmitJobTask.class.getName(),
- new GridHadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId(), createJobInfo(conf)));
-
- if (status == null)
- throw new IOException("Failed to submit job (null status obtained): " + jobId);
-
- return processStatus(status);
- }
- catch (GridClientException | IgniteCheckedException e) {
- throw new IOException("Failed to submit job.", e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public ClusterMetrics getClusterMetrics() throws IOException, InterruptedException {
- return new ClusterMetrics(0, 0, 0, 0, 0, 0, 1000, 1000, 1, 100, 0, 0);
- }
-
- /** {@inheritDoc} */
- @Override public Cluster.JobTrackerStatus getJobTrackerStatus() throws IOException, InterruptedException {
- return Cluster.JobTrackerStatus.RUNNING;
- }
-
- /** {@inheritDoc} */
- @Override public long getTaskTrackerExpiryInterval() throws IOException, InterruptedException {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public AccessControlList getQueueAdmins(String queueName) throws IOException {
- return new AccessControlList("*");
- }
-
- /** {@inheritDoc} */
- @Override public void killJob(JobID jobId) throws IOException, InterruptedException {
- try {
- cli.compute().execute(GridHadoopProtocolKillJobTask.class.getName(),
- new GridHadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId()));
- }
- catch (GridClientException e) {
- throw new IOException("Failed to kill job: " + jobId, e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void setJobPriority(JobID jobid, String priority) throws IOException, InterruptedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public boolean killTask(TaskAttemptID taskId, boolean shouldFail) throws IOException,
- InterruptedException {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public JobStatus getJobStatus(JobID jobId) throws IOException, InterruptedException {
- try {
- Long delay = conf.getLong(GridHadoopJobProperty.JOB_STATUS_POLL_DELAY.propertyName(), -1);
-
- GridHadoopProtocolTaskArguments args = delay >= 0 ?
- new GridHadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId(), delay) :
- new GridHadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId());
-
- GridHadoopJobStatus status = cli.compute().execute(GridHadoopProtocolJobStatusTask.class.getName(), args);
-
- if (status == null)
- throw new IOException("Job tracker doesn't have any information about the job: " + jobId);
-
- return processStatus(status);
- }
- catch (GridClientException e) {
- throw new IOException("Failed to get job status: " + jobId, e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Counters getJobCounters(JobID jobId) throws IOException, InterruptedException {
- try {
- final GridHadoopCounters counters = cli.compute().execute(GridHadoopProtocolJobCountersTask.class.getName(),
- new GridHadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId()));
-
- if (counters == null)
- throw new IOException("Job tracker doesn't have any information about the job: " + jobId);
-
- return new GridHadoopClientCounters(counters);
- }
- catch (GridClientException e) {
- throw new IOException("Failed to get job counters: " + jobId, e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public TaskReport[] getTaskReports(JobID jobid, TaskType type) throws IOException, InterruptedException {
- return new TaskReport[0];
- }
-
- /** {@inheritDoc} */
- @Override public String getFilesystemName() throws IOException, InterruptedException {
- return FileSystem.get(conf).getUri().toString();
- }
-
- /** {@inheritDoc} */
- @Override public JobStatus[] getAllJobs() throws IOException, InterruptedException {
- return new JobStatus[0];
- }
-
- /** {@inheritDoc} */
- @Override public TaskCompletionEvent[] getTaskCompletionEvents(JobID jobid, int fromEventId, int maxEvents)
- throws IOException, InterruptedException {
- return new TaskCompletionEvent[0];
- }
-
- /** {@inheritDoc} */
- @Override public String[] getTaskDiagnostics(TaskAttemptID taskId) throws IOException, InterruptedException {
- return new String[0];
- }
-
- /** {@inheritDoc} */
- @Override public TaskTrackerInfo[] getActiveTrackers() throws IOException, InterruptedException {
- return new TaskTrackerInfo[0];
- }
-
- /** {@inheritDoc} */
- @Override public TaskTrackerInfo[] getBlacklistedTrackers() throws IOException, InterruptedException {
- return new TaskTrackerInfo[0];
- }
-
- /** {@inheritDoc} */
- @Override public String getSystemDir() throws IOException, InterruptedException {
- Path sysDir = new Path(SYS_DIR);
-
- return sysDir.toString();
- }
-
- /** {@inheritDoc} */
- @Override public String getStagingAreaDir() throws IOException, InterruptedException {
- String usr = UserGroupInformation.getCurrentUser().getShortUserName();
-
- return GridHadoopUtils.stagingAreaDir(conf, usr).toString();
- }
-
- /** {@inheritDoc} */
- @Override public String getJobHistoryDir() throws IOException, InterruptedException {
- return JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
- }
-
- /** {@inheritDoc} */
- @Override public QueueInfo[] getQueues() throws IOException, InterruptedException {
- return new QueueInfo[0];
- }
-
- /** {@inheritDoc} */
- @Override public QueueInfo getQueue(String queueName) throws IOException, InterruptedException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException, InterruptedException {
- return new QueueAclsInfo[0];
- }
-
- /** {@inheritDoc} */
- @Override public QueueInfo[] getRootQueues() throws IOException, InterruptedException {
- return new QueueInfo[0];
- }
-
- /** {@inheritDoc} */
- @Override public QueueInfo[] getChildQueues(String queueName) throws IOException, InterruptedException {
- return new QueueInfo[0];
- }
-
- /** {@inheritDoc} */
- @Override public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer) throws IOException,
- InterruptedException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public long renewDelegationToken(Token<DelegationTokenIdentifier> token) throws IOException,
- InterruptedException {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public void cancelDelegationToken(Token<DelegationTokenIdentifier> token) throws IOException,
- InterruptedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public LogParams getLogFileParams(JobID jobID, TaskAttemptID taskAttemptID) throws IOException,
- InterruptedException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
- return PROTO_VER;
- }
-
- /** {@inheritDoc} */
- @Override public ProtocolSignature getProtocolSignature(String protocol, long clientVersion, int clientMethodsHash)
- throws IOException {
- return ProtocolSignature.getProtocolSignature(this, protocol, clientVersion, clientMethodsHash);
- }
-
- /**
- * Process received status update.
- *
- * @param status Ignite status.
- * @return Hadoop status.
- */
- private JobStatus processStatus(GridHadoopJobStatus status) {
- // IMPORTANT! This method will only work in single-threaded environment. It is valid at the moment because
- // GridHadoopClientProtocolProvider creates new instance of this class for every new job and Job class
- // serializes invocations of submitJob() and getJobStatus() methods. However, if any of these conditions will
- // change in future and either protocol will serve statuses for several jobs or status update will not be
- // serialized anymore, then we have to fallback to concurrent approach (e.g. using ConcurrentHashMap).
- // (vozerov)
- if (lastVer < status.version()) {
- lastVer = status.version();
-
- lastStatus = status;
- }
- else
- assert lastStatus != null;
-
- return GridHadoopUtils.status(lastStatus, conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocolProvider.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocolProvider.java b/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocolProvider.java
deleted file mode 100644
index a9a1c9d..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/GridHadoopClientProtocolProvider.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.client.hadoop;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.protocol.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.client.*;
-import org.apache.ignite.internal.client.marshaller.optimized.*;
-import org.apache.ignite.internal.util.future.*;
-import org.apache.ignite.internal.util.typedef.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-import static org.apache.ignite.internal.client.GridClientProtocol.*;
-import static org.apache.ignite.client.hadoop.GridHadoopClientProtocol.*;
-
-
-/**
- * Grid Hadoop client protocol provider.
- */
-public class GridHadoopClientProtocolProvider extends ClientProtocolProvider {
- /** Clients. */
- private static final ConcurrentHashMap<String, IgniteInternalFuture<GridClient>> cliMap = new ConcurrentHashMap<>();
-
- /** {@inheritDoc} */
- @Override public ClientProtocol create(Configuration conf) throws IOException {
- if (FRAMEWORK_NAME.equals(conf.get(MRConfig.FRAMEWORK_NAME))) {
- String addr = conf.get(MRConfig.MASTER_ADDRESS);
-
- if (F.isEmpty(addr))
- throw new IOException("Failed to create client protocol because server address is not specified (is " +
- MRConfig.MASTER_ADDRESS + " property set?).");
-
- if (F.eq(addr, "local"))
- throw new IOException("Local execution mode is not supported, please point " +
- MRConfig.MASTER_ADDRESS + " to real Ignite node.");
-
- return createProtocol(addr, conf);
- }
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public ClientProtocol create(InetSocketAddress addr, Configuration conf) throws IOException {
- if (FRAMEWORK_NAME.equals(conf.get(MRConfig.FRAMEWORK_NAME)))
- return createProtocol(addr.getHostString() + ":" + addr.getPort(), conf);
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void close(ClientProtocol cliProto) throws IOException {
- // No-op.
- }
-
- /**
- * Internal protocol creation routine.
- *
- * @param addr Address.
- * @param conf Configuration.
- * @return Client protocol.
- * @throws IOException If failed.
- */
- private static ClientProtocol createProtocol(String addr, Configuration conf) throws IOException {
- return new GridHadoopClientProtocol(conf, client(addr));
- }
-
- /**
- * Create client.
- *
- * @param addr Endpoint address.
- * @return Client.
- * @throws IOException If failed.
- */
- private static GridClient client(String addr) throws IOException {
- try {
- IgniteInternalFuture<GridClient> fut = cliMap.get(addr);
-
- if (fut == null) {
- GridFutureAdapter<GridClient> fut0 = new GridFutureAdapter<>();
-
- IgniteInternalFuture<GridClient> oldFut = cliMap.putIfAbsent(addr, fut0);
-
- if (oldFut != null)
- return oldFut.get();
- else {
- GridClientConfiguration cliCfg = new GridClientConfiguration();
-
- cliCfg.setProtocol(TCP);
- cliCfg.setServers(Collections.singletonList(addr));
- cliCfg.setMarshaller(new GridClientOptimizedMarshaller());
- cliCfg.setDaemon(true);
-
- try {
- GridClient cli = GridClientFactory.start(cliCfg);
-
- fut0.onDone(cli);
-
- return cli;
- }
- catch (GridClientException e) {
- fut0.onDone(e);
-
- throw new IOException("Failed to establish connection with Ignite node: " + addr, e);
- }
- }
- }
- else
- return fut.get();
- }
- catch (IgniteCheckedException e) {
- throw new IOException("Failed to establish connection with Ignite node: " + addr, e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/counter/GridHadoopClientCounterGroup.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/counter/GridHadoopClientCounterGroup.java b/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/counter/GridHadoopClientCounterGroup.java
deleted file mode 100644
index 37cd28d..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/counter/GridHadoopClientCounterGroup.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.client.hadoop.counter;
-
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.counters.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Hadoop Client API Counters adapter.
- */
-class GridHadoopClientCounterGroup implements CounterGroup {
- /** Counters. */
- private final GridHadoopClientCounters cntrs;
-
- /** Group name. */
- private final String name;
-
- /**
- * Creates new instance.
- *
- * @param cntrs Client counters instance.
- * @param name Group name.
- */
- GridHadoopClientCounterGroup(GridHadoopClientCounters cntrs, String name) {
- this.cntrs = cntrs;
- this.name = name;
- }
-
- /** {@inheritDoc} */
- @Override public String getName() {
- return name;
- }
-
- /** {@inheritDoc} */
- @Override public String getDisplayName() {
- return name;
- }
-
- /** {@inheritDoc} */
- @Override public void setDisplayName(String displayName) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void addCounter(Counter counter) {
- addCounter(counter.getName(), counter.getDisplayName(), 0);
- }
-
- /** {@inheritDoc} */
- @Override public Counter addCounter(String name, String displayName, long value) {
- final Counter counter = cntrs.findCounter(this.name, name);
-
- counter.setValue(value);
-
- return counter;
- }
-
- /** {@inheritDoc} */
- @Override public Counter findCounter(String counterName, String displayName) {
- return cntrs.findCounter(name, counterName);
- }
-
- /** {@inheritDoc} */
- @Override public Counter findCounter(String counterName, boolean create) {
- return cntrs.findCounter(name, counterName, create);
- }
-
- /** {@inheritDoc} */
- @Override public Counter findCounter(String counterName) {
- return cntrs.findCounter(name, counterName);
- }
-
- /** {@inheritDoc} */
- @Override public int size() {
- return cntrs.groupSize(name);
- }
-
- /** {@inheritDoc} */
- @Override public void incrAllCounters(CounterGroupBase<Counter> rightGroup) {
- for (final Counter counter : rightGroup)
- cntrs.findCounter(name, counter.getName()).increment(counter.getValue());
- }
-
- /** {@inheritDoc} */
- @Override public CounterGroupBase<Counter> getUnderlyingGroup() {
- return this;
- }
-
- /** {@inheritDoc} */
- @Override public Iterator<Counter> iterator() {
- return cntrs.iterateGroup(name);
- }
-
- /** {@inheritDoc} */
- @Override public void write(DataOutput out) throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-
- /** {@inheritDoc} */
- @Override public void readFields(DataInput in) throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/counter/GridHadoopClientCounters.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/counter/GridHadoopClientCounters.java b/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/counter/GridHadoopClientCounters.java
deleted file mode 100644
index 9f4ec02..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/counter/GridHadoopClientCounters.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.client.hadoop.counter;
-
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.counters.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.counter.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-import org.apache.ignite.internal.util.typedef.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Hadoop Client API Counters adapter.
- */
-public class GridHadoopClientCounters extends Counters {
- /** */
- private final Map<T2<String,String>,GridHadoopLongCounter> cntrs = new HashMap<>();
-
- /**
- * Creates new instance based on given counters.
- *
- * @param cntrs Counters to adapt.
- */
- public GridHadoopClientCounters(GridHadoopCounters cntrs) {
- for (GridHadoopCounter cntr : cntrs.all())
- if (cntr instanceof GridHadoopLongCounter)
- this.cntrs.put(new T2<>(cntr.group(), cntr.name()), (GridHadoopLongCounter) cntr);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized CounterGroup addGroup(CounterGroup grp) {
- return addGroup(grp.getName(), grp.getDisplayName());
- }
-
- /** {@inheritDoc} */
- @Override public CounterGroup addGroup(String name, String displayName) {
- return new GridHadoopClientCounterGroup(this, name);
- }
-
- /** {@inheritDoc} */
- @Override public Counter findCounter(String grpName, String cntrName) {
- return findCounter(grpName, cntrName, true);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized Counter findCounter(Enum<?> key) {
- return findCounter(key.getDeclaringClass().getName(), key.name(), true);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized Counter findCounter(String scheme, FileSystemCounter key) {
- return findCounter(String.format("FileSystem Counter (%s)", scheme), key.name());
- }
-
- /** {@inheritDoc} */
- @Override public synchronized Iterable<String> getGroupNames() {
- Collection<String> res = new HashSet<>();
-
- for (GridHadoopCounter counter : cntrs.values())
- res.add(counter.group());
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public Iterator<CounterGroup> iterator() {
- final Iterator<String> iter = getGroupNames().iterator();
-
- return new Iterator<CounterGroup>() {
- @Override public boolean hasNext() {
- return iter.hasNext();
- }
-
- @Override public CounterGroup next() {
- if (!hasNext())
- throw new NoSuchElementException();
-
- return new GridHadoopClientCounterGroup(GridHadoopClientCounters.this, iter.next());
- }
-
- @Override public void remove() {
- throw new UnsupportedOperationException("not implemented");
- }
- };
- }
-
- /** {@inheritDoc} */
- @Override public synchronized CounterGroup getGroup(String grpName) {
- return new GridHadoopClientCounterGroup(this, grpName);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized int countCounters() {
- return cntrs.size();
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void write(DataOutput out) throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void readFields(DataInput in) throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void incrAllCounters(AbstractCounters<Counter, CounterGroup> other) {
- for (CounterGroup group : other) {
- for (Counter counter : group) {
- findCounter(group.getName(), counter.getName()).increment(counter.getValue());
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object genericRight) {
- if (!(genericRight instanceof GridHadoopClientCounters))
- return false;
-
- return cntrs.equals(((GridHadoopClientCounters) genericRight).cntrs);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return cntrs.hashCode();
- }
-
- /** {@inheritDoc} */
- @Override public void setWriteAllCounters(boolean snd) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public boolean getWriteAllCounters() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public Limits limits() {
- return null;
- }
-
- /**
- * Returns size of a group.
- *
- * @param grpName Name of the group.
- * @return amount of counters in the given group.
- */
- public int groupSize(String grpName) {
- int res = 0;
-
- for (GridHadoopCounter counter : cntrs.values()) {
- if (grpName.equals(counter.group()))
- res++;
- }
-
- return res;
- }
-
- /**
- * Returns counters iterator for specified group.
- *
- * @param grpName Name of the group to iterate.
- * @return Counters iterator.
- */
- public Iterator<Counter> iterateGroup(String grpName) {
- Collection<Counter> grpCounters = new ArrayList<>();
-
- for (GridHadoopLongCounter counter : cntrs.values()) {
- if (grpName.equals(counter.group()))
- grpCounters.add(new GridHadoopV2Counter(counter));
- }
-
- return grpCounters.iterator();
- }
-
- /**
- * Find a counter in the group.
- *
- * @param grpName The name of the counter group.
- * @param cntrName The name of the counter.
- * @param create Create the counter if not found if true.
- * @return The counter that was found or added or {@code null} if create is false.
- */
- public Counter findCounter(String grpName, String cntrName, boolean create) {
- T2<String, String> key = new T2<>(grpName, cntrName);
-
- GridHadoopLongCounter internalCntr = cntrs.get(key);
-
- if (internalCntr == null & create) {
- internalCntr = new GridHadoopLongCounter(grpName,cntrName);
-
- cntrs.put(key, new GridHadoopLongCounter(grpName,cntrName));
- }
-
- return internalCntr == null ? null : new GridHadoopV2Counter(internalCntr);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/package.html
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/package.html b/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/package.html
deleted file mode 100644
index f182598..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/client/hadoop/package.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<body>
- <!-- Package description. -->
- Ignite Hadoop client protocol.
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java
new file mode 100644
index 0000000..66e9761
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.fs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.counter.*;
+import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
+import org.apache.ignite.internal.util.typedef.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Statistic writer implementation that writes info into any Hadoop file system.
+ */
+public class IgniteHadoopFileSystemCounterWriter implements HadoopCounterWriter {
+ /** */
+ public static final String PERFORMANCE_COUNTER_FILE_NAME = "performance";
+
+ /** */
+ private static final String DEFAULT_USER_NAME = "anonymous";
+
+ /** */
+ public static final String COUNTER_WRITER_DIR_PROPERTY = "ignite.counters.fswriter.directory";
+
+ /** */
+ private static final String USER_MACRO = "${USER}";
+
+ /** */
+ private static final String DEFAULT_COUNTER_WRITER_DIR = "/user/" + USER_MACRO;
+
+ /** {@inheritDoc} */
+ @Override public void write(HadoopJobInfo jobInfo, HadoopJobId jobId, HadoopCounters cntrs)
+ throws IgniteCheckedException {
+
+ Configuration hadoopCfg = new Configuration();
+
+ for (Map.Entry<String, String> e : ((HadoopDefaultJobInfo)jobInfo).properties().entrySet())
+ hadoopCfg.set(e.getKey(), e.getValue());
+
+ String user = jobInfo.user();
+
+ if (F.isEmpty(user))
+ user = DEFAULT_USER_NAME;
+
+ String dir = jobInfo.property(COUNTER_WRITER_DIR_PROPERTY);
+
+ if (dir == null)
+ dir = DEFAULT_COUNTER_WRITER_DIR;
+
+ Path jobStatPath = new Path(new Path(dir.replace(USER_MACRO, user)), jobId.toString());
+
+ HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(cntrs, null);
+
+ try {
+ FileSystem fs = jobStatPath.getFileSystem(hadoopCfg);
+
+ fs.mkdirs(jobStatPath);
+
+ try (PrintStream out = new PrintStream(fs.create(new Path(jobStatPath, PERFORMANCE_COUNTER_FILE_NAME)))) {
+ for (T2<String, Long> evt : perfCntr.evts()) {
+ out.print(evt.get1());
+ out.print(':');
+ out.println(evt.get2().toString());
+ }
+
+ out.flush();
+ }
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
new file mode 100644
index 0000000..98f2e46
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.fs;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.*;
+import org.apache.hadoop.ipc.*;
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
+
+/**
+ * Adapter to use any Hadoop file system {@link FileSystem} as {@link IgfsSecondaryFileSystem}.
+ */
+public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, AutoCloseable {
+
+ /** Hadoop file system. */
+ private final FileSystem fileSys;
+
+ /** Properties of file system */
+ private final Map<String, String> props = new HashMap<>();
+
+ /**
+ * Simple constructor that is to be used by default.
+ *
+ * @param uri URI of file system.
+ * @throws IgniteCheckedException In case of error.
+ */
+ public IgniteHadoopIgfsSecondaryFileSystem(String uri) throws IgniteCheckedException {
+ this(uri, null);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param uri URI of file system.
+ * @param cfgPath Additional path to Hadoop configuration.
+ * @throws IgniteCheckedException In case of error.
+ */
+ public IgniteHadoopIgfsSecondaryFileSystem(@Nullable String uri, @Nullable String cfgPath) throws IgniteCheckedException {
+ try {
+ SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(uri, cfgPath);
+
+ fileSys = secProvider.createFileSystem();
+
+ uri = secProvider.uri().toString();
+
+ if (!uri.endsWith("/"))
+ uri += "/";
+
+ props.put(SECONDARY_FS_CONFIG_PATH, cfgPath);
+ props.put(SECONDARY_FS_URI, uri);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /**
+ * Convert IGFS path into Hadoop path.
+ *
+ * @param path IGFS path.
+ * @return Hadoop path.
+ */
+ private Path convert(IgfsPath path) {
+ URI uri = fileSys.getUri();
+
+ return new Path(uri.getScheme(), uri.getAuthority(), path.toString());
+ }
+
+ /**
+ * Heuristically checks if exception was caused by invalid HDFS version and returns appropriate exception.
+ *
+ * @param e Exception to check.
+ * @param detailMsg Detailed error message.
+ * @return Appropriate exception.
+ */
+ @SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "unchecked"})
+ private IgfsException handleSecondaryFsError(IOException e, String detailMsg) {
+ boolean wrongVer = X.hasCause(e, RemoteException.class) ||
+ (e.getMessage() != null && e.getMessage().contains("Failed on local"));
+
+ return !wrongVer ? cast(detailMsg, e) :
+ new IgfsInvalidHdfsVersionException("HDFS version you are connecting to differs from local " +
+ "version.", e); }
+
+ /**
+ * Cast IO exception to IGFS exception.
+ *
+ * @param e IO exception.
+ * @return IGFS exception.
+ */
+ public static IgfsException cast(String msg, IOException e) {
+ if (e instanceof FileNotFoundException)
+ return new IgfsFileNotFoundException(e);
+ else if (e instanceof ParentNotDirectoryException)
+ return new IgfsParentNotDirectoryException(msg, e);
+ else if (e instanceof PathIsNotEmptyDirectoryException)
+ return new IgfsDirectoryNotEmptyException(e);
+ else if (e instanceof PathExistsException)
+ return new IgfsPathAlreadyExistsException(msg, e);
+ else
+ return new IgfsException(msg, e);
+ }
+
+ /**
+ * Convert Hadoop FileStatus properties to map.
+ *
+ * @param status File status.
+ * @return IGFS attributes.
+ */
+ private static Map<String, String> properties(FileStatus status) {
+ FsPermission perm = status.getPermission();
+
+ if (perm == null)
+ perm = FsPermission.getDefault();
+
+ return F.asMap(PROP_PERMISSION, String.format("%04o", perm.toShort()), PROP_USER_NAME, status.getOwner(),
+ PROP_GROUP_NAME, status.getGroup());
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean exists(IgfsPath path) {
+ try {
+ return fileSys.exists(convert(path));
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to check file existence [path=" + path + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public IgfsFile update(IgfsPath path, Map<String, String> props) {
+ HadoopIgfsProperties props0 = new HadoopIgfsProperties(props);
+
+ try {
+ if (props0.userName() != null || props0.groupName() != null)
+ fileSys.setOwner(convert(path), props0.userName(), props0.groupName());
+
+ if (props0.permission() != null)
+ fileSys.setPermission(convert(path), props0.permission());
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to update file properties [path=" + path + "]");
+ }
+
+ //Result is not used in case of secondary FS.
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void rename(IgfsPath src, IgfsPath dest) {
+ // Delegate to the secondary file system.
+ try {
+ if (!fileSys.rename(convert(src), convert(dest)))
+ throw new IgfsException("Failed to rename (secondary file system returned false) " +
+ "[src=" + src + ", dest=" + dest + ']');
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to rename file [src=" + src + ", dest=" + dest + ']');
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean delete(IgfsPath path, boolean recursive) {
+ try {
+ return fileSys.delete(convert(path), recursive);
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to delete file [path=" + path + ", recursive=" + recursive + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdirs(IgfsPath path) {
+ try {
+ if (!fileSys.mkdirs(convert(path)))
+ throw new IgniteException("Failed to make directories [path=" + path + "]");
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to make directories [path=" + path + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) {
+ try {
+ if (!fileSys.mkdirs(convert(path), new HadoopIgfsProperties(props).permission()))
+ throw new IgniteException("Failed to make directories [path=" + path + ", props=" + props + "]");
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to make directories [path=" + path + ", props=" + props + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsPath> listPaths(IgfsPath path) {
+ try {
+ FileStatus[] statuses = fileSys.listStatus(convert(path));
+
+ if (statuses == null)
+ throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
+
+ Collection<IgfsPath> res = new ArrayList<>(statuses.length);
+
+ for (FileStatus status : statuses)
+ res.add(new IgfsPath(path, status.getPath().getName()));
+
+ return res;
+ }
+ catch (FileNotFoundException ignored) {
+ throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsFile> listFiles(IgfsPath path) {
+ try {
+ FileStatus[] statuses = fileSys.listStatus(convert(path));
+
+ if (statuses == null)
+ throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
+
+ Collection<IgfsFile> res = new ArrayList<>(statuses.length);
+
+ for (FileStatus status : statuses) {
+ IgfsFileInfo fsInfo = status.isDirectory() ? new IgfsFileInfo(true, properties(status)) :
+ new IgfsFileInfo((int)status.getBlockSize(), status.getLen(), null, null, false,
+ properties(status));
+
+ res.add(new IgfsFileImpl(new IgfsPath(path, status.getPath().getName()), fsInfo, 1));
+ }
+
+ return res;
+ }
+ catch (FileNotFoundException ignored) {
+ throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsSecondaryFileSystemPositionedReadable open(IgfsPath path, int bufSize) {
+ return new HadoopIgfsSecondaryFileSystemPositionedReadable(fileSys, convert(path), bufSize);
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputStream create(IgfsPath path, boolean overwrite) {
+ try {
+ return fileSys.create(convert(path), overwrite);
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to create file [path=" + path + ", overwrite=" + overwrite + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
+ long blockSize, @Nullable Map<String, String> props) {
+ HadoopIgfsProperties props0 =
+ new HadoopIgfsProperties(props != null ? props : Collections.<String, String>emptyMap());
+
+ try {
+ return fileSys.create(convert(path), props0.permission(), overwrite, bufSize, (short)replication, blockSize,
+ null);
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to create file [path=" + path + ", props=" + props +
+ ", overwrite=" + overwrite + ", bufSize=" + bufSize + ", replication=" + replication +
+ ", blockSize=" + blockSize + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputStream append(IgfsPath path, int bufSize, boolean create,
+ @Nullable Map<String, String> props) {
+ try {
+ return fileSys.append(convert(path), bufSize);
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to append file [path=" + path + ", bufSize=" + bufSize + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsFile info(final IgfsPath path) {
+ try {
+ final FileStatus status = fileSys.getFileStatus(convert(path));
+
+ if (status == null)
+ return null;
+
+ final Map<String, String> props = properties(status);
+
+ return new IgfsFile() {
+ @Override public IgfsPath path() {
+ return path;
+ }
+
+ @Override public boolean isFile() {
+ return status.isFile();
+ }
+
+ @Override public boolean isDirectory() {
+ return status.isDirectory();
+ }
+
+ @Override public int blockSize() {
+ return (int)status.getBlockSize();
+ }
+
+ @Override public long groupBlockSize() {
+ return status.getBlockSize();
+ }
+
+ @Override public long accessTime() {
+ return status.getAccessTime();
+ }
+
+ @Override public long modificationTime() {
+ return status.getModificationTime();
+ }
+
+ @Override public String property(String name) throws IllegalArgumentException {
+ String val = props.get(name);
+
+ if (val == null)
+ throw new IllegalArgumentException("File property not found [path=" + path + ", name=" + name + ']');
+
+ return val;
+ }
+
+ @Nullable @Override public String property(String name, @Nullable String dfltVal) {
+ String val = props.get(name);
+
+ return val == null ? dfltVal : val;
+ }
+
+ @Override public long length() {
+ return status.getLen();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, String> properties() {
+ return props;
+ }
+ };
+
+ }
+ catch (FileNotFoundException ignore) {
+ return null;
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public long usedSpaceSize() {
+ try {
+ return fileSys.getContentSummary(new Path(fileSys.getUri())).getSpaceConsumed();
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to get used space size of file system.");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Map<String, String> properties() {
+ return props;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ try {
+ fileSys.close();
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/498dcfab/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/package.html
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/package.html b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/package.html
new file mode 100644
index 0000000..1d78952
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/package.html
@@ -0,0 +1,24 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<body>
+ <!-- Package description. -->
+ Ignite Hadoop Accelerator file system API.
+</body>
+</html>