You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2016/06/06 16:03:05 UTC
[01/17] ignite git commit: IGNITE-3228: Hadoop: workaround/fix for
inefficient memory usage.
Repository: ignite
Updated Branches:
refs/heads/ignite-3216 [created] efdd7b346
IGNITE-3228: Hadoop: workaround/fix for inefficient memory usage.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2490b0ae
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2490b0ae
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2490b0ae
Branch: refs/heads/ignite-3216
Commit: 2490b0aef17595247f8a6ae29482bc900e2a9a8d
Parents: 52a2637
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Jun 2 09:11:09 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Jun 2 09:11:09 2016 +0300
----------------------------------------------------------------------
.../shuffle/collections/HadoopMultimapBase.java | 90 +++++++++++++++++---
1 file changed, 76 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2490b0ae/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
index e6995ca..7dcff3d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
@@ -30,7 +30,6 @@ import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
import org.apache.ignite.internal.processors.hadoop.shuffle.streams.HadoopDataInStream;
import org.apache.ignite.internal.processors.hadoop.shuffle.streams.HadoopDataOutStream;
import org.apache.ignite.internal.processors.hadoop.shuffle.streams.HadoopOffheapBuffer;
-import org.apache.ignite.internal.util.GridLongList;
import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory;
import org.jetbrains.annotations.Nullable;
@@ -48,7 +47,7 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
protected final int pageSize;
/** */
- private final Collection<GridLongList> allPages = new ConcurrentLinkedQueue<>();
+ private final Collection<Page> allPages = new ConcurrentLinkedQueue<>();
/**
* @param jobInfo Job info.
@@ -64,11 +63,12 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
}
/**
- * @param ptrs Page pointers.
+ * @param page Page.
*/
- private void deallocate(GridLongList ptrs) {
- while (!ptrs.isEmpty())
- mem.release(ptrs.remove(), ptrs.remove());
+ private void deallocate(Page page) {
+ assert page != null;
+
+ mem.release(page.ptr, page.size);
}
/**
@@ -105,8 +105,8 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
/** {@inheritDoc} */
@Override public void close() {
- for (GridLongList list : allPages)
- deallocate(list);
+ for (Page page : allPages)
+ deallocate(page);
}
/**
@@ -190,8 +190,8 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
/** */
private long writeStart;
- /** Size and pointer pairs list. */
- private final GridLongList pages = new GridLongList(16);
+ /** Current page. */
+ private Page curPage;
/**
* @param ctx Task context.
@@ -222,11 +222,10 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
private long allocateNextPage(long requestedSize) {
int writtenSize = writtenSize();
- long newPageSize = Math.max(writtenSize + requestedSize, pageSize);
+ long newPageSize = nextPageSize(writtenSize + requestedSize);
long newPagePtr = mem.allocate(newPageSize);
- pages.add(newPageSize);
- pages.add(newPagePtr);
+ System.out.println("ALLOCATED: " + newPageSize);
HadoopOffheapBuffer b = out.buffer();
@@ -240,10 +239,50 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
writeStart = newPagePtr;
+ // At this point old page is not needed, so we release it.
+ Page oldPage = curPage;
+
+ curPage = new Page(newPagePtr, newPageSize);
+
+ if (oldPage != null)
+ allPages.add(oldPage);
+
return b.move(requestedSize);
}
/**
+ * Get next page size.
+ *
+ * @param required Required amount of data.
+ * @return Next page size.
+ */
+ private long nextPageSize(long required) {
+ long pages = (required / pageSize) + 1;
+
+ long pagesPow2 = nextPowerOfTwo(pages);
+
+ return pagesPow2 * pageSize;
+ }
+
+ /**
+ * Get next power of two which greater or equal to the given number. Naive implementation.
+ *
+ * @param val Number
+ * @return Nearest pow2.
+ */
+ private long nextPowerOfTwo(long val) {
+ long res = 1;
+
+ while (res < val)
+ res = res << 1;
+
+ if (res < 0)
+ throw new IllegalArgumentException("Value is too big to find positive pow2: " + val);
+
+ return res;
+ }
+
+ /**
* @return Fixed pointer.
*/
private long fixAlignment() {
@@ -317,7 +356,8 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
/** {@inheritDoc} */
@Override public void close() throws IgniteCheckedException {
- allPages.add(pages);
+ if (curPage != null)
+ allPages.add(curPage);
keySer.close();
valSer.close();
@@ -372,4 +412,26 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
throw new UnsupportedOperationException();
}
}
+
+ /**
+ * Page.
+ */
+ private static class Page {
+ /** Pointer. */
+ private final long ptr;
+
+ /** Size. */
+ private final long size;
+
+ /**
+ * Constructor.
+ *
+ * @param ptr Pointer.
+ * @param size Size.
+ */
+ public Page(long ptr, long size) {
+ this.ptr = ptr;
+ this.size = size;
+ }
+ }
}
\ No newline at end of file
[17/17] ignite git commit: IGNITE-3216
Posted by av...@apache.org.
IGNITE-3216
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/efdd7b34
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/efdd7b34
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/efdd7b34
Branch: refs/heads/ignite-3216
Commit: efdd7b346349e71083edcfc547d4fd5a5aef5d9b
Parents: 5254957
Author: Anton Vinogradov <av...@apache.org>
Authored: Mon Jun 6 19:02:43 2016 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Mon Jun 6 19:02:43 2016 +0300
----------------------------------------------------------------------
.../org/apache/ignite/internal/util/IgniteUtils.java | 8 ++++----
.../apache/ignite/internal/util/lang/GridFunc.java | 14 +++++---------
2 files changed, 9 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/efdd7b34/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index c898909..012d001 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -8550,7 +8550,7 @@ public abstract class IgniteUtils {
*/
public static Collection<InetAddress> toInetAddresses(Collection<String> addrs,
Collection<String> hostNames) throws IgniteCheckedException {
- List<InetAddress> res = new ArrayList<>(addrs.size());
+ Set<InetAddress> res = new HashSet<>(addrs.size());
Iterator<String> hostNamesIt = hostNames.iterator();
@@ -8583,7 +8583,7 @@ public abstract class IgniteUtils {
throw new IgniteCheckedException("Addresses can not be resolved [addr=" + addrs +
", hostNames=" + hostNames + ']');
- return F.viewListReadOnly(res, F.<InetAddress>identity());
+ return F.viewSetReadOnly(res, F.<InetAddress>identity());
}
/**
@@ -8609,7 +8609,7 @@ public abstract class IgniteUtils {
*/
public static Collection<InetSocketAddress> toSocketAddresses(Collection<String> addrs,
Collection<String> hostNames, int port) {
- List<InetSocketAddress> res = new ArrayList<>(addrs.size());
+ Set<InetSocketAddress> res = new HashSet<>(addrs.size());
Iterator<String> hostNamesIt = hostNames.iterator();
@@ -8630,7 +8630,7 @@ public abstract class IgniteUtils {
res.add(new InetSocketAddress(addr, port));
}
- return F.viewListReadOnly(res, F.<InetSocketAddress>identity());
+ return F.viewSetReadOnly(res, F.<InetSocketAddress>identity());
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/efdd7b34/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
index 8eeca6b..39052dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
@@ -1695,33 +1695,29 @@ public class GridFunc {
}
/**
- * Creates read-only light-weight view on given list with provided transformation.
+ * Creates read-only light-weight view on given set with provided transformation.
* Resulting list will only "have" {@code transformed} elements. Note that only wrapping
* list will be created and no duplication of data will occur.
*
* @param c Input list that serves as a base for the view.
* @param trans Transformation closure.
* @param <T1> Type of the list.
- * @return Light-weight view on given list with provided transformation.
+ * @return Light-weight view on given set with provided transformation.
*/
@SuppressWarnings("RedundantTypeArguments")
- public static <T1, T2> List<T2> viewListReadOnly(@Nullable final List<? extends T1> c,
+ public static <T1, T2> Set<T2> viewSetReadOnly(@Nullable final Set<? extends T1> c,
final IgniteClosure<? super T1, T2> trans) {
A.notNull(trans, "trans");
if (isEmpty(c))
- return Collections.emptyList();
+ return Collections.emptySet();
assert c != null;
- return new GridSerializableList<T2>() {
+ return new GridSerializableSet<T2>() {
/** */
private static final long serialVersionUID = 3126625219739967068L;
- @Override public T2 get(int idx) {
- return trans.apply(c.get(idx));
- }
-
@NotNull
@Override public Iterator<T2> iterator() {
return F.<T1, T2>iterator(c, trans, true);
[14/17] ignite git commit: Merge remote-tracking branch
'upstream/gridgain-7.5.25' into gridgain-7.5.25
Posted by av...@apache.org.
Merge remote-tracking branch 'upstream/gridgain-7.5.25' into gridgain-7.5.25
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2c0d95f2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2c0d95f2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2c0d95f2
Branch: refs/heads/ignite-3216
Commit: 2c0d95f23abf710af8b277f8d3df06e6d35441fa
Parents: 1308b12 6cb9947
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Mon Jun 6 15:41:42 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Mon Jun 6 15:41:42 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/binary/BinaryUtils.java | 16 ++
.../processors/cache/CacheObjectContext.java | 3 +
.../GridCacheBinaryObjectsAbstractSelfTest.java | 78 +++++++-
.../unsafe/GridOffheapSnapTreeSelfTest.java | 2 +-
.../query/h2/opt/GridH2AbstractKeyValueRow.java | 23 +--
.../query/h2/opt/GridH2KeyValueRowOffheap.java | 17 +-
.../cache/IgniteCacheOffheapIndexScanTest.java | 195 +++++++++++++++++++
.../IgniteCacheQuerySelfTestSuite.java | 2 +
8 files changed, 321 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
[05/17] ignite git commit: IGNITE-3175 BigDecimal fields are not
supported if query is executed from IgniteRDD: fix java.sql.Date mapping
Posted by av...@apache.org.
IGNITE-3175 BigDecimal fields are not supported if query is executed from IgniteRDD: fix java.sql.Date mapping
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4506c215
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4506c215
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4506c215
Branch: refs/heads/ignite-3216
Commit: 4506c215b43ffb5e6319c318196e713c8c5aefbd
Parents: c10097e
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Thu Jun 2 16:56:20 2016 +0300
Committer: Valentin Kulichenko <va...@gmail.com>
Committed: Fri Jun 3 19:49:59 2016 +0300
----------------------------------------------------------------------
.../src/main/scala/org/apache/ignite/spark/IgniteRDD.scala | 3 ++-
.../ignite/spark/JavaStandaloneIgniteRDDSelfTest.java | 9 ++++++++-
.../org/apache/ignite/spark/EntityTestAllTypeFields.scala | 7 +++++--
3 files changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4506c215/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
index 9ef9846..fa96212 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
@@ -315,9 +315,10 @@ class IgniteRDD[K, V] (
case "java.lang.Long" \u21d2 LongType
case "java.lang.Float" \u21d2 FloatType
case "java.lang.Double" \u21d2 DoubleType
- case "java.math.BigDecimal" \u21d2 DecimalType.SYSTEM_DEFAULT
+ case "java.math.BigDecimal" \u21d2 DataTypes.createDecimalType()
case "java.lang.String" \u21d2 StringType
case "java.util.Date" \u21d2 DateType
+ case "java.sql.Date" \u21d2 DateType
case "java.sql.Timestamp" \u21d2 TimestampType
case "[B" \u21d2 BinaryType
http://git-wip-us.apache.org/repos/asf/ignite/blob/4506c215/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java b/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java
index e600c6c..343158b 100644
--- a/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java
+++ b/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java
@@ -39,6 +39,7 @@ import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import scala.Tuple2;
+import java.io.Serializable;
import java.lang.reflect.Field;
import java.math.BigDecimal;
import java.util.List;
@@ -279,10 +280,16 @@ public class JavaStandaloneIgniteRDDSelfTest extends GridCommonAbstractTest {
assertTrue(String.format("+++ Fail on %s field", fieldName),
((Comparable<BigDecimal>)val).compareTo((BigDecimal)res) == 0);
}
+ else if (val instanceof java.sql.Date)
+ assertEquals(String.format("+++ Fail on %s field", fieldName),
+ val.toString(), df.collect()[0].get(0).toString());
else if (val.getClass().isArray())
assertTrue(String.format("+++ Fail on %s field", fieldName), 1 <= df.count());
- else
+ else {
+ assertTrue(String.format("+++ Fail on %s field", fieldName), df.collect().length > 0);
+ assertTrue(String.format("+++ Fail on %s field", fieldName), df.collect()[0].size() > 0);
assertEquals(String.format("+++ Fail on %s field", fieldName), val, df.collect()[0].get(0));
+ }
info(String.format("+++ Query on the filed: %s : %s passed", fieldName, f.getType().getSimpleName()));
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4506c215/modules/spark/src/test/scala/org/apache/ignite/spark/EntityTestAllTypeFields.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/EntityTestAllTypeFields.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/EntityTestAllTypeFields.scala
index a936091..36d8274 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/EntityTestAllTypeFields.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/EntityTestAllTypeFields.scala
@@ -35,7 +35,9 @@ class EntityTestAllTypeFields(
@ScalarCacheQuerySqlField(index = true) val dateVal: Date,
@ScalarCacheQuerySqlField(index = true) val timestampVal: Timestamp,
@ScalarCacheQuerySqlField(index = true) val byteArrVal: Array[Byte],
- @ScalarCacheQuerySqlField(index = true) val bigDecVal: java.math.BigDecimal
+ @ScalarCacheQuerySqlField(index = true) val bigDecVal: java.math.BigDecimal,
+ @ScalarCacheQuerySqlField(index = true) val javaSqlDate: java.sql.Date
+
) extends Serializable {
def this(
i: Int
@@ -52,6 +54,7 @@ class EntityTestAllTypeFields(
new Date(i),
new Timestamp(i),
Array(i.toByte, i.toByte),
- new java.math.BigDecimal(i.toString))
+ new java.math.BigDecimal(i.toString),
+ new java.sql.Date(i))
}
}
[13/17] ignite git commit: IGNITE-3256: IGFS: Removed "exists" check
during input stream close.
Posted by av...@apache.org.
IGNITE-3256: IGFS: Removed "exists" check during input stream close.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1308b122
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1308b122
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1308b122
Branch: refs/heads/ignite-3216
Commit: 1308b122f84af3c98656ac11d8bed606c6b4ecb5
Parents: e409b67
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Mon Jun 6 15:40:59 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Mon Jun 6 15:40:59 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/processors/igfs/IgfsInputStreamImpl.java | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1308b122/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
index de7071a..ca2f9f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
@@ -317,13 +317,9 @@ public class IgfsInputStreamImpl extends IgfsInputStreamAdapter {
pendingFutsLock.unlock();
}
}
-
- // Safety to ensure no orphaned data blocks exist in case file was concurrently deleted.
- if (!meta.exists(fileInfo.id()))
- data.delete(fileInfo);
}
}
- catch (IgniteCheckedException e) {
+ catch (Exception e) {
throw new IOException("File to close the file: " + path, e);
}
finally {
[06/17] ignite git commit: IGFS-3246: IGFS: Infrastructure for client
optimizations.
Posted by av...@apache.org.
IGFS-3246: IGFS: Infrastructure for client optimizations.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/dcd722a3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/dcd722a3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/dcd722a3
Branch: refs/heads/ignite-3216
Commit: dcd722a3dc55d656e3517764c3b162d63f0d1ba0
Parents: 4506c21
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Sun Jun 5 20:57:15 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Sun Jun 5 20:57:15 2016 +0300
----------------------------------------------------------------------
.../internal/cluster/ClusterGroupAdapter.java | 6 +
.../ignite/internal/cluster/ClusterGroupEx.java | 14 ++-
.../internal/processors/igfs/IgfsContext.java | 14 +--
.../internal/processors/igfs/IgfsImpl.java | 2 -
.../processors/igfs/IgfsMetaManager.java | 62 +++++++++-
.../processors/igfs/IgfsNodePredicate.java | 80 +++++++++++++
.../internal/processors/igfs/IgfsProcessor.java | 37 ++++--
.../internal/processors/igfs/IgfsUtils.java | 21 ++++
.../igfs/client/IgfsClientAbstractCallable.java | 112 +++++++++++++++++++
.../multijvm/IgniteClusterProcessProxy.java | 5 +
10 files changed, 325 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/dcd722a3/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupAdapter.java
index 75168a1..c664f1e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupAdapter.java
@@ -51,6 +51,7 @@ import org.apache.ignite.internal.IgniteServicesImpl;
import org.apache.ignite.internal.IgnitionEx;
import org.apache.ignite.internal.executor.GridExecutorService;
import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.processors.igfs.IgfsNodePredicate;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -602,6 +603,11 @@ public class ClusterGroupAdapter implements ClusterGroupEx, Externalizable {
}
/** {@inheritDoc} */
+ @Override public ClusterGroup forIgfsMetadataDataNodes(@Nullable String igfsName, @Nullable String metaCacheName) {
+ return forPredicate(new IgfsNodePredicate(igfsName)).forDataNodes(metaCacheName);
+ }
+
+ /** {@inheritDoc} */
@Override public final ClusterGroup forHost(ClusterNode node) {
A.notNull(node, "node");
http://git-wip-us.apache.org/repos/asf/ignite/blob/dcd722a3/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupEx.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupEx.java
index 59da7cf..21533a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupEx.java
@@ -29,7 +29,7 @@ public interface ClusterGroupEx extends ClusterGroup {
* Creates projection for specified subject ID.
*
* @param subjId Subject ID.
- * @return Internal projection.
+ * @return Cluster group.
*/
public ClusterGroupEx forSubjectId(UUID subjId);
@@ -40,5 +40,15 @@ public interface ClusterGroupEx extends ClusterGroup {
* @param clientNodes Flag to include client nodes.
* @return Cluster group.
*/
- public ClusterGroup forCacheNodes(@Nullable String cacheName, boolean affNodes, boolean nearNodes, boolean clientNodes);
+ public ClusterGroup forCacheNodes(@Nullable String cacheName, boolean affNodes, boolean nearNodes,
+ boolean clientNodes);
+
+ /**
+ * Create projection for IGFS server nodes.
+ *
+ * @param igfsName IGFS name.
+ * @param metaCacheName Metadata cache name.
+ * @return Cluster group.
+ */
+ public ClusterGroup forIgfsMetadataDataNodes(@Nullable String igfsName, @Nullable String metaCacheName);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/dcd722a3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
index 2b08f28..a638bf3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
@@ -24,11 +24,8 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.FileSystemConfiguration;
import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.util.typedef.F;
import org.jetbrains.annotations.Nullable;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGFS;
-
/**
* IGFS context holding all required components for IGFS instance.
*/
@@ -178,16 +175,7 @@ public class IgfsContext {
* @return {@code True} if node has IGFS with this name, {@code false} otherwise.
*/
public boolean igfsNode(ClusterNode node) {
- assert node != null;
-
- IgfsAttributes[] igfs = node.attribute(ATTR_IGFS);
-
- if (igfs != null)
- for (IgfsAttributes attrs : igfs)
- if (F.eq(cfg.getName(), attrs.igfsName()))
- return true;
-
- return false;
+ return IgfsUtils.isIgfsNode(node, cfg.getName());
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/dcd722a3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 967d962..2796839 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -96,13 +96,11 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import static org.apache.ignite.events.EventType.EVT_IGFS_DIR_DELETED;
-import static org.apache.ignite.events.EventType.EVT_IGFS_DIR_RENAMED;
import static org.apache.ignite.events.EventType.EVT_IGFS_FILE_CLOSED_READ;
import static org.apache.ignite.events.EventType.EVT_IGFS_FILE_CLOSED_WRITE;
import static org.apache.ignite.events.EventType.EVT_IGFS_FILE_DELETED;
import static org.apache.ignite.events.EventType.EVT_IGFS_FILE_OPENED_READ;
import static org.apache.ignite.events.EventType.EVT_IGFS_FILE_OPENED_WRITE;
-import static org.apache.ignite.events.EventType.EVT_IGFS_FILE_RENAMED;
import static org.apache.ignite.events.EventType.EVT_IGFS_META_UPDATED;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
http://git-wip-us.apache.org/repos/asf/ignite/blob/dcd722a3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index a4212ba..35e77c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -18,10 +18,13 @@
package org.apache.ignite.internal.processors.igfs;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCompute;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteInterruptedException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterGroup;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.cluster.ClusterTopologyException;
import org.apache.ignite.configuration.FileSystemConfiguration;
import org.apache.ignite.events.EventType;
import org.apache.ignite.events.IgfsEvent;
@@ -37,6 +40,7 @@ import org.apache.ignite.igfs.IgfsPathIsNotDirectoryException;
import org.apache.ignite.igfs.IgfsPathNotFoundException;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystemPositionedReadable;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
@@ -44,6 +48,7 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.GridCacheInternal;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientAbstractCallable;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaDirectoryCreateProcessor;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaFileCreateProcessor;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaFileLockProcessor;
@@ -139,13 +144,21 @@ public class IgfsMetaManager extends IgfsManager {
/** Relaxed flag. */
private final boolean relaxed;
+ /** Client flag. */
+ private final boolean client;
+
+ /** Compute facade for client tasks. */
+ private IgniteCompute cliCompute;
+
/**
* Constructor.
*
* @param relaxed Relaxed mode flag.
+ * @param client Client flag.
*/
- public IgfsMetaManager(boolean relaxed) {
+ public IgfsMetaManager(boolean relaxed, boolean client) {
this.relaxed = relaxed;
+ this.client = client;
}
/**
@@ -217,6 +230,53 @@ public class IgfsMetaManager extends IgfsManager {
}
/**
+ * @return Client flag.
+ */
+ boolean isClient() {
+ return client;
+ }
+
+ /**
+ * Run client task.
+ *
+ * @param task Task.
+ * @return Result.
+ */
+ <T> T runClientTask(IgfsClientAbstractCallable<T> task) {
+ try {
+ return clientCompute().call(task);
+ }
+ catch (ClusterTopologyException e) {
+ throw new IgfsException("Failed to execute operation because there are no IGFS metadata nodes left." , e);
+ }
+ }
+
+ /**
+ * Get compute facade for client tasks.
+ *
+ * @return Compute facade.
+ */
+ private IgniteCompute clientCompute() {
+ assert client;
+
+ IgniteCompute cliCompute0 = cliCompute;
+
+ if (cliCompute0 == null) {
+ IgniteEx ignite = igfsCtx.kernalContext().grid();
+
+ ClusterGroup cluster = ignite.cluster().forIgfsMetadataDataNodes(cfg.getName(), cfg.getMetaCacheName());
+
+ cliCompute0 = ignite.compute(cluster);
+
+ cliCompute = cliCompute0;
+ }
+
+ assert cliCompute0 != null;
+
+ return cliCompute0;
+ }
+
+ /**
* Return nodes where meta cache is defined.
*
* @return Nodes where meta cache is defined.
http://git-wip-us.apache.org/repos/asf/ignite/blob/dcd722a3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNodePredicate.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNodePredicate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNodePredicate.java
new file mode 100644
index 0000000..e3a658a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNodePredicate.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs;
+
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * IGFS node predicate.
+ */
+public class IgfsNodePredicate implements IgnitePredicate<ClusterNode>, Binarylizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** IGFS name. */
+ private String igfsName;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsNodePredicate() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ */
+ public IgfsNodePredicate(@Nullable String igfsName) {
+ this.igfsName = igfsName;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean apply(ClusterNode node) {
+ return IgfsUtils.isIgfsNode(node, igfsName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+ BinaryRawWriter rawWriter = writer.rawWriter();
+
+ rawWriter.writeString(igfsName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+ BinaryRawReader rawReader = reader.rawReader();
+
+ igfsName = rawReader.readString();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsNodePredicate.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/dcd722a3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java
index 778de99..92b43cb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java
@@ -95,10 +95,12 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
/** {@inheritDoc} */
@Override public void start() throws IgniteCheckedException {
- if (ctx.config().isDaemon())
+ IgniteConfiguration igniteCfg = ctx.config();
+
+ if (igniteCfg.isDaemon())
return;
- FileSystemConfiguration[] cfgs = ctx.config().getFileSystemConfiguration();
+ FileSystemConfiguration[] cfgs = igniteCfg.getFileSystemConfiguration();
assert cfgs != null && cfgs.length > 0;
@@ -108,10 +110,27 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
for (FileSystemConfiguration cfg : cfgs) {
FileSystemConfiguration cfg0 = new FileSystemConfiguration(cfg);
+ boolean metaClient = true;
+
+ CacheConfiguration[] cacheCfgs = igniteCfg.getCacheConfiguration();
+
+ if (cacheCfgs != null) {
+ for (CacheConfiguration cacheCfg : cacheCfgs) {
+ if (F.eq(cacheCfg.getName(), cfg.getMetaCacheName())) {
+ metaClient = false;
+
+ break;
+ }
+ }
+ }
+
+ if (igniteCfg.isClientMode() != null && igniteCfg.isClientMode())
+ metaClient = true;
+
IgfsContext igfsCtx = new IgfsContext(
ctx,
cfg0,
- new IgfsMetaManager(cfg0.isRelaxedConsistency()),
+ new IgfsMetaManager(cfg0.isRelaxedConsistency(), metaClient),
new IgfsDataManager(),
new IgfsServerManager(),
new IgfsFragmentizerManager());
@@ -126,19 +145,17 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
if (log.isDebugEnabled())
log.debug("IGFS processor started.");
- IgniteConfiguration gridCfg = ctx.config();
-
// Node doesn't have IGFS if it:
// is daemon;
// doesn't have configured IGFS;
// doesn't have configured caches.
- if (gridCfg.isDaemon() || F.isEmpty(gridCfg.getFileSystemConfiguration()) ||
- F.isEmpty(gridCfg.getCacheConfiguration()))
+ if (igniteCfg.isDaemon() || F.isEmpty(igniteCfg.getFileSystemConfiguration()) ||
+ F.isEmpty(igniteCfg.getCacheConfiguration()))
return;
final Map<String, CacheConfiguration> cacheCfgs = new HashMap<>();
- F.forEach(gridCfg.getCacheConfiguration(), new CI1<CacheConfiguration>() {
+ F.forEach(igniteCfg.getCacheConfiguration(), new CI1<CacheConfiguration>() {
@Override public void apply(CacheConfiguration c) {
cacheCfgs.put(c.getName(), c);
}
@@ -146,9 +163,9 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
Collection<IgfsAttributes> attrVals = new ArrayList<>();
- assert gridCfg.getFileSystemConfiguration() != null;
+ assert igniteCfg.getFileSystemConfiguration() != null;
- for (FileSystemConfiguration igfsCfg : gridCfg.getFileSystemConfiguration()) {
+ for (FileSystemConfiguration igfsCfg : igniteCfg.getFileSystemConfiguration()) {
CacheConfiguration cacheCfg = cacheCfgs.get(igfsCfg.getDataCacheName());
if (cacheCfg == null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/dcd722a3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
index 1b97565..e45e34e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
@@ -56,6 +56,7 @@ import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_RETRIES_COUNT;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGFS;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
@@ -681,4 +682,24 @@ public class IgfsUtils {
else
return null;
}
+
+ /**
+ * Check whether provided node contains IGFS with the given name.
+ *
+ * @param node Node.
+ * @param igfsName IGFS name.
+ * @return {@code True} if it contains IGFS.
+ */
+ public static boolean isIgfsNode(ClusterNode node, String igfsName) {
+ assert node != null;
+
+ IgfsAttributes[] igfs = node.attribute(ATTR_IGFS);
+
+ if (igfs != null)
+ for (IgfsAttributes attrs : igfs)
+ if (F.eq(igfsName, attrs.igfsName()))
+ return true;
+
+ return false;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/dcd722a3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAbstractCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAbstractCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAbstractCallable.java
new file mode 100644
index 0000000..b83ed13
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAbstractCallable.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.processors.igfs.IgfsEx;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Abstract callable for IGFS tasks initiated on client node and passed to data node.
+ */
+public abstract class IgfsClientAbstractCallable<T> implements IgniteCallable<T>, Binarylizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** IGFS name. */
+ protected String igfsName;
+
+ /** Injected instance. */
+ @IgniteInstanceResource
+ private transient Ignite ignite;
+
+ /**
+ * Default constructor.
+ */
+ protected IgfsClientAbstractCallable() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ */
+ protected IgfsClientAbstractCallable(@Nullable String igfsName) {
+ this.igfsName = igfsName;
+ }
+
+ /** {@inheritDoc} */
+ @Override public final T call() throws Exception {
+ assert ignite != null;
+
+ IgfsEx igfs = (IgfsEx)ignite.fileSystem(igfsName);
+
+ return call0(igfs.context());
+ }
+
+ /**
+ * Execute task.
+ *
+ * @param ctx IGFS ocntext.
+ * @return Result.
+ * @throws Exception If failed.
+ */
+ protected abstract T call0(IgfsContext ctx) throws Exception;
+
+ /** {@inheritDoc} */
+ @Override public final void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+ BinaryRawWriter rawWriter = writer.rawWriter();
+
+ rawWriter.writeString(igfsName);
+
+ writeBinary0(rawWriter);
+ }
+
+ /** {@inheritDoc} */
+ @Override public final void readBinary(BinaryReader reader) throws BinaryObjectException {
+ BinaryRawReader rawReader = reader.rawReader();
+
+ igfsName = rawReader.readString();
+
+ readBinary0(rawReader);
+ }
+
+ /**
+ * Write binary.
+ *
+ * @param rawWriter Raw writer.
+ */
+ protected abstract void writeBinary0(BinaryRawWriter rawWriter);
+
+ /**
+ * Read binary.
+ *
+ * @param rawReader Raw reader.
+ */
+ protected abstract void readBinary0(BinaryRawReader rawReader);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/dcd722a3/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
index 633e9d0..76a88d9 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
@@ -69,6 +69,11 @@ public class IgniteClusterProcessProxy implements IgniteClusterEx {
}
/** {@inheritDoc} */
+ @Override public ClusterGroup forIgfsMetadataDataNodes(@Nullable String igfsName, @Nullable String metaCacheName) {
+ throw new UnsupportedOperationException("Operation is not supported yet.");
+ }
+
+ /** {@inheritDoc} */
@Override public ClusterNode localNode() {
return compute.call(new LocalNodeTask());
}
[03/17] ignite git commit: IGNITE-3232 - Inline transformers for
IgniteRDD.savePairs and IgniteRDD.saveValues
Posted by av...@apache.org.
IGNITE-3232 - Inline transformers for IgniteRDD.savePairs and IgniteRDD.saveValues
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0e4ef3b2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0e4ef3b2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0e4ef3b2
Branch: refs/heads/ignite-3216
Commit: 0e4ef3b2e842015ef27aa107a5ecdcc10a73a327
Parents: 50a8ec2
Author: Valentin Kulichenko <va...@gmail.com>
Authored: Thu Jun 2 16:03:12 2016 +0300
Committer: Valentin Kulichenko <va...@gmail.com>
Committed: Fri Jun 3 19:47:00 2016 +0300
----------------------------------------------------------------------
.../org/apache/ignite/spark/IgniteRDD.scala | 75 +++++++++++++++++++
.../org/apache/ignite/spark/JavaIgniteRDD.scala | 9 +++
.../org/apache/ignite/spark/IgniteRDDSpec.scala | 77 +++++++++++++++++++-
3 files changed, 160 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0e4ef3b2/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
index 0d8e730..036dfe6 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
@@ -180,6 +180,39 @@ class IgniteRDD[K, V] (
}
/**
+ * Saves values from given RDD into Ignite. A unique key will be generated for each value of the given RDD.
+ *
+ * @param rdd RDD instance to save values from.
+ * @param f Transformation function.
+ */
+ def saveValues[T](rdd: RDD[T], f: (T, IgniteContext[K, V]) \u21d2 V) = {
+ rdd.foreachPartition(it \u21d2 {
+ val ig = ic.ignite()
+
+ ensureCache()
+
+ val locNode = ig.cluster().localNode()
+
+ val node: Option[ClusterNode] = ig.cluster().forHost(locNode).nodes().find(!_.eq(locNode))
+
+ val streamer = ig.dataStreamer[Object, V](cacheName)
+
+ try {
+ it.foreach(t \u21d2 {
+ val value = f(t, ic)
+
+ val key = affinityKeyFunc(value, node.orNull)
+
+ streamer.addData(key, value)
+ })
+ }
+ finally {
+ streamer.close()
+ }
+ })
+ }
+
+ /**
* Saves values from the given key-value RDD into Ignite.
*
* @param rdd RDD instance to save values from.
@@ -209,6 +242,48 @@ class IgniteRDD[K, V] (
}
/**
+ * Saves values from the given RDD into Ignite.
+ *
+ * @param rdd RDD instance to save values from.
+ * @param f Transformation function.
+ * @param overwrite Boolean flag indicating whether the call on this method should overwrite existing
+ * values in Ignite cache.
+ */
+ def savePairs[T](rdd: RDD[T], f: (T, IgniteContext[K, V]) \u21d2 (K, V), overwrite: Boolean) = {
+ rdd.foreachPartition(it \u21d2 {
+ val ig = ic.ignite()
+
+ // Make sure to deploy the cache
+ ensureCache()
+
+ val streamer = ig.dataStreamer[K, V](cacheName)
+
+ try {
+ streamer.allowOverwrite(overwrite)
+
+ it.foreach(t \u21d2 {
+ val tup = f(t, ic)
+
+ streamer.addData(tup._1, tup._2)
+ })
+ }
+ finally {
+ streamer.close()
+ }
+ })
+ }
+
+ /**
+ * Saves values from the given RDD into Ignite.
+ *
+ * @param rdd RDD instance to save values from.
+ * @param f Transformation function.
+ */
+ def savePairs[T](rdd: RDD[T], f: (T, IgniteContext[K, V]) \u21d2 (K, V)): Unit = {
+ savePairs(rdd, f, overwrite = false)
+ }
+
+ /**
* Removes all values from the underlying Ignite cache.
*/
def clear(): Unit = {
http://git-wip-us.apache.org/repos/asf/ignite/blob/0e4ef3b2/modules/spark/src/main/scala/org/apache/ignite/spark/JavaIgniteRDD.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/JavaIgniteRDD.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/JavaIgniteRDD.scala
index 2e8702e..cac0e15 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/JavaIgniteRDD.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/JavaIgniteRDD.scala
@@ -80,12 +80,21 @@ class JavaIgniteRDD[K, V](override val rdd: IgniteRDD[K, V])
def saveValues(jrdd: JavaRDD[V]) = rdd.saveValues(JavaRDD.toRDD(jrdd))
+ def saveValues[T](jrdd: JavaRDD[T], f: (T, IgniteContext[K, V]) \u21d2 V) = rdd.saveValues(JavaRDD.toRDD(jrdd), f)
+
def savePairs(jrdd: JavaPairRDD[K, V]) = {
val rrdd: RDD[(K, V)] = JavaPairRDD.toRDD(jrdd)
rdd.savePairs(rrdd)
}
+ def savePairs[T](jrdd: JavaRDD[T], f: (T, IgniteContext[K, V]) \u21d2 (K, V), overwrite: Boolean = false) = {
+ rdd.savePairs(JavaRDD.toRDD(jrdd), f, overwrite)
+ }
+
+ def savePairs[T](jrdd: JavaRDD[T], f: (T, IgniteContext[K, V]) \u21d2 (K, V)): Unit =
+ savePairs(jrdd, f, overwrite = false)
+
def clear(): Unit = rdd.clear()
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0e4ef3b2/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala
index 4e90bc8..61040d9 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala
@@ -26,6 +26,7 @@ import org.apache.spark.SparkContext
import org.junit.runner.RunWith
import org.scalatest._
import org.scalatest.junit.JUnitRunner
+import scala.collection.JavaConversions._
import IgniteRDDSpec._
@@ -34,7 +35,7 @@ import scala.annotation.meta.field
@RunWith(classOf[JUnitRunner])
class IgniteRDDSpec extends FunSpec with Matchers with BeforeAndAfterAll with BeforeAndAfterEach {
describe("IgniteRDD") {
- it("should successfully store data to ignite") {
+ it("should successfully store data to ignite using savePairs") {
val sc = new SparkContext("local[*]", "test")
try {
@@ -59,6 +60,80 @@ class IgniteRDDSpec extends FunSpec with Matchers with BeforeAndAfterAll with Be
}
}
+ it("should successfully store data to ignite using savePairs with inline transformation") {
+ val sc = new SparkContext("local[*]", "test")
+
+ try {
+ val ic = new IgniteContext[String, String](sc,
+ () \u21d2 configuration("client", client = true))
+
+ // Save pairs ("0", "val0"), ("1", "val1"), ... to Ignite cache.
+ ic.fromCache(PARTITIONED_CACHE_NAME).savePairs(
+ sc.parallelize(0 to 10000, 2), (i: Int, ic) \u21d2 (String.valueOf(i), "val" + i))
+
+ // Check cache contents.
+ val ignite = Ignition.ignite("grid-0")
+
+ for (i \u2190 0 to 10000) {
+ val res = ignite.cache[String, String](PARTITIONED_CACHE_NAME).get(String.valueOf(i))
+
+ assert(res != null, "Value was not put to cache for key: " + i)
+ assert("val" + i == res, "Invalid value stored for key: " + i)
+ }
+ }
+ finally {
+ sc.stop()
+ }
+ }
+
+ it("should successfully store data to ignite using saveValues") {
+ val sc = new SparkContext("local[*]", "test")
+
+ try {
+ val ic = new IgniteContext[String, String](sc,
+ () \u21d2 configuration("client", client = true))
+
+ // Save pairs ("0", "val0"), ("1", "val1"), ... to Ignite cache.
+ ic.fromCache(PARTITIONED_CACHE_NAME).saveValues(
+ sc.parallelize(0 to 10000, 2).map(i \u21d2 "val" + i))
+
+ // Check cache contents.
+ val ignite = Ignition.ignite("grid-0")
+
+ val values = ignite.cache[String, String](PARTITIONED_CACHE_NAME).toList.map(e \u21d2 e.getValue)
+
+ for (i \u2190 0 to 10000)
+ assert(values.contains("val" + i), "Value not found for index: " + i)
+ }
+ finally {
+ sc.stop()
+ }
+ }
+
+ it("should successfully store data to ignite using saveValues with inline transformation") {
+ val sc = new SparkContext("local[*]", "test")
+
+ try {
+ val ic = new IgniteContext[String, String](sc,
+ () \u21d2 configuration("client", client = true))
+
+ // Save pairs ("0", "val0"), ("1", "val1"), ... to Ignite cache.
+ ic.fromCache(PARTITIONED_CACHE_NAME).saveValues(
+ sc.parallelize(0 to 10000, 2), (i: Int, ic) \u21d2 "val" + i)
+
+ // Check cache contents.
+ val ignite = Ignition.ignite("grid-0")
+
+ val values = ignite.cache[String, String](PARTITIONED_CACHE_NAME).toList.map(e \u21d2 e.getValue)
+
+ for (i \u2190 0 to 10000)
+ assert(values.contains("val" + i), "Value not found for index: " + i)
+ }
+ finally {
+ sc.stop()
+ }
+ }
+
it("should successfully read data from ignite") {
val sc = new SparkContext("local[*]", "test")
[04/17] ignite git commit: IGNITE-3232 - Fixed the number of
iterations to find an affinity key.
Posted by av...@apache.org.
IGNITE-3232 - Fixed the number of iterations to find an affinity key.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c10097e0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c10097e0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c10097e0
Branch: refs/heads/ignite-3216
Commit: c10097e0a89ae2c3e0c29283243e96d19281ee3b
Parents: 0e4ef3b
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Thu Jun 2 22:52:39 2016 +0300
Committer: Valentin Kulichenko <va...@gmail.com>
Committed: Fri Jun 3 19:47:13 2016 +0300
----------------------------------------------------------------------
.../spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10097e0/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
index 036dfe6..9ef9846 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
@@ -334,7 +334,8 @@ class IgniteRDD[K, V] (
private def affinityKeyFunc(value: V, node: ClusterNode): IgniteUuid = {
val aff = ic.ignite().affinity[IgniteUuid](cacheName)
- Stream.from(1, 1000).map(_ \u21d2 IgniteUuid.randomUuid()).find(node == null || aff.mapKeyToNode(_).eq(node))
+ Stream.from(1, Math.max(1000, aff.partitions() * 2))
+ .map(_ \u21d2 IgniteUuid.randomUuid()).find(node == null || aff.mapKeyToNode(_).eq(node))
.getOrElse(IgniteUuid.randomUuid())
}
}
[11/17] ignite git commit: IGNITE-3240: fixed serialization of String
and other primitive arrays by CacheObjectBinaryProcessorImpl
Posted by av...@apache.org.
IGNITE-3240: fixed serialization of String and other primitive arrays by CacheObjectBinaryProcessorImpl
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9e9252c2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9e9252c2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9e9252c2
Branch: refs/heads/ignite-3216
Commit: 9e9252c2e4ed7380b3933909ac297f6618708980
Parents: e409b67
Author: Denis Magda <dm...@gridgain.com>
Authored: Mon Jun 6 10:20:13 2016 +0300
Committer: Denis Magda <dm...@gridgain.com>
Committed: Mon Jun 6 10:20:13 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/binary/BinaryUtils.java | 16 ++++
.../processors/cache/CacheObjectContext.java | 3 +
.../GridCacheBinaryObjectsAbstractSelfTest.java | 78 +++++++++++++++++++-
3 files changed, 95 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e9252c2/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index e431474..0250e3b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -655,6 +655,22 @@ public class BinaryUtils {
}
/**
+ * @param arr Array to check.
+ * @return {@code true} if this array is of a known type.
+ */
+ public static boolean knownArray(Object arr) {
+ if (arr == null)
+ return false;
+
+ Class<?> cls = arr.getClass();
+
+ return cls == byte[].class || cls == short[].class || cls == int[].class || cls == long[].class ||
+ cls == float[].class || cls == double[].class || cls == char[].class || cls == boolean[].class ||
+ cls == String[].class || cls == UUID[].class || cls == Date[].class || cls == Timestamp[].class ||
+ cls == BigDecimal[].class;
+ }
+
+ /**
* Attempts to create a new collection of the same known type. Will return null if collection type is unknown.
*
* @param col Collection.
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e9252c2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java
index d22bc75..d7fdb83 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java
@@ -199,6 +199,9 @@ import org.apache.ignite.internal.util.typedef.F;
* @return Result.
*/
public Object[] unwrapBinariesInArrayIfNeeded(Object[] arr, boolean keepBinary, boolean cpy) {
+ if (BinaryUtils.knownArray(arr))
+ return arr;
+
Object[] res = new Object[arr.length];
for (int i = 0; i < arr.length; i++)
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e9252c2/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
index f776146..56cca39 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
@@ -17,13 +17,17 @@
package org.apache.ignite.internal.processors.cache.binary;
+import java.math.BigDecimal;
+import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import javax.cache.Cache;
import javax.cache.processor.EntryProcessor;
import javax.cache.processor.EntryProcessorException;
@@ -60,6 +64,7 @@ import org.apache.ignite.lang.IgniteBiInClosure;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
import org.apache.ignite.transactions.TransactionConcurrency;
@@ -422,6 +427,75 @@ public abstract class GridCacheBinaryObjectsAbstractSelfTest extends GridCommonA
/**
* @throws Exception If failed.
*/
+ public void testBasicArrays() throws Exception {
+ IgniteCache<Integer, Object> cache = jcache(0);
+
+ checkArrayClass(cache, new String[] {"abc"});
+
+ checkArrayClass(cache, new byte[] {1});
+
+ checkArrayClass(cache, new short[] {1});
+
+ checkArrayClass(cache, new int[] {1});
+
+ checkArrayClass(cache, new long[] {1});
+
+ checkArrayClass(cache, new float[] {1});
+
+ checkArrayClass(cache, new double[] {1});
+
+ checkArrayClass(cache, new char[] {'a'});
+
+ checkArrayClass(cache, new boolean[] {false});
+
+ checkArrayClass(cache, new UUID[] {UUID.randomUUID()});
+
+ checkArrayClass(cache, new Date[] {new Date()});
+
+ checkArrayClass(cache, new Timestamp[] {new Timestamp(System.currentTimeMillis())});
+
+ checkArrayClass(cache, new BigDecimal[] {new BigDecimal(100)});
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCustomArrays() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-3244");
+
+ IgniteCache<Integer, TestObject[]> cache = jcache(0);
+
+ for (int i = 0; i < ENTRY_CNT; i++) {
+ TestObject[] arr = new TestObject[] {new TestObject(i)};
+
+ cache.put(0, arr);
+ }
+
+
+ for (int i = 0; i < ENTRY_CNT; i++) {
+ TestObject[] obj = cache.get(i);
+
+ assertEquals(1, obj.length);
+ assertEquals(i, obj[0].val);
+ }
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @param arr Array to check.
+ */
+ private void checkArrayClass(IgniteCache<Integer, Object> cache, Object arr) {
+ cache.put(0, arr);
+
+ Object res = cache.get(0);
+
+ assertEquals(arr.getClass(), res.getClass());
+ GridTestUtils.deepEquals(arr, res);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testGetTx1() throws Exception {
checkGetTx(PESSIMISTIC, REPEATABLE_READ);
}
@@ -651,7 +725,6 @@ public abstract class GridCacheBinaryObjectsAbstractSelfTest extends GridCommonA
for (int j = 0; j < 10; j++)
keys.add(i++);
-
cacheBinaryAsync.getAll(keys);
Map<Integer, BinaryObject> objs = cacheBinaryAsync.<Map<Integer, BinaryObject>>future().get();
@@ -1045,7 +1118,8 @@ public abstract class GridCacheBinaryObjectsAbstractSelfTest extends GridCommonA
* No-op entry processor.
*/
private static class ObjectEntryProcessor implements EntryProcessor<Integer, TestObject, Boolean> {
- @Override public Boolean process(MutableEntry<Integer, TestObject> entry, Object... args) throws EntryProcessorException {
+ @Override
+ public Boolean process(MutableEntry<Integer, TestObject> entry, Object... args) throws EntryProcessorException {
TestObject obj = entry.getValue();
entry.setValue(new TestObject(obj.val));
[02/17] ignite git commit: IGNITE-3228: Minor fix.
Posted by av...@apache.org.
IGNITE-3228: Minor fix.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/50a8ec24
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/50a8ec24
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/50a8ec24
Branch: refs/heads/ignite-3216
Commit: 50a8ec24fcae6cb673a257789724dd5a2dbb734a
Parents: 2490b0a
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Jun 2 11:47:38 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Jun 2 11:47:38 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/shuffle/collections/HadoopMultimapBase.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/50a8ec24/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
index 7dcff3d..39b7c51 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
@@ -225,8 +225,6 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
long newPageSize = nextPageSize(writtenSize + requestedSize);
long newPagePtr = mem.allocate(newPageSize);
- System.out.println("ALLOCATED: " + newPageSize);
-
HadoopOffheapBuffer b = out.buffer();
b.set(newPagePtr, newPageSize);
[10/17] ignite git commit: ignite-gg-11181 - scanCount with offheap
index fix
Posted by av...@apache.org.
ignite-gg-11181 - scanCount with offheap index fix
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a0c6ae54
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a0c6ae54
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a0c6ae54
Branch: refs/heads/ignite-3216
Commit: a0c6ae542cdc1f885b038beef168a35b14e23ad5
Parents: e409b67
Author: Sergi Vladykin <se...@gmail.com>
Authored: Mon Jun 6 00:14:05 2016 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Mon Jun 6 00:14:05 2016 +0300
----------------------------------------------------------------------
.../unsafe/GridOffheapSnapTreeSelfTest.java | 2 +-
.../query/h2/opt/GridH2AbstractKeyValueRow.java | 23 +--
.../query/h2/opt/GridH2KeyValueRowOffheap.java | 17 +-
.../cache/IgniteCacheOffheapIndexScanTest.java | 195 +++++++++++++++++++
.../IgniteCacheQuerySelfTestSuite.java | 2 +
5 files changed, 226 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a0c6ae54/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridOffheapSnapTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridOffheapSnapTreeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridOffheapSnapTreeSelfTest.java
index 463b6dc..92d9ec2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridOffheapSnapTreeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridOffheapSnapTreeSelfTest.java
@@ -313,7 +313,7 @@ public class GridOffheapSnapTreeSelfTest extends GridCommonAbstractTest {
}
@Override public int hashCode() {
- return ptr;
+ throw new IllegalStateException();
}
@Override public String toString() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a0c6ae54/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2AbstractKeyValueRow.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2AbstractKeyValueRow.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2AbstractKeyValueRow.java
index ca5442a..fe6851d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2AbstractKeyValueRow.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2AbstractKeyValueRow.java
@@ -351,41 +351,37 @@ public abstract class GridH2AbstractKeyValueRow extends GridH2Row {
/** {@inheritDoc} */
@Override public void setKeyAndVersion(SearchRow old) {
- assert false;
+ throw new IllegalStateException();
}
/** {@inheritDoc} */
@Override public void setKey(long key) {
- assert false;
+ throw new IllegalStateException();
}
/** {@inheritDoc} */
@Override public Row getCopy() {
- assert false;
-
- return null;
+ throw new IllegalStateException();
}
/** {@inheritDoc} */
@Override public void setDeleted(boolean deleted) {
- assert false;
+ throw new IllegalStateException();
}
/** {@inheritDoc} */
@Override public long getKey() {
- assert false;
-
- return 0;
+ throw new IllegalStateException();
}
/** {@inheritDoc} */
@Override public void setSessionId(int sesId) {
- assert false;
+ throw new IllegalStateException();
}
/** {@inheritDoc} */
@Override public void setVersion(int ver) {
- assert false;
+ throw new IllegalStateException();
}
/**
@@ -469,4 +465,9 @@ public abstract class GridH2AbstractKeyValueRow extends GridH2Row {
throw new IllegalStateException();
}
}
+
+ /** {@inheritDoc} */
+ @Override public final int hashCode() {
+ throw new IllegalStateException();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a0c6ae54/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2KeyValueRowOffheap.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2KeyValueRowOffheap.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2KeyValueRowOffheap.java
index 2dd9f25..ee68431 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2KeyValueRowOffheap.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2KeyValueRowOffheap.java
@@ -170,7 +170,7 @@ public class GridH2KeyValueRowOffheap extends GridH2AbstractKeyValueRow {
}
}
else
- assert false : col;
+ throw new IllegalStateException("Column: " + col);
Data data = Data.create(null, bytes);
@@ -378,4 +378,19 @@ public class GridH2KeyValueRowOffheap extends GridH2AbstractKeyValueRow {
@Override protected void addOffheapRowId(SB sb) {
sb.a('-').a(ptr);
}
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+
+ if (obj instanceof GridH2KeyValueRowOffheap) {
+ GridH2KeyValueRowOffheap row = (GridH2KeyValueRowOffheap)obj;
+
+ if (pointer() == row.pointer())
+ return true;
+ }
+
+ return false;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a0c6ae54/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapIndexScanTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapIndexScanTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapIndexScanTest.java
new file mode 100644
index 0000000..dbc8a65
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapIndexScanTest.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.io.Serializable;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheMode.LOCAL;
+
+/**
+ * Based scanCount with offheap index issue.
+ */
+public class IgniteCacheOffheapIndexScanTest extends GridCommonAbstractTest {
+ /** */
+ private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static IgniteCache<Integer, Object> cache;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+ disco.setIpFinder(ipFinder);
+
+ cfg.setDiscoverySpi(disco);
+
+ CacheConfiguration<?,?> cacheCfg = new CacheConfiguration<>();
+
+ cacheCfg.setCacheMode(LOCAL);
+ cacheCfg.setMemoryMode(CacheMemoryMode.OFFHEAP_TIERED);
+ cacheCfg.setSqlOnheapRowCacheSize(256);
+ cacheCfg.setIndexedTypes(
+ Integer.class, Person.class
+ );
+
+ cfg.setCacheConfiguration(cacheCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ startGridsMultiThreaded(1, false);
+
+ cache = grid(0).cache(null);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testQueryPlan() throws Exception {
+ for (int i = 0 ; i < 1000; i++)
+ cache.put(i, new Person(i, "firstName" + i, "lastName" + i, i % 100));
+
+ final AtomicBoolean end = new AtomicBoolean();
+
+ IgniteInternalFuture<?> fut = multithreadedAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ while(!end.get())
+ cache.query(new SqlFieldsQuery("select _val from Person")).getAll();
+
+ return null;
+ }
+ }, 5);
+
+ for (int i = 0; i < 150; i++) {
+ String plan = (String)cache.query(new SqlFieldsQuery(
+ "explain analyze select count(*) from Person where salary = 50")).getAll().get(0).get(0);
+
+ assertTrue(plan, plan.contains("scanCount: 11 "));
+
+ Thread.sleep(100);
+ }
+
+ end.set(true);
+
+ fut.get();
+ }
+
+ /**
+ * Person record used for query test.
+ */
+ public static class Person implements Serializable {
+ /** Person ID. */
+ @QuerySqlField(index = true)
+ private int id;
+
+ /** Organization ID. */
+ @QuerySqlField(index = true)
+ private int orgId;
+
+ /** First name (not-indexed). */
+ @QuerySqlField
+ private String firstName;
+
+ /** Last name (not indexed). */
+ @QuerySqlField
+ private String lastName;
+
+ /** Salary. */
+ @QuerySqlField(index = true)
+ private double salary;
+
+ /**
+ * Constructs empty person.
+ */
+ public Person() {
+ // No-op.
+ }
+
+ /**
+ * Constructs person record that is not linked to any organization.
+ *
+ * @param id Person ID.
+ * @param firstName First name.
+ * @param lastName Last name.
+ * @param salary Salary.
+ */
+ public Person(int id, String firstName, String lastName, double salary) {
+ this(id, 0, firstName, lastName, salary);
+ }
+
+ /**
+ * Constructs person record.
+ *
+ * @param id Person ID.
+ * @param orgId Organization ID.
+ * @param firstName First name.
+ * @param lastName Last name.
+ * @param salary Salary.
+ */
+ public Person(int id, int orgId, String firstName, String lastName, double salary) {
+ this.id = id;
+ this.orgId = orgId;
+ this.firstName = firstName;
+ this.lastName = lastName;
+ this.salary = salary;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ return this == o || (o instanceof Person) && id == ((Person)o).id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "Person [firstName=" + firstName +
+ ", id=" + id +
+ ", orgId=" + orgId +
+ ", lastName=" + lastName +
+ ", salary=" + salary +
+ ']';
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a0c6ae54/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index 3c0e424..c82fd79 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -29,6 +29,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheCollocatedQuerySel
import org.apache.ignite.internal.processors.cache.IgniteCacheDuplicateEntityConfigurationSelfTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheLargeResultSelfTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapEvictQueryTest;
+import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapIndexScanTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapTieredMultithreadedSelfTest;
import org.apache.ignite.internal.processors.cache.IgniteCachePartitionedQueryMultiThreadedSelfTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheQueryEvictsMultiThreadedSelfTest;
@@ -100,6 +101,7 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
suite.addTestSuite(IgniteCacheOffheapEvictQueryTest.class);
suite.addTestSuite(IgniteCacheSqlQueryMultiThreadedSelfTest.class);
suite.addTestSuite(IgniteCacheOffheapTieredMultithreadedSelfTest.class);
+ suite.addTestSuite(IgniteCacheOffheapIndexScanTest.class);
suite.addTestSuite(IgniteCacheQueryNodeRestartSelfTest.class);
suite.addTestSuite(IgniteCacheQueryNodeRestartSelfTest2.class);
suite.addTestSuite(IgniteCacheClientQueryReplicatedNodeRestartSelfTest.class);
[07/17] ignite git commit: IGNITE-3247: IGFS: Optimized non-stream
client operations.
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
index 25c54e4..48eb665 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
@@ -27,7 +27,9 @@ import org.apache.ignite.internal.processors.igfs.IgfsBackupsPrimarySelfTest;
import org.apache.ignite.internal.processors.igfs.IgfsBlockMessageSystemPoolStarvationSelfTest;
import org.apache.ignite.internal.processors.igfs.IgfsCachePerBlockLruEvictionPolicySelfTest;
import org.apache.ignite.internal.processors.igfs.IgfsCacheSelfTest;
-import org.apache.ignite.internal.processors.igfs.IgfsClientCacheSelfTest;
+import org.apache.ignite.internal.processors.igfs.IgfsDualAsyncClientSelfTest;
+import org.apache.ignite.internal.processors.igfs.IgfsDualSyncClientSelfTest;
+import org.apache.ignite.internal.processors.igfs.IgfsPrimaryClientSelfTest;
import org.apache.ignite.internal.processors.igfs.IgfsDataManagerSelfTest;
import org.apache.ignite.internal.processors.igfs.IgfsDualAsyncSelfTest;
import org.apache.ignite.internal.processors.igfs.IgfsDualSyncSelfTest;
@@ -42,6 +44,7 @@ import org.apache.ignite.internal.processors.igfs.IgfsOneClientNodeTest;
import org.apache.ignite.internal.processors.igfs.IgfsPrimaryOffheapTieredSelfTest;
import org.apache.ignite.internal.processors.igfs.IgfsPrimaryOffheapValuesSelfTest;
import org.apache.ignite.internal.processors.igfs.IgfsPrimaryOptimziedMarshallerSelfTest;
+import org.apache.ignite.internal.processors.igfs.IgfsPrimaryRelaxedClientSelfTest;
import org.apache.ignite.internal.processors.igfs.IgfsPrimaryRelaxedSelfTest;
import org.apache.ignite.internal.processors.igfs.IgfsPrimarySelfTest;
import org.apache.ignite.internal.processors.igfs.IgfsProcessorSelfTest;
@@ -95,7 +98,11 @@ public class IgniteIgfsTestSuite extends TestSuite {
suite.addTest(new TestSuite(IgfsModesSelfTest.class));
suite.addTest(new TestSuite(IgfsMetricsSelfTest.class));
- suite.addTest(new TestSuite(IgfsClientCacheSelfTest.class));
+ suite.addTest(new TestSuite(IgfsPrimaryClientSelfTest.class));
+ suite.addTest(new TestSuite(IgfsPrimaryRelaxedClientSelfTest.class));
+ suite.addTest(new TestSuite(IgfsDualSyncClientSelfTest.class));
+ suite.addTest(new TestSuite(IgfsDualAsyncClientSelfTest.class));
+
suite.addTest(new TestSuite(IgfsOneClientNodeTest.class));
suite.addTest(new TestSuite(IgfsModeResolverSelfTest.class));
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1DualAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1DualAbstractTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1DualAbstractTest.java
index 81f5743..c5f3425 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1DualAbstractTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1DualAbstractTest.java
@@ -110,7 +110,7 @@ public abstract class Hadoop1DualAbstractTest extends IgfsDualAbstractSelfTest {
* @throws IOException On failure.
*/
protected void startUnderlying() throws Exception {
- startGridWithIgfs(GRID_NAME, IGFS_NAME, PRIMARY, null, SECONDARY_REST_CFG);
+ startGridWithIgfs(GRID_NAME, IGFS_NAME, PRIMARY, null, SECONDARY_REST_CFG, secondaryIpFinder);
}
/**
[08/17] ignite git commit: IGNITE-3247: IGFS: Optimized non-stream
client operations.
Posted by av...@apache.org.
IGNITE-3247: IGFS: Optimized non-stream client operations.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b96afb2f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b96afb2f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b96afb2f
Branch: refs/heads/ignite-3216
Commit: b96afb2fac5df060d340b7b92627186f3c82532b
Parents: dcd722a
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Sun Jun 5 21:00:53 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Sun Jun 5 21:00:53 2016 +0300
----------------------------------------------------------------------
.../java/org/apache/ignite/igfs/IgfsPath.java | 26 +++-
.../org/apache/ignite/igfs/IgfsPathSummary.java | 32 ++++-
.../processors/igfs/IgfsBlockLocationImpl.java | 87 ++++++++++--
.../internal/processors/igfs/IgfsFileImpl.java | 34 ++++-
.../internal/processors/igfs/IgfsImpl.java | 63 ++++++++-
.../processors/igfs/IgfsMetaManager.java | 2 +-
.../internal/processors/igfs/IgfsUtils.java | 34 +++++
.../igfs/client/IgfsClientAbstractCallable.java | 19 ++-
.../igfs/client/IgfsClientAffinityCallable.java | 95 +++++++++++++
.../igfs/client/IgfsClientDeleteCallable.java | 77 ++++++++++
.../igfs/client/IgfsClientExistsCallable.java | 58 ++++++++
.../igfs/client/IgfsClientInfoCallable.java | 59 ++++++++
.../client/IgfsClientListFilesCallable.java | 61 ++++++++
.../client/IgfsClientListPathsCallable.java | 60 ++++++++
.../igfs/client/IgfsClientMkdirsCallable.java | 82 +++++++++++
.../igfs/client/IgfsClientRenameCallable.java | 80 +++++++++++
.../igfs/client/IgfsClientSetTimesCallable.java | 87 ++++++++++++
.../igfs/client/IgfsClientSizeCallable.java | 59 ++++++++
.../igfs/client/IgfsClientSummaryCallable.java | 59 ++++++++
.../igfs/client/IgfsClientUpdateCallable.java | 81 +++++++++++
.../ignite/internal/util/IgniteUtils.java | 40 ++++++
.../processors/igfs/IgfsAbstractSelfTest.java | 49 ++++++-
.../igfs/IgfsClientCacheSelfTest.java | 139 -------------------
.../igfs/IgfsDualAsyncClientSelfTest.java | 28 ++++
.../igfs/IgfsDualSyncClientSelfTest.java | 28 ++++
.../processors/igfs/IgfsOneClientNodeTest.java | 5 +-
.../igfs/IgfsPrimaryClientSelfTest.java | 30 ++++
.../igfs/IgfsPrimaryRelaxedClientSelfTest.java | 28 ++++
.../ignite/testsuites/IgniteIgfsTestSuite.java | 11 +-
.../ignite/igfs/Hadoop1DualAbstractTest.java | 2 +-
30 files changed, 1336 insertions(+), 179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java
index f1ceb2e..c705274 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java
@@ -268,16 +268,32 @@ public final class IgfsPath implements Comparable<IgfsPath>, Externalizable, Bin
/** {@inheritDoc} */
@Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
- BinaryRawWriter out = writer.rawWriter();
-
- out.writeString(path);
+ writeRawBinary(writer.rawWriter());
}
/** {@inheritDoc} */
@Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
- BinaryRawReader in = reader.rawReader();
+ readRawBinary(reader.rawReader());
+ }
+
+ /**
+ * Write raw binary.
+ *
+ * @param writer Raw writer.
+ * @throws BinaryObjectException If failed.
+ */
+ public void writeRawBinary(BinaryRawWriter writer) throws BinaryObjectException {
+ writer.writeString(path);
+ }
- path = in.readString();
+ /**
+ * Read raw binary.
+ *
+ * @param reader Raw reader.
+ * @throws BinaryObjectException If failed.
+ */
+ public void readRawBinary(BinaryRawReader reader) throws BinaryObjectException {
+ path = reader.readString();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPathSummary.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPathSummary.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPathSummary.java
index e84e128..7d60b95 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPathSummary.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPathSummary.java
@@ -21,12 +21,20 @@ import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
* Path summary: total files count, total directories count, total length.
*/
-public class IgfsPathSummary implements Externalizable {
+public class IgfsPathSummary implements Externalizable, Binarylizable {
/** */
private static final long serialVersionUID = 0L;
@@ -134,6 +142,28 @@ public class IgfsPathSummary implements Externalizable {
}
/** {@inheritDoc} */
+ @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+ BinaryRawWriter rawWriter = writer.rawWriter();
+
+ rawWriter.writeInt(filesCnt);
+ rawWriter.writeInt(dirCnt);
+ rawWriter.writeLong(totalLen);
+
+ IgfsUtils.writePath(rawWriter, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+ BinaryRawReader rawReader = reader.rawReader();
+
+ filesCnt = rawReader.readInt();
+ dirCnt = rawReader.readInt();
+ totalLen = rawReader.readLong();
+
+ path = IgfsUtils.readPath(rawReader);
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return S.toString(IgfsPathSummary.class, this);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockLocationImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockLocationImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockLocationImpl.java
index 0ec31ba..2d4a0af 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockLocationImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockLocationImpl.java
@@ -27,6 +27,12 @@ import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.igfs.IgfsBlockLocation;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -37,7 +43,7 @@ import org.apache.ignite.internal.util.typedef.internal.U;
/**
* File block location in the grid.
*/
-public class IgfsBlockLocationImpl implements IgfsBlockLocation, Externalizable {
+public class IgfsBlockLocationImpl implements IgfsBlockLocation, Externalizable, Binarylizable {
/** */
private static final long serialVersionUID = 0L;
@@ -155,13 +161,7 @@ public class IgfsBlockLocationImpl implements IgfsBlockLocation, Externalizable
return S.toString(IgfsBlockLocationImpl.class, this);
}
- /**
- * Writes this object to data output. Note that this is not externalizable
- * interface because we want to eliminate any marshaller.
- *
- * @param out Data output to write.
- * @throws IOException If write failed.
- */
+ /** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
assert names != null;
assert hosts != null;
@@ -189,13 +189,7 @@ public class IgfsBlockLocationImpl implements IgfsBlockLocation, Externalizable
out.writeUTF(host);
}
- /**
- * Reads object from data input. Note we do not use externalizable interface
- * to eliminate marshaller.
- *
- * @param in Data input.
- * @throws IOException If read failed.
- */
+ /** {@inheritDoc} */
@Override public void readExternal(ObjectInput in) throws IOException {
start = in.readLong();
len = in.readLong();
@@ -226,6 +220,69 @@ public class IgfsBlockLocationImpl implements IgfsBlockLocation, Externalizable
hosts.add(in.readUTF());
}
+ /** {@inheritDoc} */
+ @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+ BinaryRawWriter rawWriter = writer.rawWriter();
+
+ assert names != null;
+ assert hosts != null;
+
+ rawWriter.writeLong(start);
+ rawWriter.writeLong(len);
+
+ rawWriter.writeBoolean(nodeIds != null);
+
+ if (nodeIds != null) {
+ rawWriter.writeInt(nodeIds.size());
+
+ for (UUID nodeId : nodeIds)
+ U.writeUuid(rawWriter, nodeId);
+ }
+
+ rawWriter.writeInt(names.size());
+
+ for (String name : names)
+ rawWriter.writeString(name);
+
+ rawWriter.writeInt(hosts.size());
+
+ for (String host : hosts)
+ rawWriter.writeString(host);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+ BinaryRawReader rawReader = reader.rawReader();
+
+ start = rawReader.readLong();
+ len = rawReader.readLong();
+
+ int size;
+
+ if (rawReader.readBoolean()) {
+ size = rawReader.readInt();
+
+ nodeIds = new ArrayList<>(size);
+
+ for (int i = 0; i < size; i++)
+ nodeIds.add(U.readUuid(rawReader));
+ }
+
+ size = rawReader.readInt();
+
+ names = new ArrayList<>(size);
+
+ for (int i = 0; i < size; i++)
+ names.add(rawReader.readString());
+
+ size = rawReader.readInt();
+
+ hosts = new ArrayList<>(size);
+
+ for (int i = 0; i < size; i++)
+ hosts.add(rawReader.readString());
+ }
+
/**
* Converts collection of rich nodes to block location data.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java
index be8d0fc..9f79f42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java
@@ -17,6 +17,12 @@
package org.apache.ignite.internal.processors.igfs;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.internal.util.typedef.internal.A;
@@ -35,7 +41,7 @@ import java.util.Map;
/**
* File or directory information.
*/
-public final class IgfsFileImpl implements IgfsFile, Externalizable {
+public final class IgfsFileImpl implements IgfsFile, Externalizable, Binarylizable {
/** */
private static final long serialVersionUID = 0L;
@@ -229,6 +235,32 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable {
}
/** {@inheritDoc} */
+ @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+ BinaryRawWriter rawWriter = writer.rawWriter();
+
+ IgfsUtils.writePath(rawWriter, path);
+ rawWriter.writeInt(blockSize);
+ rawWriter.writeLong(grpBlockSize);
+ rawWriter.writeLong(len);
+ IgfsUtils.writeProperties(rawWriter, props);
+ rawWriter.writeLong(accessTime);
+ rawWriter.writeLong(modificationTime);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+ BinaryRawReader rawReader = reader.rawReader();
+
+ path = IgfsUtils.readPath(rawReader);
+ blockSize = rawReader.readInt();
+ grpBlockSize = rawReader.readLong();
+ len = rawReader.readLong();
+ props = IgfsUtils.readProperties(rawReader);
+ accessTime = rawReader.readLong();
+ modificationTime = rawReader.readLong();
+ }
+
+ /** {@inheritDoc} */
@Override public int hashCode() {
return path.hashCode();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 2796839..b7b3fac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -55,6 +55,18 @@ import org.apache.ignite.internal.managers.communication.GridMessageListener;
import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.hadoop.HadoopPayloadAware;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientAffinityCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientDeleteCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientExistsCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientInfoCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientListFilesCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientListPathsCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientMkdirsCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientRenameCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientSetTimesCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientSizeCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientSummaryCallable;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientUpdateCallable;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.util.GridSpinBusyLock;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
@@ -530,6 +542,9 @@ public final class IgfsImpl implements IgfsEx {
@Override public boolean exists(final IgfsPath path) {
A.notNull(path, "path");
+ if (meta.isClient())
+ return meta.runClientTask(new IgfsClientExistsCallable(cfg.getName(), path));
+
return safeOp(new Callable<Boolean>() {
@Override public Boolean call() throws Exception {
if (log.isDebugEnabled())
@@ -575,6 +590,9 @@ public final class IgfsImpl implements IgfsEx {
@Override @Nullable public IgfsFile info(final IgfsPath path) {
A.notNull(path, "path");
+ if (meta.isClient())
+ return meta.runClientTask(new IgfsClientInfoCallable(cfg.getName(), path));
+
return safeOp(new Callable<IgfsFile>() {
@Override public IgfsFile call() throws Exception {
if (log.isDebugEnabled())
@@ -591,6 +609,9 @@ public final class IgfsImpl implements IgfsEx {
@Override public IgfsPathSummary summary(final IgfsPath path) {
A.notNull(path, "path");
+ if (meta.isClient())
+ return meta.runClientTask(new IgfsClientSummaryCallable(cfg.getName(), path));
+
return safeOp(new Callable<IgfsPathSummary>() {
@Override public IgfsPathSummary call() throws Exception {
if (log.isDebugEnabled())
@@ -616,6 +637,9 @@ public final class IgfsImpl implements IgfsEx {
A.notNull(props, "props");
A.ensure(!props.isEmpty(), "!props.isEmpty()");
+ if (meta.isClient())
+ return meta.runClientTask(new IgfsClientUpdateCallable(cfg.getName(), path, props));
+
return safeOp(new Callable<IgfsFile>() {
@Override public IgfsFile call() throws Exception {
if (log.isDebugEnabled())
@@ -662,6 +686,12 @@ public final class IgfsImpl implements IgfsEx {
A.notNull(src, "src");
A.notNull(dest, "dest");
+ if (meta.isClient()) {
+ meta.runClientTask(new IgfsClientRenameCallable(cfg.getName(), src, dest));
+
+ return;
+ }
+
safeOp(new Callable<Void>() {
@Override public Void call() throws Exception {
if (log.isDebugEnabled())
@@ -708,6 +738,9 @@ public final class IgfsImpl implements IgfsEx {
@Override public boolean delete(final IgfsPath path, final boolean recursive) {
A.notNull(path, "path");
+ if (meta.isClient())
+ return meta.runClientTask(new IgfsClientDeleteCallable(cfg.getName(), path, recursive));
+
return safeOp(new Callable<Boolean>() {
@Override public Boolean call() throws Exception {
if (log.isDebugEnabled())
@@ -761,6 +794,12 @@ public final class IgfsImpl implements IgfsEx {
@Override public void mkdirs(final IgfsPath path, @Nullable final Map<String, String> props) {
A.notNull(path, "path");
+ if (meta.isClient()) {
+ meta.runClientTask(new IgfsClientMkdirsCallable(cfg.getName(), path, props));
+
+ return ;
+ }
+
safeOp(new Callable<Void>() {
@Override public Void call() throws Exception {
if (log.isDebugEnabled())
@@ -790,6 +829,9 @@ public final class IgfsImpl implements IgfsEx {
@Override public Collection<IgfsPath> listPaths(final IgfsPath path) {
A.notNull(path, "path");
+ if (meta.isClient())
+ meta.runClientTask(new IgfsClientListPathsCallable(cfg.getName(), path));
+
return safeOp(new Callable<Collection<IgfsPath>>() {
@Override public Collection<IgfsPath> call() throws Exception {
if (log.isDebugEnabled())
@@ -840,6 +882,9 @@ public final class IgfsImpl implements IgfsEx {
@Override public Collection<IgfsFile> listFiles(final IgfsPath path) {
A.notNull(path, "path");
+ if (meta.isClient())
+ meta.runClientTask(new IgfsClientListFilesCallable(cfg.getName(), path));
+
return safeOp(new Callable<Collection<IgfsFile>>() {
@Override public Collection<IgfsFile> call() throws Exception {
if (log.isDebugEnabled())
@@ -1146,11 +1191,17 @@ public final class IgfsImpl implements IgfsEx {
@Override public void setTimes(final IgfsPath path, final long accessTime, final long modificationTime) {
A.notNull(path, "path");
+ if (accessTime == -1 && modificationTime == -1)
+ return;
+
+ if (meta.isClient()) {
+ meta.runClientTask(new IgfsClientSetTimesCallable(cfg.getName(), path, accessTime, modificationTime));
+
+ return;
+ }
+
safeOp(new Callable<Void>() {
@Override public Void call() throws Exception {
- if (accessTime == -1 && modificationTime == -1)
- return null;
-
FileDescriptor desc = getFileDescriptor(path);
if (desc == null) {
@@ -1197,6 +1248,9 @@ public final class IgfsImpl implements IgfsEx {
A.ensure(start >= 0, "start >= 0");
A.ensure(len >= 0, "len >= 0");
+ if (meta.isClient())
+ return meta.runClientTask(new IgfsClientAffinityCallable(cfg.getName(), path, start, len, maxLen));
+
return safeOp(new Callable<Collection<IgfsBlockLocation>>() {
@Override public Collection<IgfsBlockLocation> call() throws Exception {
if (log.isDebugEnabled())
@@ -1278,6 +1332,9 @@ public final class IgfsImpl implements IgfsEx {
@Override public long size(final IgfsPath path) {
A.notNull(path, "path");
+ if (meta.isClient())
+ return meta.runClientTask(new IgfsClientSizeCallable(cfg.getName(), path));
+
return safeOp(new Callable<Long>() {
@Override public Long call() throws Exception {
IgniteUuid nextId = meta.fileId(path);
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index 35e77c4..3295249 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -247,7 +247,7 @@ public class IgfsMetaManager extends IgfsManager {
return clientCompute().call(task);
}
catch (ClusterTopologyException e) {
- throw new IgfsException("Failed to execute operation because there are no IGFS metadata nodes left." , e);
+ throw new IgfsException("Failed to execute operation because there are no IGFS metadata nodes." , e);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
index e45e34e..6fa9877 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
@@ -684,6 +684,40 @@ public class IgfsUtils {
}
/**
+ * Write IGFS path.
+ *
+ * @param writer Writer.
+ * @param path Path.
+ */
+ public static void writePath(BinaryRawWriter writer, @Nullable IgfsPath path) {
+ if (path != null) {
+ writer.writeBoolean(true);
+
+ path.writeRawBinary(writer);
+ }
+ else
+ writer.writeBoolean(false);
+ }
+
+ /**
+ * Read IGFS path.
+ *
+ * @param reader Reader.
+ * @return Path.
+ */
+ @Nullable public static IgfsPath readPath(BinaryRawReader reader) {
+ if (reader.readBoolean()) {
+ IgfsPath path = new IgfsPath();
+
+ path.readRawBinary(reader);
+
+ return path;
+ }
+ else
+ return null;
+ }
+
+ /**
* Check whether provided node contains IGFS with the given name.
*
* @param node Node.
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAbstractCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAbstractCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAbstractCallable.java
index b83ed13..d9c3456 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAbstractCallable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAbstractCallable.java
@@ -24,8 +24,10 @@ import org.apache.ignite.binary.BinaryRawWriter;
import org.apache.ignite.binary.BinaryReader;
import org.apache.ignite.binary.BinaryWriter;
import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.internal.processors.igfs.IgfsContext;
import org.apache.ignite.internal.processors.igfs.IgfsEx;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.jetbrains.annotations.Nullable;
@@ -40,6 +42,9 @@ public abstract class IgfsClientAbstractCallable<T> implements IgniteCallable<T>
/** IGFS name. */
protected String igfsName;
+ /** Path for operation. */
+ protected IgfsPath path;
+
/** Injected instance. */
@IgniteInstanceResource
private transient Ignite ignite;
@@ -55,9 +60,11 @@ public abstract class IgfsClientAbstractCallable<T> implements IgniteCallable<T>
* Constructor.
*
* @param igfsName IGFS name.
+ * @param path Path.
*/
- protected IgfsClientAbstractCallable(@Nullable String igfsName) {
+ protected IgfsClientAbstractCallable(@Nullable String igfsName, @Nullable IgfsPath path) {
this.igfsName = igfsName;
+ this.path = path;
}
/** {@inheritDoc} */
@@ -83,6 +90,7 @@ public abstract class IgfsClientAbstractCallable<T> implements IgniteCallable<T>
BinaryRawWriter rawWriter = writer.rawWriter();
rawWriter.writeString(igfsName);
+ IgfsUtils.writePath(rawWriter, path);
writeBinary0(rawWriter);
}
@@ -92,6 +100,7 @@ public abstract class IgfsClientAbstractCallable<T> implements IgniteCallable<T>
BinaryRawReader rawReader = reader.rawReader();
igfsName = rawReader.readString();
+ path = IgfsUtils.readPath(rawReader);
readBinary0(rawReader);
}
@@ -101,12 +110,16 @@ public abstract class IgfsClientAbstractCallable<T> implements IgniteCallable<T>
*
* @param rawWriter Raw writer.
*/
- protected abstract void writeBinary0(BinaryRawWriter rawWriter);
+ protected void writeBinary0(BinaryRawWriter rawWriter) {
+ // No-op.
+ }
/**
* Read binary.
*
* @param rawReader Raw reader.
*/
- protected abstract void readBinary0(BinaryRawReader rawReader);
+ protected void readBinary0(BinaryRawReader rawReader) {
+ // No-op.
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAffinityCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAffinityCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAffinityCallable.java
new file mode 100644
index 0000000..1668f36
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientAffinityCallable.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.igfs.IgfsBlockLocation;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Collection;
+
+/**
+ * IGFS client affinity callable.
+ */
+public class IgfsClientAffinityCallable extends IgfsClientAbstractCallable<Collection<IgfsBlockLocation>> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Start. */
+ private long start;
+
+ /** Length. */
+ private long len;
+
+ /** Maximum length. */
+ private long maxLen;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientAffinityCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ * @param start Start.
+ * @param len Length.
+ * @param maxLen Maximum length.
+ */
+ public IgfsClientAffinityCallable(@Nullable String igfsName, IgfsPath path, long start, long len, long maxLen) {
+ super(igfsName, path);
+
+ this.start = start;
+ this.len = len;
+ this.maxLen = maxLen;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Collection<IgfsBlockLocation> call0(IgfsContext ctx) throws Exception {
+ return ctx.igfs().affinity(path, start, len, maxLen);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary0(BinaryRawWriter writer) throws BinaryObjectException {
+ writer.writeLong(start);
+ writer.writeLong(len);
+ writer.writeLong(maxLen);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary0(BinaryRawReader reader) throws BinaryObjectException {
+ start = reader.readLong();
+ len = reader.readLong();
+ maxLen = reader.readLong();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientAffinityCallable.class, this);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientDeleteCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientDeleteCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientDeleteCallable.java
new file mode 100644
index 0000000..c1b8be8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientDeleteCallable.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * IGFS client delete callable.
+ */
+public class IgfsClientDeleteCallable extends IgfsClientAbstractCallable<Boolean> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Recursion flag. */
+ private boolean recursive;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientDeleteCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ * @param recursive Recursive flag.
+ */
+ public IgfsClientDeleteCallable(@Nullable String igfsName, IgfsPath path, boolean recursive) {
+ super(igfsName, path);
+
+ this.recursive = recursive;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Boolean call0(IgfsContext ctx) throws Exception {
+ return ctx.igfs().delete(path, recursive);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary0(BinaryRawWriter writer) throws BinaryObjectException {
+ writer.writeBoolean(recursive);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary0(BinaryRawReader reader) throws BinaryObjectException {
+ recursive = reader.readBoolean();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientDeleteCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientExistsCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientExistsCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientExistsCallable.java
new file mode 100644
index 0000000..04b63d8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientExistsCallable.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * IGFS client exists callable.
+ */
+public class IgfsClientExistsCallable extends IgfsClientAbstractCallable<Boolean> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientExistsCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ */
+ public IgfsClientExistsCallable(@Nullable String igfsName, IgfsPath path) {
+ super(igfsName, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Boolean call0(IgfsContext ctx) throws Exception {
+ return ctx.igfs().exists(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientExistsCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientInfoCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientInfoCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientInfoCallable.java
new file mode 100644
index 0000000..f97c3c4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientInfoCallable.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.igfs.IgfsFile;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * IGFS client info callable.
+ */
+public class IgfsClientInfoCallable extends IgfsClientAbstractCallable<IgfsFile> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientInfoCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ */
+ public IgfsClientInfoCallable(@Nullable String igfsName, IgfsPath path) {
+ super(igfsName, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgfsFile call0(IgfsContext ctx) throws Exception {
+ return ctx.igfs().info(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientInfoCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientListFilesCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientListFilesCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientListFilesCallable.java
new file mode 100644
index 0000000..325e714
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientListFilesCallable.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.igfs.IgfsFile;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Collection;
+
+/**
+ * IGFS client list files callable.
+ */
+public class IgfsClientListFilesCallable extends IgfsClientAbstractCallable<Collection<IgfsFile>> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientListFilesCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ */
+ public IgfsClientListFilesCallable(@Nullable String igfsName, IgfsPath path) {
+ super(igfsName, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Collection<IgfsFile> call0(IgfsContext ctx) throws Exception {
+ return ctx.igfs().listFiles(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientListFilesCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientListPathsCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientListPathsCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientListPathsCallable.java
new file mode 100644
index 0000000..78b4c84
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientListPathsCallable.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Collection;
+
+/**
+ * IGFS client list paths callable.
+ */
+public class IgfsClientListPathsCallable extends IgfsClientAbstractCallable<Collection<IgfsPath>> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientListPathsCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ */
+ public IgfsClientListPathsCallable(@Nullable String igfsName, IgfsPath path) {
+ super(igfsName, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Collection<IgfsPath> call0(IgfsContext ctx) throws Exception {
+ return ctx.igfs().listPaths(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientListPathsCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientMkdirsCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientMkdirsCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientMkdirsCallable.java
new file mode 100644
index 0000000..944da6f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientMkdirsCallable.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Map;
+
+/**
+ * IGFS client mkdirs callable.
+ */
+public class IgfsClientMkdirsCallable extends IgfsClientAbstractCallable<Void> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Properties. */
+ private Map<String, String> props;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientMkdirsCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ * @param props Properties.
+ */
+ public IgfsClientMkdirsCallable(@Nullable String igfsName, IgfsPath path, @Nullable Map<String, String> props) {
+ super(igfsName, path);
+
+ this.props = props;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Void call0(IgfsContext ctx) throws Exception {
+ ctx.igfs().mkdirs(path, props);
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary0(BinaryRawWriter writer) throws BinaryObjectException {
+ IgfsUtils.writeProperties(writer, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary0(BinaryRawReader reader) throws BinaryObjectException {
+ props = IgfsUtils.readProperties(reader);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientMkdirsCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientRenameCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientRenameCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientRenameCallable.java
new file mode 100644
index 0000000..55afb83
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientRenameCallable.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * IGFS client rename callable.
+ */
+public class IgfsClientRenameCallable extends IgfsClientAbstractCallable<Void> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Destination path. */
+ private IgfsPath destPath;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientRenameCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param srcPath Source path.
+ * @param destPath Destination path.
+ */
+ public IgfsClientRenameCallable(@Nullable String igfsName, IgfsPath srcPath, IgfsPath destPath) {
+ super(igfsName, srcPath);
+
+ this.destPath = destPath;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Void call0(IgfsContext ctx) throws Exception {
+ ctx.igfs().rename(path, destPath);
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary0(BinaryRawWriter writer) throws BinaryObjectException {
+ IgfsUtils.writePath(writer, destPath);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary0(BinaryRawReader reader) throws BinaryObjectException {
+ destPath = IgfsUtils.readPath(reader);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientRenameCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientSetTimesCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientSetTimesCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientSetTimesCallable.java
new file mode 100644
index 0000000..277effc
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientSetTimesCallable.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * IGFS client set times callable.
+ */
+public class IgfsClientSetTimesCallable extends IgfsClientAbstractCallable<Void> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Access time. */
+ private long accessTime;
+
+ /** Modification time. */
+ private long modificationTime;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientSetTimesCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ * @param accessTime Access time.
+ * @param modificationTime Modification time.
+ */
+ public IgfsClientSetTimesCallable(@Nullable String igfsName, IgfsPath path, long accessTime,
+ long modificationTime) {
+ super(igfsName, path);
+
+ this.accessTime = accessTime;
+ this.modificationTime = modificationTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Void call0(IgfsContext ctx) throws Exception {
+ ctx.igfs().setTimes(path, accessTime, modificationTime);
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary0(BinaryRawWriter writer) throws BinaryObjectException {
+ writer.writeLong(accessTime);
+ writer.writeLong(modificationTime);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary0(BinaryRawReader reader) throws BinaryObjectException {
+ accessTime = reader.readLong();
+ modificationTime = reader.readLong();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientSetTimesCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientSizeCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientSizeCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientSizeCallable.java
new file mode 100644
index 0000000..474a940
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientSizeCallable.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.igfs.IgfsFile;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * IGFS client size callable.
+ */
+public class IgfsClientSizeCallable extends IgfsClientAbstractCallable<Long> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientSizeCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ */
+ public IgfsClientSizeCallable(@Nullable String igfsName, IgfsPath path) {
+ super(igfsName, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Long call0(IgfsContext ctx) throws Exception {
+ return ctx.igfs().size(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientSizeCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientSummaryCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientSummaryCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientSummaryCallable.java
new file mode 100644
index 0000000..7e29029
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientSummaryCallable.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.igfs.IgfsPathSummary;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * IGFS client summary callable.
+ */
+public class IgfsClientSummaryCallable extends IgfsClientAbstractCallable<IgfsPathSummary> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientSummaryCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ */
+ public IgfsClientSummaryCallable(@Nullable String igfsName, IgfsPath path) {
+ super(igfsName, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgfsPathSummary call0(IgfsContext ctx) throws Exception {
+ return ctx.igfs().summary(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientSummaryCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientUpdateCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientUpdateCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientUpdateCallable.java
new file mode 100644
index 0000000..4acf4eb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/IgfsClientUpdateCallable.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client;
+
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.igfs.IgfsFile;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Map;
+
+/**
+ * IGFS client update callable.
+ */
+public class IgfsClientUpdateCallable extends IgfsClientAbstractCallable<IgfsFile> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Properties. */
+ private Map<String, String> props;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientUpdateCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ * @param props Properties.
+ */
+ public IgfsClientUpdateCallable(@Nullable String igfsName, IgfsPath path, @Nullable Map<String, String> props) {
+ super(igfsName, path);
+
+ this.props = props;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgfsFile call0(IgfsContext ctx) throws Exception {
+ return ctx.igfs().update(path, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary0(BinaryRawWriter writer) throws BinaryObjectException {
+ IgfsUtils.writeProperties(writer, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary0(BinaryRawReader reader) throws BinaryObjectException {
+ props = IgfsUtils.readProperties(reader);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientUpdateCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 3717d31..c898909 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -149,6 +149,8 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteInterruptedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
import org.apache.ignite.cluster.ClusterGroupEmptyException;
import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.cluster.ClusterNode;
@@ -4710,6 +4712,44 @@ public abstract class IgniteUtils {
}
/**
+ * Writes UUID to binary writer.
+ *
+ * @param out Output Binary writer.
+ * @param uid UUID to write.
+ * @throws IOException If write failed.
+ */
+ public static void writeUuid(BinaryRawWriter out, UUID uid) {
+ // Write null flag.
+ if (uid != null) {
+ out.writeBoolean(true);
+
+ out.writeLong(uid.getMostSignificantBits());
+ out.writeLong(uid.getLeastSignificantBits());
+ }
+ else
+ out.writeBoolean(false);
+ }
+
+ /**
+ * Reads UUID from binary reader.
+ *
+ * @param in Binary reader.
+ * @return Read UUID.
+ * @throws IOException If read failed.
+ */
+ @Nullable public static UUID readUuid(BinaryRawReader in) {
+ // If UUID is not null.
+ if (in.readBoolean()) {
+ long most = in.readLong();
+ long least = in.readLong();
+
+ return new UUID(most, least);
+ }
+ else
+ return null;
+ }
+
+ /**
* Writes {@link org.apache.ignite.lang.IgniteUuid} to output stream. This method is meant to be used by
* implementations of {@link Externalizable} interface.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
index 2f6c7bf..3fb7b91 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteFileSystem;
+import org.apache.ignite.Ignition;
import org.apache.ignite.cache.CacheMemoryMode;
import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -55,6 +56,7 @@ import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
import org.jetbrains.annotations.Nullable;
@@ -189,6 +191,12 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
/** Memory mode. */
protected final CacheMemoryMode memoryMode;
+ /** IP finder for primary topology. */
+ protected final TcpDiscoveryVmIpFinder primaryIpFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** IP finder for secondary topology. */
+ protected final TcpDiscoveryVmIpFinder secondaryIpFinder = new TcpDiscoveryVmIpFinder(true);
+
static {
PRIMARY_REST_CFG = new IgfsIpcEndpointConfiguration();
@@ -233,6 +241,13 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
}
/**
+ * @return Client flag.
+ */
+ protected boolean client() {
+ return false;
+ }
+
+ /**
* @return Use optimzied marshaller flag.
*/
protected boolean useOptimizedMarshaller() {
@@ -258,9 +273,29 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
@Override protected void beforeTestsStarted() throws Exception {
igfsSecondaryFileSystem = createSecondaryFileSystemStack();
- Ignite ignite = startGridWithIgfs("ignite", "igfs", mode, igfsSecondaryFileSystem, PRIMARY_REST_CFG);
+ if (client()) {
+ // Start server.
+ startGridWithIgfs("ignite_srv", "igfs", mode, igfsSecondaryFileSystem, null, primaryIpFinder);
- igfs = (IgfsImpl) ignite.fileSystem("igfs");
+ // Start client.
+ Ignition.setClientMode(true);
+
+ try {
+ Ignite ignite = startGridWithIgfs("ignite", "igfs", mode, igfsSecondaryFileSystem,
+ PRIMARY_REST_CFG, primaryIpFinder);
+
+ igfs = (IgfsImpl) ignite.fileSystem("igfs");
+ }
+ finally {
+ Ignition.setClientMode(false);
+ }
+ }
+ else {
+ Ignite ignite = startGridWithIgfs("ignite", "igfs", mode, igfsSecondaryFileSystem,
+ PRIMARY_REST_CFG, primaryIpFinder);
+
+ igfs = (IgfsImpl) ignite.fileSystem("igfs");
+ }
}
/**
@@ -271,7 +306,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
*/
protected IgfsSecondaryFileSystem createSecondaryFileSystemStack() throws Exception {
Ignite igniteSecondary = startGridWithIgfs("ignite-secondary", "igfs-secondary", PRIMARY, null,
- SECONDARY_REST_CFG);
+ SECONDARY_REST_CFG, secondaryIpFinder);
IgfsEx secondaryIgfsImpl = (IgfsEx) igniteSecondary.fileSystem("igfs-secondary");
@@ -300,12 +335,14 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param mode IGFS mode.
* @param secondaryFs Secondary file system (optional).
* @param restCfg Rest configuration string (optional).
+ * @param ipFinder IP finder.
* @return Started grid instance.
* @throws Exception If failed.
*/
@SuppressWarnings("unchecked")
protected Ignite startGridWithIgfs(String gridName, String igfsName, IgfsMode mode,
- @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable IgfsIpcEndpointConfiguration restCfg) throws Exception {
+ @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable IgfsIpcEndpointConfiguration restCfg,
+ TcpDiscoveryIpFinder ipFinder) throws Exception {
FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName("dataCache");
@@ -322,6 +359,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
dataCacheCfg.setName("dataCache");
+ dataCacheCfg.setNearConfiguration(null);
dataCacheCfg.setCacheMode(PARTITIONED);
dataCacheCfg.setNearConfiguration(null);
dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
@@ -334,6 +372,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
metaCacheCfg.setName("metaCache");
+ metaCacheCfg.setNearConfiguration(null);
metaCacheCfg.setCacheMode(REPLICATED);
metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
@@ -347,7 +386,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+ discoSpi.setIpFinder(ipFinder);
prepareCacheConfigurations(dataCacheCfg, metaCacheCfg);
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsClientCacheSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsClientCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsClientCacheSelfTest.java
deleted file mode 100644
index 8e8eac1..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsClientCacheSelfTest.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.igfs;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.FileSystemConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper;
-import org.apache.ignite.igfs.IgfsMode;
-import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
-import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
-import static org.apache.ignite.cache.CacheMode.REPLICATED;
-
-/**
- * Test for igfs with nodes in client mode (see {@link IgniteConfiguration#setClientMode(boolean)}.
- */
-public class IgfsClientCacheSelfTest extends IgfsAbstractSelfTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** Meta-information cache name. */
- private static final String META_CACHE_NAME = "meta";
-
- /** Data cache name. */
- private static final String DATA_CACHE_NAME = null;
-
- /**
- * Constructor.
- */
- public IgfsClientCacheSelfTest() {
- super(IgfsMode.PRIMARY);
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- igfsSecondaryFileSystem = createSecondaryFileSystemStack();
-
- Ignite ignitePrimary = G.start(getConfiguration(getTestGridName(1)));
-
- igfs = (IgfsImpl) ignitePrimary.fileSystem("igfs");
- }
-
- /**{@inheritDoc} */
- protected IgfsSecondaryFileSystem createSecondaryFileSystemStack() throws Exception {
- Ignite igniteSecondary = G.start(getConfiguration(getTestGridName(0)));
-
- IgfsEx secondaryIgfsImpl = (IgfsEx)igniteSecondary.fileSystem("igfs");
-
- igfsSecondary = new IgfsExUniversalFileSystemAdapter(secondaryIgfsImpl);
-
- return secondaryIgfsImpl.asSecondary();
- }
-
- /**
- *
- * @param gridName Grid name.
- * @return Ignite configuration.
- * @throws Exception If failed.
- */
- protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setCacheConfiguration(
- cacheConfiguration(META_CACHE_NAME),
- cacheConfiguration(DATA_CACHE_NAME)
- );
-
- TcpDiscoverySpi disco = new TcpDiscoverySpi();
-
- disco.setIpFinder(IP_FINDER);
-
- if (!gridName.equals(getTestGridName(0))) {
- cfg.setClientMode(true);
-
- disco.setForceServerMode(true);
- }
-
- cfg.setDiscoverySpi(disco);
-
- FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
-
- igfsCfg.setMetaCacheName(META_CACHE_NAME);
- igfsCfg.setDataCacheName(DATA_CACHE_NAME);
- igfsCfg.setName("igfs");
-
- cfg.setFileSystemConfiguration(igfsCfg);
-
- return cfg;
- }
-
- /**
- * @param cacheName Cache name.
- * @return Cache configuration.
- */
- protected CacheConfiguration cacheConfiguration(String cacheName) {
- CacheConfiguration<?,?> cacheCfg = defaultCacheConfiguration();
-
- cacheCfg.setName(cacheName);
-
- cacheCfg.setNearConfiguration(null);
-
- if (META_CACHE_NAME.equals(cacheName))
- cacheCfg.setCacheMode(REPLICATED);
- else {
- cacheCfg.setCacheMode(PARTITIONED);
-
- cacheCfg.setBackups(0);
- cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
- }
-
- cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- cacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- return cacheCfg;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAsyncClientSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAsyncClientSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAsyncClientSelfTest.java
new file mode 100644
index 0000000..e453346
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAsyncClientSelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs;
+
+/**
+ * Tests for DUAL_ASYNC mode and client cache.
+ */
+public class IgfsDualAsyncClientSelfTest extends IgfsDualAsyncSelfTest {
+ /** {@inheritDoc} */
+ @Override protected boolean client() {
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualSyncClientSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualSyncClientSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualSyncClientSelfTest.java
new file mode 100644
index 0000000..bb3da32
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualSyncClientSelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs;
+
+/**
+ * Tests for DUAL_SYNC mode and client cache.
+ */
+public class IgfsDualSyncClientSelfTest extends IgfsDualSyncSelfTest {
+ /** {@inheritDoc} */
+ @Override protected boolean client() {
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsOneClientNodeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsOneClientNodeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsOneClientNodeTest.java
index 25d75a2..3bfd372 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsOneClientNodeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsOneClientNodeTest.java
@@ -101,7 +101,6 @@ public class IgfsOneClientNodeTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
public void testStartIgfs() throws Exception {
-
final IgfsImpl igfs = (IgfsImpl) grid(0).fileSystem("igfs");
GridTestUtils.assertThrows(log, new Callable<Object>() {
@@ -109,7 +108,7 @@ public class IgfsOneClientNodeTest extends GridCommonAbstractTest {
IgfsAbstractSelfTest.create(igfs, new IgfsPath[]{new IgfsPath("/dir")}, null);
return null;
}
- }, IgfsException.class, "Cache server nodes not found.");
+ }, IgfsException.class, "Failed to execute operation because there are no IGFS metadata nodes.");
GridTestUtils.assertThrows(log, new Callable<Object>() {
@Override public Object call() throws Exception {
@@ -119,7 +118,7 @@ public class IgfsOneClientNodeTest extends GridCommonAbstractTest {
return null;
}
- }, IgfsException.class, "Cache server nodes not found.");
+ }, IgfsException.class, "Failed to execute operation because there are no IGFS metadata nodes.");
GridTestUtils.assertThrows(log, new Callable<Object>() {
@Override public Object call() throws Exception {
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryClientSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryClientSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryClientSelfTest.java
new file mode 100644
index 0000000..12fd93c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryClientSelfTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ * Test for IGFS with nodes in client mode (see {@link IgniteConfiguration#setClientMode(boolean)}.
+ */
+public class IgfsPrimaryClientSelfTest extends IgfsPrimarySelfTest {
+ /** {@inheritDoc} */
+ @Override protected boolean client() {
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b96afb2f/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryRelaxedClientSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryRelaxedClientSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryRelaxedClientSelfTest.java
new file mode 100644
index 0000000..64bce8e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryRelaxedClientSelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs;
+
+/**
+ * Tests for PRIMARY mode and relaxed consistency model with client cache.
+ */
+public class IgfsPrimaryRelaxedClientSelfTest extends IgfsPrimaryRelaxedSelfTest {
+ /** {@inheritDoc} */
+ @Override protected boolean client() {
+ return true;
+ }
+}
\ No newline at end of file
[09/17] ignite git commit: IGNITE-3248: Optimized fileIds() usage.
Posted by av...@apache.org.
IGNITE-3248: Optimized fileIds() usage.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e409b67a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e409b67a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e409b67a
Branch: refs/heads/ignite-3216
Commit: e409b67a14ed67025a4bdd5824d4cc3a02f4e920
Parents: b96afb2
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Sun Jun 5 21:04:03 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Sun Jun 5 21:04:03 2016 +0300
----------------------------------------------------------------------
.../internal/processors/igfs/IgfsImpl.java | 18 +++---
.../processors/igfs/IgfsInputStreamImpl.java | 2 +-
.../processors/igfs/IgfsMetaManager.java | 34 ++++++++--
.../meta/IgfsClientMetaIdsForPathCallable.java | 65 ++++++++++++++++++++
.../meta/IgfsClientMetaInfoForPathCallable.java | 63 +++++++++++++++++++
.../processors/igfs/IgfsOneClientNodeTest.java | 2 +-
6 files changed, 167 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e409b67a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index b7b3fac..9087ff0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -660,7 +660,7 @@ public final class IgfsImpl implements IgfsEx {
return new IgfsFileImpl(path, info, data.groupBlockSize());
}
- List<IgniteUuid> fileIds = meta.fileIds(path);
+ List<IgniteUuid> fileIds = meta.idsForPath(path);
IgniteUuid fileId = fileIds.get(fileIds.size() - 1);
@@ -938,7 +938,8 @@ public final class IgfsImpl implements IgfsEx {
}
}
}
- } else if (mode == PRIMARY) {
+ }
+ else if (mode == PRIMARY) {
checkConflictWithPrimary(path);
throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);
@@ -993,7 +994,7 @@ public final class IgfsImpl implements IgfsEx {
return os;
}
- IgfsEntryInfo info = meta.info(meta.fileId(path));
+ IgfsEntryInfo info = meta.infoForPath(path);
if (info == null) {
checkConflictWithPrimary(path);
@@ -1144,7 +1145,7 @@ public final class IgfsImpl implements IgfsEx {
return new IgfsEventAwareOutputStream(path, desc.info(), bufferSize(bufSize), mode, batch);
}
- final List<IgniteUuid> ids = meta.fileIds(path);
+ final List<IgniteUuid> ids = meta.idsForPath(path);
final IgniteUuid id = ids.get(ids.size() - 1);
@@ -1259,8 +1260,7 @@ public final class IgfsImpl implements IgfsEx {
IgfsMode mode = resolveMode(path);
// Check memory first.
- IgniteUuid fileId = meta.fileId(path);
- IgfsEntryInfo info = meta.info(fileId);
+ IgfsEntryInfo info = meta.infoForPath(path);
if (info == null && mode != PRIMARY) {
assert mode == DUAL_SYNC || mode == DUAL_ASYNC;
@@ -1469,7 +1469,7 @@ public final class IgfsImpl implements IgfsEx {
@Nullable private FileDescriptor getFileDescriptor(IgfsPath path) throws IgniteCheckedException {
assert path != null;
- List<IgniteUuid> ids = meta.fileIds(path);
+ List<IgniteUuid> ids = meta.idsForPath(path);
IgfsEntryInfo fileInfo = meta.info(ids.get(ids.size() - 1));
@@ -1645,13 +1645,13 @@ public final class IgfsImpl implements IgfsEx {
switch (mode) {
case PRIMARY:
- info = meta.info(meta.fileId(path));
+ info = meta.infoForPath(path);
break;
case DUAL_SYNC:
case DUAL_ASYNC:
- info = meta.info(meta.fileId(path));
+ info = meta.infoForPath(path);
if (info == null) {
try {
http://git-wip-us.apache.org/repos/asf/ignite/blob/e409b67a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
index 447be93..de7071a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
@@ -319,7 +319,7 @@ public class IgfsInputStreamImpl extends IgfsInputStreamAdapter {
}
// Safety to ensure no orphaned data blocks exist in case file was concurrently deleted.
- if (!meta.exists(fileInfo.id()))
+ if (!meta.exists(fileInfo.id()))
data.delete(fileInfo);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e409b67a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index 3295249..fa748f8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -49,6 +49,8 @@ import org.apache.ignite.internal.processors.cache.GridCacheInternal;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.igfs.client.IgfsClientAbstractCallable;
+import org.apache.ignite.internal.processors.igfs.client.meta.IgfsClientMetaIdsForPathCallable;
+import org.apache.ignite.internal.processors.igfs.client.meta.IgfsClientMetaInfoForPathCallable;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaDirectoryCreateProcessor;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaFileCreateProcessor;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaFileLockProcessor;
@@ -2097,6 +2099,28 @@ public class IgfsMetaManager extends IgfsManager {
}
/**
+ * Get info for the given path.
+ *
+ * @param path Path.
+ * @return Info.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable public IgfsEntryInfo infoForPath(IgfsPath path) throws IgniteCheckedException {
+ return client ? runClientTask(new IgfsClientMetaInfoForPathCallable(cfg.getName(), path)) : info(fileId(path));
+ }
+
+ /**
+ * Get IDs for the given path.
+ *
+ * @param path Path.
+ * @return IDs.
+ * @throws IgniteCheckedException If failed.
+ */
+ public List<IgniteUuid> idsForPath(IgfsPath path) throws IgniteCheckedException {
+ return client ? runClientTask(new IgfsClientMetaIdsForPathCallable(cfg.getName(), path)) : fileIds(path);
+ }
+
+ /**
* Open file in DUAL mode.
*
* @param fs Secondary file system.
@@ -2106,15 +2130,14 @@ public class IgfsMetaManager extends IgfsManager {
* @throws IgniteCheckedException If input stream open has failed.
*/
public IgfsSecondaryInputStreamDescriptor openDual(final IgfsSecondaryFileSystem fs, final IgfsPath path,
- final int bufSize)
- throws IgniteCheckedException {
+ final int bufSize) throws IgniteCheckedException {
if (busyLock.enterBusy()) {
try {
assert fs != null;
assert path != null;
// First, try getting file info without any transactions and synchronization.
- IgfsEntryInfo info = info(fileId(path));
+ IgfsEntryInfo info = infoForPath(path);
if (info != null) {
if (!info.isFile())
@@ -2176,7 +2199,7 @@ public class IgfsMetaManager extends IgfsManager {
if (busyLock.enterBusy()) {
try {
// First, try getting file info without any transactions and synchronization.
- IgfsEntryInfo info = info(fileId(path));
+ IgfsEntryInfo info = infoForPath(path);
if (info != null)
return info;
@@ -2652,10 +2675,9 @@ public class IgfsMetaManager extends IgfsManager {
List<List<IgniteUuid>> pathIds = new ArrayList<>(paths.length);
for (IgfsPath path : paths)
- pathIds.add(fileIds(path));
+ pathIds.add(idsForPath(path));
// Start pessimistic.
-
try (IgniteInternalTx tx = startTx()) {
// Lock the very first existing parents and possibly the leaf as well.
Map<IgfsPath, IgfsPath> pathToParent = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/ignite/blob/e409b67a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/meta/IgfsClientMetaIdsForPathCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/meta/IgfsClientMetaIdsForPathCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/meta/IgfsClientMetaIdsForPathCallable.java
new file mode 100644
index 0000000..7b3d142
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/meta/IgfsClientMetaIdsForPathCallable.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client.meta;
+
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.processors.igfs.IgfsMetaManager;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientAbstractCallable;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.List;
+
+/**
+ * Get entry info for the given path.
+ */
+public class IgfsClientMetaIdsForPathCallable extends IgfsClientAbstractCallable<List<IgniteUuid>> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientMetaIdsForPathCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ */
+ public IgfsClientMetaIdsForPathCallable(@Nullable String igfsName, IgfsPath path) {
+ super(igfsName, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected List<IgniteUuid> call0(IgfsContext ctx) throws Exception {
+ IgfsMetaManager meta = ctx.meta();
+
+ return meta.idsForPath(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientMetaIdsForPathCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e409b67a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/meta/IgfsClientMetaInfoForPathCallable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/meta/IgfsClientMetaInfoForPathCallable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/meta/IgfsClientMetaInfoForPathCallable.java
new file mode 100644
index 0000000..cb31663
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/client/meta/IgfsClientMetaInfoForPathCallable.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs.client.meta;
+
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.igfs.IgfsContext;
+import org.apache.ignite.internal.processors.igfs.IgfsEntryInfo;
+import org.apache.ignite.internal.processors.igfs.IgfsMetaManager;
+import org.apache.ignite.internal.processors.igfs.client.IgfsClientAbstractCallable;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Get entry info for the given path.
+ */
+public class IgfsClientMetaInfoForPathCallable extends IgfsClientAbstractCallable<IgfsEntryInfo> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * Default constructor.
+ */
+ public IgfsClientMetaInfoForPathCallable() {
+ // NO-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param igfsName IGFS name.
+ * @param path Path.
+ */
+ public IgfsClientMetaInfoForPathCallable(@Nullable String igfsName, IgfsPath path) {
+ super(igfsName, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgfsEntryInfo call0(IgfsContext ctx) throws Exception {
+ IgfsMetaManager meta = ctx.meta();
+
+ return meta.infoForPath(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsClientMetaInfoForPathCallable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e409b67a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsOneClientNodeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsOneClientNodeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsOneClientNodeTest.java
index 3bfd372..c5f85bc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsOneClientNodeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsOneClientNodeTest.java
@@ -128,6 +128,6 @@ public class IgfsOneClientNodeTest extends GridCommonAbstractTest {
return null;
}
- }, IgfsException.class, "Cache server nodes not found.");
+ }, IgfsException.class, "Failed to execute operation because there are no IGFS metadata nodes.");
}
}
\ No newline at end of file
[12/17] ignite git commit: Merge branch 'gridgain-7.5.25' of
https://github.com/gridgain/apache-ignite into ignite-gg-11181
Posted by av...@apache.org.
Merge branch 'gridgain-7.5.25' of https://github.com/gridgain/apache-ignite into ignite-gg-11181
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6cb9947a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6cb9947a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6cb9947a
Branch: refs/heads/ignite-3216
Commit: 6cb9947a96f7bfd92266a1b01fb5fef09d2269b2
Parents: a0c6ae5 9e9252c
Author: Sergi Vladykin <se...@gmail.com>
Authored: Mon Jun 6 13:33:06 2016 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Mon Jun 6 13:33:06 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/binary/BinaryUtils.java | 16 ++++
.../processors/cache/CacheObjectContext.java | 3 +
.../GridCacheBinaryObjectsAbstractSelfTest.java | 78 +++++++++++++++++++-
3 files changed, 95 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[16/17] ignite git commit: IGNITE-3258: IGFS: Secondary file system
input stream is opened only when it is really needed.
Posted by av...@apache.org.
IGNITE-3258: IGFS: Secondary file system input stream is opened only when it is really needed.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/52549574
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/52549574
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/52549574
Branch: refs/heads/ignite-3216
Commit: 525495742158de7b077078af372280943b52d87d
Parents: 672b495
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Mon Jun 6 17:50:58 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Mon Jun 6 17:50:58 2016 +0300
----------------------------------------------------------------------
...zySecondaryFileSystemPositionedReadable.java | 77 ++++++++++++++++++++
.../processors/igfs/IgfsMetaManager.java | 18 ++++-
.../processors/igfs/IgfsAbstractSelfTest.java | 3 +
.../processors/igfs/IgfsModesSelfTest.java | 1 +
.../igfs/HadoopFIleSystemFactorySelfTest.java | 1 +
5 files changed, 98 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/52549574/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java
new file mode 100644
index 0000000..0a57c34
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs;
+
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
+import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystemPositionedReadable;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+import java.io.IOException;
+
+/**
+ * Lazy readable entity which is opened on demand.
+ */
+public class IgfsLazySecondaryFileSystemPositionedReadable implements IgfsSecondaryFileSystemPositionedReadable {
+ /** File system. */
+ private final IgfsSecondaryFileSystem fs;
+
+ /** Path. */
+ private final IgfsPath path;
+
+ /** Buffer size. */
+ private final int bufSize;
+
+ /** Target stream. */
+ private IgfsSecondaryFileSystemPositionedReadable target;
+
+ /**
+ * Constructor.
+ *
+ * @param fs File system.
+ * @param path Path.
+ * @param bufSize Buffer size.
+ */
+ public IgfsLazySecondaryFileSystemPositionedReadable(IgfsSecondaryFileSystem fs, IgfsPath path, int bufSize) {
+ assert fs != null;
+ assert path != null;
+
+ this.fs = fs;
+ this.path = path;
+ this.bufSize = bufSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(long pos, byte[] buf, int off, int len) throws IOException {
+ if (target == null)
+ target = fs.open(path, bufSize);
+
+ return target.read(pos, buf, off, len);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IOException {
+ if (target != null)
+ target.close();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgfsLazySecondaryFileSystemPositionedReadable.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/52549574/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index fa748f8..1dd4c53 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -2144,7 +2144,7 @@ public class IgfsMetaManager extends IgfsManager {
throw fsException(new IgfsPathIsDirectoryException("Failed to open file (not a file): " +
path));
- return new IgfsSecondaryInputStreamDescriptor(info, fs.open(path, bufSize));
+ return new IgfsSecondaryInputStreamDescriptor(info, lazySecondaryReader(fs, path, bufSize));
}
// If failed, try synchronize.
@@ -2160,7 +2160,8 @@ public class IgfsMetaManager extends IgfsManager {
throw fsException(new IgfsPathIsDirectoryException("Failed to open file " +
"(not a file): " + path));
- return new IgfsSecondaryInputStreamDescriptor(infos.get(path), fs.open(path, bufSize));
+ return new IgfsSecondaryInputStreamDescriptor(infos.get(path),
+ lazySecondaryReader(fs, path, bufSize));
}
@Override public IgfsSecondaryInputStreamDescriptor onFailure(@Nullable Exception err)
@@ -2184,6 +2185,19 @@ public class IgfsMetaManager extends IgfsManager {
}
/**
+ * Create lazy secondary file system reader.
+ *
+ * @param fs File system.
+ * @param path Path.
+ * @param bufSize Buffer size.
+ * @return Lazy reader.
+ */
+ private static IgfsLazySecondaryFileSystemPositionedReadable lazySecondaryReader(IgfsSecondaryFileSystem fs,
+ IgfsPath path, int bufSize) {
+ return new IgfsLazySecondaryFileSystemPositionedReadable(fs, path, bufSize);
+ }
+
+ /**
* Synchronizes with secondary file system.
*
* @param fs File system.
http://git-wip-us.apache.org/repos/asf/ignite/blob/52549574/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
index 3fb7b91..76a038d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
@@ -1082,6 +1082,9 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
createFile(igfs.asSecondary(), FILE, true, chunk);
checkFileContent(igfs, FILE, chunk);
+
+ // Read again when the whole file is in memory.
+ checkFileContent(igfs, FILE, chunk);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/52549574/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java
index df537bc..1e54f8c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java
@@ -102,6 +102,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest {
igfsCfg.setMetaCacheName("replicated");
igfsCfg.setName("igfs");
igfsCfg.setBlockSize(512 * 1024);
+ igfsCfg.setInitializeDefaultPathModes(true);
if (setNullMode)
igfsCfg.setDefaultMode(null);
http://git-wip-us.apache.org/repos/asf/ignite/blob/52549574/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFIleSystemFactorySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFIleSystemFactorySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFIleSystemFactorySelfTest.java
index 1d02f0f..e4c64ff 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFIleSystemFactorySelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFIleSystemFactorySelfTest.java
@@ -226,6 +226,7 @@ public class HadoopFIleSystemFactorySelfTest extends IgfsCommonAbstractTest {
igfsCfg.setDefaultMode(dfltMode);
igfsCfg.setIpcEndpointConfiguration(endpointCfg);
igfsCfg.setSecondaryFileSystem(secondaryFs);
+ igfsCfg.setInitializeDefaultPathModes(true);
CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
[15/17] ignite git commit: IGNITE-3257: IGFS:
FileSystemConfiguration.DFLT_INIT_DFLT_PATH_MODES has been changed to
"false".
Posted by av...@apache.org.
IGNITE-3257: IGFS: FileSystemConfiguration.DFLT_INIT_DFLT_PATH_MODES has been changed to "false".
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/672b495a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/672b495a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/672b495a
Branch: refs/heads/ignite-3216
Commit: 672b495a43905281cf10d7204b5ee411bc9706cc
Parents: 2c0d95f
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Mon Jun 6 15:50:13 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Mon Jun 6 15:50:13 2016 +0300
----------------------------------------------------------------------
.../org/apache/ignite/configuration/FileSystemConfiguration.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/672b495a/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java
index 518bbf6..625ba95 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java
@@ -83,7 +83,7 @@ public class FileSystemConfiguration {
public static final boolean DFLT_IPC_ENDPOINT_ENABLED = true;
/** Default value of whether to initialize default path modes. */
- public static final boolean DFLT_INIT_DFLT_PATH_MODES = true;
+ public static final boolean DFLT_INIT_DFLT_PATH_MODES = false;
/** Default value of metadata co-location flag. */
public static final boolean DFLT_COLOCATE_META = true;