You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/08/25 08:39:43 UTC
[01/18] ignite git commit: IGNITE-6052 class rename was reverted
Repository: ignite
Updated Branches:
refs/heads/ignite-6149 42d5d63bc -> c13877f2f
IGNITE-6052 class rename was reverted
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f39cae9f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f39cae9f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f39cae9f
Branch: refs/heads/ignite-6149
Commit: f39cae9f9340ba9d5a10f9f3cb9fb3f641f3ac7d
Parents: 2c9057a
Author: Sergey Chugunov <se...@gmail.com>
Authored: Mon Aug 21 20:05:05 2017 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Aug 22 13:42:08 2017 +0300
----------------------------------------------------------------------
.../internal/processors/cluster/GridClusterStateProcessor.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f39cae9f/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 3ad75cc..cdfab21 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -506,7 +506,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
IgniteCompute comp = ((ClusterGroupAdapter)ctx.cluster().get().forServers()).compute();
- IgniteFuture<Void> fut = comp.runAsync(new ChangeGlobalStateComputeRequest(activate));
+ IgniteFuture<Void> fut = comp.runAsync(new ClientChangeGlobalStateComputeRequest(activate));
fut.listen(new CI1<IgniteFuture>() {
@Override public void apply(IgniteFuture fut) {
@@ -884,7 +884,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
/**
*
*/
- private static class ChangeGlobalStateComputeRequest implements IgniteRunnable {
+ private static class ClientChangeGlobalStateComputeRequest implements IgniteRunnable {
/** */
private static final long serialVersionUID = 0L;
@@ -898,7 +898,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
/**
* @param activate New cluster state.
*/
- private ChangeGlobalStateComputeRequest(boolean activate) {
+ private ClientChangeGlobalStateComputeRequest(boolean activate) {
this.activate = activate;
}
[13/18] ignite git commit: IGNITE-6136 Fixed version for demo.
Posted by sb...@apache.org.
IGNITE-6136 Fixed version for demo.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e1bf8d7b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e1bf8d7b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e1bf8d7b
Branch: refs/heads/ignite-6149
Commit: e1bf8d7badee88c405b398ff5abb7e75bfaf2cde
Parents: 46ec148
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Thu Aug 24 14:57:36 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Aug 24 14:57:36 2017 +0700
----------------------------------------------------------------------
.../web-console/frontend/app/modules/agent/AgentManager.service.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e1bf8d7b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
index 67e6704..2b9d945 100644
--- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
+++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
@@ -99,7 +99,7 @@ export default class IgniteAgentManager {
this.connectionSbj = new BehaviorSubject(new ConnectionState(cluster));
- this.clusterVersion = '2.0.0';
+ this.clusterVersion = '2.1.0';
if (!$root.IgniteDemoMode) {
this.connectionSbj.subscribe({
[15/18] ignite git commit: ignite-6175 JVM Crash in Ignite Binary
Objects Simple Mapper Basic suite
Posted by sb...@apache.org.
ignite-6175 JVM Crash in Ignite Binary Objects Simple Mapper Basic suite
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/24306bad
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/24306bad
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/24306bad
Branch: refs/heads/ignite-6149
Commit: 24306badde225ee4b3edec53b2ae8d3e1c0bff8d
Parents: 3ab523c
Author: EdShangGG <es...@gridgain.com>
Authored: Thu Aug 24 19:15:24 2017 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Thu Aug 24 19:15:24 2017 +0300
----------------------------------------------------------------------
.../processors/database/BPlusTreeSelfTest.java | 39 +++++++++++++++-----
1 file changed, 30 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/24306bad/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index 4a32df2..9c0d791 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -54,6 +54,7 @@ import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.GridRandom;
import org.apache.ignite.internal.util.GridStripedLock;
import org.apache.ignite.internal.util.IgniteTree;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
import org.apache.ignite.internal.util.lang.GridCursor;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.SB;
@@ -112,11 +113,11 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
/** */
private static final Collection<Long> rmvdIds = new GridConcurrentHashSet<>();
+ /** Stop. */
+ private final AtomicBoolean stop = new AtomicBoolean();
-// /** {@inheritDoc} */
-// @Override protected long getTestTimeout() {
-// return 25 * 60 * 1000;
-// }
+ /** Future. */
+ private volatile GridCompoundFuture<?, ?> asyncRunFut;
/**
* Check that we do not keep any locks at the moment.
@@ -127,6 +128,8 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
+ stop.set(false);
+
long seed = System.nanoTime();
X.println("Test seed: " + seed + "L; // ");
@@ -156,6 +159,18 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
rnd = null;
try {
+ if (asyncRunFut != null && !asyncRunFut.isDone()) {
+ stop.set(true);
+
+ try {
+ asyncRunFut.cancel();
+ asyncRunFut.get(60000);
+ }
+ catch (Throwable ex) {
+ //Ignore
+ }
+ }
+
if (reuseList != null) {
long size = reuseList.recycledPagesCount();
@@ -1316,7 +1331,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
IgniteInternalFuture<?> fut = multithreadedAsync(new Callable<Object>() {
@Override public Object call() throws Exception {
- for (int i = 0; i < loops; i++) {
+ for (int i = 0; i < loops && !stop.get(); i++) {
final Long x = (long)DataStructure.randomInt(CNT);
final int op = DataStructure.randomInt(4);
@@ -1402,8 +1417,6 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
}
}, Runtime.getRuntime().availableProcessors(), "put-remove");
- final AtomicBoolean stop = new AtomicBoolean();
-
IgniteInternalFuture<?> fut2 = multithreadedAsync(new Callable<Void>() {
@Override public Void call() throws Exception {
while (!stop.get()) {
@@ -1442,14 +1455,22 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
}
}, 4, "find");
+
+ asyncRunFut = new GridCompoundFuture<>();
+
+ asyncRunFut.add((IgniteInternalFuture)fut);
+ asyncRunFut.add((IgniteInternalFuture)fut2);
+ asyncRunFut.add((IgniteInternalFuture)fut3);
+
+ asyncRunFut.markInitialized();
+
try {
fut.get(getTestTimeout(), TimeUnit.MILLISECONDS);
}
finally {
stop.set(true);
- fut2.get();
- fut3.get();
+ asyncRunFut.get();
}
GridCursor<Long> cursor = tree.find(null, null);
[17/18] ignite git commit: ignite-4003 Preliminary changes in
GridNioServer for async connect
Posted by sb...@apache.org.
ignite-4003 Preliminary changes in GridNioServer for async connect
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/79d47f87
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/79d47f87
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/79d47f87
Branch: refs/heads/ignite-6149
Commit: 79d47f87383411319254c9182e09c30158cbd72d
Parents: e1eb1b9
Author: sboikov <sb...@gridgain.com>
Authored: Fri Aug 25 11:07:57 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Aug 25 11:07:57 2017 +0300
----------------------------------------------------------------------
.../GridClientConnectionManagerAdapter.java | 1 -
.../connection/GridClientNioTcpConnection.java | 2 +-
.../internal/util/GridSpinReadWriteLock.java | 2 +-
.../internal/util/nio/GridNioKeyAttachment.java | 33 +++
.../util/nio/GridNioRecoveryDescriptor.java | 3 +-
.../ignite/internal/util/nio/GridNioServer.java | 248 +++++++++++++++----
.../util/nio/GridSelectorNioSessionImpl.java | 28 +--
.../internal/util/nio/ssl/GridNioSslFilter.java | 12 +-
.../communication/tcp/TcpCommunicationSpi.java | 29 ++-
.../IgniteCacheMessageWriteTimeoutTest.java | 4 +-
.../internal/util/nio/GridNioSelfTest.java | 2 +-
.../spi/GridTcpSpiForwardingSelfTest.java | 1 +
.../GridAbstractCommunicationSelfTest.java | 27 +-
...mmunicationSpiConcurrentConnectSelfTest.java | 28 ++-
...dTcpCommunicationSpiRecoveryAckSelfTest.java | 39 ++-
...GridTcpCommunicationSpiRecoverySelfTest.java | 47 +++-
...CommunicationRecoveryAckClosureSelfTest.java | 36 ++-
.../tcp/TcpCommunicationSpiDropNodesTest.java | 3 +-
.../HadoopExternalCommunication.java | 5 +-
19 files changed, 446 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
index e325897..aa06322 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
@@ -183,7 +183,6 @@ public abstract class GridClientConnectionManagerAdapter implements GridClientCo
GridNioSslFilter sslFilter = new GridNioSslFilter(sslCtx, true, ByteOrder.nativeOrder(), gridLog);
sslFilter.directMode(false);
- sslFilter.clientMode(true);
filters = new GridNioFilter[]{codecFilter, sslFilter};
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
index 18ce6e4..f72a009 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
@@ -235,7 +235,7 @@ public class GridClientNioTcpConnection extends GridClientConnection {
meta.put(GridNioSslFilter.HANDSHAKE_FUT_META_KEY, sslHandshakeFut);
}
- ses = (GridNioSession)srv.createSession(ch, meta).get();
+ ses = (GridNioSession)srv.createSession(ch, meta, false, null).get();
if (sslHandshakeFut != null)
sslHandshakeFut.get();
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/main/java/org/apache/ignite/internal/util/GridSpinReadWriteLock.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridSpinReadWriteLock.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridSpinReadWriteLock.java
index 4f23979..8fef887 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridSpinReadWriteLock.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridSpinReadWriteLock.java
@@ -70,7 +70,7 @@ public class GridSpinReadWriteLock {
private int writeLockEntryCnt;
/**
- * Acquires write lock.
+ * Acquires read lock.
*/
@SuppressWarnings("BusyWait")
public void readLock() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioKeyAttachment.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioKeyAttachment.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioKeyAttachment.java
new file mode 100644
index 0000000..0c6ec2e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioKeyAttachment.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.nio;
+
+/**
+ *
+ */
+interface GridNioKeyAttachment {
+ /**
+ * @return {@code True} if session was created.
+ */
+ boolean hasSession();
+
+ /**
+ * @return Session if it was created.
+ */
+ GridSelectorNioSessionImpl session();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
index 7496728..af7b757 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
@@ -31,7 +31,6 @@ import org.jetbrains.annotations.Nullable;
/**
* Recovery information for single node.
*/
-@Deprecated // To be splitted into separate classes for in/out data when do not need maintain backward compatibility.
public class GridNioRecoveryDescriptor {
/** Number of acknowledged messages. */
private long acked;
@@ -260,7 +259,7 @@ public class GridNioRecoveryDescriptor {
/**
* @param node Node.
- * @return {@code True} if node is not null and has the same order as initial remtoe node.
+ * @return {@code True} if node is not null and has the same order as initial remote node.
*/
public boolean nodeAlive(@Nullable ClusterNode node) {
return node != null && node.order() == this.node.order();
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index a04a735..0dd7dd6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -113,6 +113,12 @@ public class GridNioServer<T> {
/** SSL write buf limit. */
private static final int WRITE_BUF_LIMIT = GridNioSessionMetaKey.nextUniqueKey();
+ /** Session future meta key. */
+ public static final int RECOVERY_DESC_META_KEY = GridNioSessionMetaKey.nextUniqueKey();
+
+ /** Selection key meta key. */
+ private static final int WORKER_IDX_META_KEY = GridNioSessionMetaKey.nextUniqueKey();
+
/** */
private static final boolean DISABLE_KEYSET_OPTIMIZATION =
IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_NO_SELECTOR_OPTS);
@@ -465,7 +471,7 @@ public class GridNioServer<T> {
* @return Future for operation.
*/
public GridNioFuture<Boolean> close(GridNioSession ses) {
- assert ses instanceof GridSelectorNioSessionImpl;
+ assert ses instanceof GridSelectorNioSessionImpl : ses;
GridSelectorNioSessionImpl impl = (GridSelectorNioSessionImpl)ses;
@@ -811,17 +817,32 @@ public class GridNioServer<T> {
*
* @param ch Channel to register within the server and create session for.
* @param meta Optional meta for new session.
+ * @param async Async connection.
+ * @param lsnr Listener that should be invoked in NIO thread.
* @return Future to get session.
*/
- public GridNioFuture<GridNioSession> createSession(final SocketChannel ch,
- @Nullable Map<Integer, ?> meta) {
+ public GridNioFuture<GridNioSession> createSession(
+ final SocketChannel ch,
+ @Nullable Map<Integer, Object> meta,
+ boolean async,
+ @Nullable IgniteInClosure<? super IgniteInternalFuture<GridNioSession>> lsnr
+ ) {
try {
if (!closed) {
ch.configureBlocking(false);
NioOperationFuture<GridNioSession> req = new NioOperationFuture<>(ch, false, meta);
- offerBalanced(req);
+ if (async) {
+ assert meta != null;
+
+ req.op = NioOperation.CONNECT;
+ }
+
+ if (lsnr != null)
+ req.listen(lsnr);
+
+ offerBalanced(req, meta);
return req;
}
@@ -835,6 +856,29 @@ public class GridNioServer<T> {
}
/**
+ * @param ch Channel.
+ * @param meta Session meta.
+ */
+ public GridNioFuture<GridNioSession> cancelConnect(final SocketChannel ch, Map<Integer, ?> meta) {
+ if (!closed) {
+ NioOperationFuture<GridNioSession> req = new NioOperationFuture<>(ch, false, meta);
+
+ req.op = NioOperation.CANCEL_CONNECT;
+
+ Integer idx = (Integer)meta.get(WORKER_IDX_META_KEY);
+
+ assert idx != null : meta;
+
+ clientWorkers.get(idx).offer(req);
+
+ return req;
+ }
+ else
+ return new GridNioFinishedFuture<>(
+ new IgniteCheckedException("Failed to cancel connection, server is stopped."));
+ }
+
+ /**
* Gets configurable write timeout for this session. If not set, default value is {@link #DFLT_SES_WRITE_TIMEOUT}.
*
* @return Write timeout in milliseconds.
@@ -920,9 +964,10 @@ public class GridNioServer<T> {
/**
* @param req Request to balance.
+ * @param meta Session metadata.
*/
- private synchronized void offerBalanced(NioOperationFuture req) {
- assert req.operation() == NioOperation.REGISTER : req;
+ private synchronized void offerBalanced(NioOperationFuture req, @Nullable Map<Integer, Object> meta) {
+ assert req.operation() == NioOperation.REGISTER || req.operation() == NioOperation.CONNECT : req;
assert req.socketChannel() != null : req;
int workers = clientWorkers.size();
@@ -960,6 +1005,9 @@ public class GridNioServer<T> {
else
balanceIdx = 0;
+ if (meta != null)
+ meta.put(WORKER_IDX_META_KEY, balanceIdx);
+
clientWorkers.get(balanceIdx).offer(req);
}
@@ -1788,6 +1836,38 @@ public class GridNioServer<T> {
while ((req0 = changeReqs.poll()) != null) {
switch (req0.operation()) {
+ case CONNECT: {
+ NioOperationFuture fut = (NioOperationFuture)req0;
+
+ SocketChannel ch = fut.socketChannel();
+
+ try {
+ ch.register(selector, SelectionKey.OP_CONNECT, fut);
+ }
+ catch (IOException e) {
+ fut.onDone(new IgniteCheckedException("Failed to register channel on selector", e));
+ }
+
+ break;
+ }
+
+ case CANCEL_CONNECT: {
+ NioOperationFuture req = (NioOperationFuture)req0;
+
+ SocketChannel ch = req.socketChannel();
+
+ SelectionKey key = ch.keyFor(selector);
+
+ if (key != null)
+ key.cancel();
+
+ U.closeQuiet(ch);
+
+ req.onDone();
+
+ break;
+ }
+
case REGISTER: {
register((NioOperationFuture)req0);
@@ -1998,8 +2078,12 @@ public class GridNioServer<T> {
log.debug("Closing all connected client sockets.");
// Close all channels registered with selector.
- for (SelectionKey key : selector.keys())
- close((GridSelectorNioSessionImpl)key.attachment(), null);
+ for (SelectionKey key : selector.keys()) {
+ GridNioKeyAttachment attach = (GridNioKeyAttachment)key.attachment();
+
+ if (attach != null && attach.hasSession())
+ close(attach.session(), null);
+ }
if (log.isDebugEnabled())
log.debug("Closing NIO selector.");
@@ -2173,11 +2257,17 @@ public class GridNioServer<T> {
if (!key.isValid())
continue;
- GridSelectorNioSessionImpl ses = (GridSelectorNioSessionImpl)key.attachment();
+ GridNioKeyAttachment attach = (GridNioKeyAttachment)key.attachment();
- assert ses != null;
+ assert attach != null;
try {
+ if (!attach.hasSession() && key.isConnectable()) {
+ processConnect(key);
+
+ continue;
+ }
+
if (key.isReadable())
processRead(key);
@@ -2196,7 +2286,12 @@ public class GridNioServer<T> {
// No-op.
}
- U.warn(log, "Failed to process selector key (will close): " + ses, e);
+ GridSelectorNioSessionImpl ses = attach.session();
+
+ if (!closed)
+ U.error(log, "Failed to process selector key [ses=" + ses + ']', e);
+ else if (log.isDebugEnabled())
+ log.debug("Failed to process selector key [ses=" + ses + ", err=" + e + ']');
close(ses, new GridNioException(e));
}
@@ -2225,11 +2320,17 @@ public class GridNioServer<T> {
if (!key.isValid())
continue;
- GridSelectorNioSessionImpl ses = (GridSelectorNioSessionImpl)key.attachment();
+ GridNioKeyAttachment attach = (GridNioKeyAttachment)key.attachment();
- assert ses != null;
+ assert attach != null;
try {
+ if (!attach.hasSession() && key.isConnectable()) {
+ processConnect(key);
+
+ continue;
+ }
+
if (key.isReadable())
processRead(key);
@@ -2248,10 +2349,12 @@ public class GridNioServer<T> {
// No-op.
}
- if (!closed)
- U.warn(log, "Failed to process selector key (will close): " + ses, e);
+ GridSelectorNioSessionImpl ses = attach.session();
- close(ses, new GridNioException(e));
+ if (!closed)
+ U.error(log, "Failed to process selector key [ses=" + ses + ']', e);
+ else if (log.isDebugEnabled())
+ log.debug("Failed to process selector key [ses=" + ses + ", err=" + e + ']');
}
}
}
@@ -2265,7 +2368,12 @@ public class GridNioServer<T> {
long now = U.currentTimeMillis();
for (SelectionKey key : keys) {
- GridSelectorNioSessionImpl ses = (GridSelectorNioSessionImpl)key.attachment();
+ GridNioKeyAttachment attach = (GridNioKeyAttachment)key.attachment();
+
+ if (attach == null || !attach.hasSession())
+ continue;
+
+ GridSelectorNioSessionImpl ses = attach.session();
try {
long writeTimeout0 = writeTimeout;
@@ -2303,12 +2411,12 @@ public class GridNioServer<T> {
/**
* Registers given socket channel to the selector, creates a session and notifies the listener.
*
- * @param req Registration request.
+ * @param fut Registration future.
*/
- private void register(NioOperationFuture<GridNioSession> req) {
- assert req != null;
+ private void register(NioOperationFuture<GridNioSession> fut) {
+ assert fut != null;
- SocketChannel sockCh = req.socketChannel();
+ SocketChannel sockCh = fut.socketChannel();
assert sockCh != null;
@@ -2334,24 +2442,53 @@ public class GridNioServer<T> {
filterChain,
(InetSocketAddress)sockCh.getLocalAddress(),
(InetSocketAddress)sockCh.getRemoteAddress(),
- req.accepted(),
+ fut.accepted(),
sndQueueLimit,
writeBuf,
readBuf);
- Map<Integer, ?> meta = req.meta();
+ Map<Integer, ?> meta = fut.meta();
if (meta != null) {
for (Entry<Integer, ?> e : meta.entrySet())
ses.addMeta(e.getKey(), e.getValue());
+
+ if (!ses.accepted()) {
+ GridNioRecoveryDescriptor desc =
+ (GridNioRecoveryDescriptor)meta.get(RECOVERY_DESC_META_KEY);
+
+ if (desc != null) {
+ ses.outRecoveryDescriptor(desc);
+
+ if (!desc.pairedConnections())
+ ses.inRecoveryDescriptor(desc);
+ }
+ }
}
- SelectionKey key = sockCh.register(selector, SelectionKey.OP_READ, ses);
+ SelectionKey key;
- ses.key(key);
+ if (!sockCh.isRegistered()) {
+ assert fut.op == NioOperation.REGISTER : fut.op;
+
+ key = sockCh.register(selector, SelectionKey.OP_READ, ses);
+
+ ses.key(key);
- if (!ses.accepted())
resend(ses);
+ }
+ else {
+ assert fut.op == NioOperation.CONNECT : fut.op;
+
+ key = sockCh.keyFor(selector);
+
+ key.attach(ses);
+
+ key.interestOps(key.interestOps() & (~SelectionKey.OP_CONNECT));
+ key.interestOps(key.interestOps() | SelectionKey.OP_READ);
+
+ ses.key(key);
+ }
sessions.add(ses);
workerSessions.add(ses);
@@ -2359,12 +2496,12 @@ public class GridNioServer<T> {
try {
filterChain.onSessionOpened(ses);
- req.onDone(ses);
+ fut.onDone(ses);
}
catch (IgniteCheckedException e) {
close(ses, e);
- req.onDone(e);
+ fut.onDone(e);
}
if (closed)
@@ -2486,6 +2623,31 @@ public class GridNioServer<T> {
}
/**
+ * @param key Key.
+ * @throws IOException If failed.
+ */
+ @SuppressWarnings("unchecked")
+ private void processConnect(SelectionKey key) throws IOException {
+ SocketChannel ch = (SocketChannel)key.channel();
+
+ NioOperationFuture<GridNioSession> sesFut = (NioOperationFuture<GridNioSession>)key.attachment();
+
+ assert sesFut != null;
+
+ try {
+ if (ch.finishConnect())
+ register(sesFut);
+ }
+ catch (IOException e) {
+ U.closeQuiet(ch);
+
+ sesFut.onDone(new GridNioException("Failed to connect to node", e));
+
+ throw e;
+ }
+ }
+
+ /**
* Processes read-available event on the key.
*
* @param key Key that is ready to be read.
@@ -2690,7 +2852,7 @@ public class GridNioServer<T> {
if (log.isDebugEnabled())
log.debug("Accepted new client connection: " + sockCh.socket().getRemoteSocketAddress());
- addRegistrationReq(sockCh);
+ addRegistrationRequest(sockCh);
}
}
}
@@ -2701,15 +2863,21 @@ public class GridNioServer<T> {
*
* @param sockCh Socket channel to be registered on one of the selectors.
*/
- private void addRegistrationReq(SocketChannel sockCh) {
- offerBalanced(new NioOperationFuture(sockCh));
+ private void addRegistrationRequest(SocketChannel sockCh) {
+ offerBalanced(new NioOperationFuture<>(sockCh, true, null), null);
}
}
/**
* Asynchronous operation that may be requested on selector.
*/
- enum NioOperation {
+ private enum NioOperation {
+ /** Register connect key selection. */
+ CONNECT,
+
+ /** Cancel connect. */
+ CANCEL_CONNECT,
+
/** Register read key selection. */
REGISTER,
@@ -2914,7 +3082,7 @@ public class GridNioServer<T> {
* Class for requesting write and session close operations.
*/
private static class NioOperationFuture<R> extends GridNioFutureImpl<R> implements SessionWriteRequest,
- SessionChangeRequest {
+ SessionChangeRequest, GridNioKeyAttachment {
/** Socket channel in register request. */
@GridToStringExclude
private SocketChannel sockCh;
@@ -2942,15 +3110,6 @@ public class GridNioServer<T> {
private boolean skipRecovery;
/**
- * Creates registration request for a given socket channel.
- *
- * @param sockCh Socket channel to register on selector.
- */
- NioOperationFuture(SocketChannel sockCh) {
- this(sockCh, true, null);
- }
-
- /**
* @param sockCh Socket channel.
* @param accepted {@code True} if socket has been accepted.
* @param meta Optional meta.
@@ -3038,6 +3197,11 @@ public class GridNioServer<T> {
}
/** {@inheritDoc} */
+ @Override public boolean hasSession() {
+ return ses != null;
+ }
+
+ /** {@inheritDoc} */
public NioOperation operation() {
return op;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
index 66f9176..71a3b8d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
@@ -37,7 +37,7 @@ import org.jsr166.ConcurrentLinkedDeque8;
* Note that this implementation requires non-null values for local and remote
* socket addresses.
*/
-class GridSelectorNioSessionImpl extends GridNioSessionImpl {
+class GridSelectorNioSessionImpl extends GridNioSessionImpl implements GridNioKeyAttachment {
/** Pending write requests. */
private final ConcurrentLinkedDeque8<SessionWriteRequest> queue = new ConcurrentLinkedDeque8<>();
@@ -129,6 +129,16 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
}
}
+ /** {@inheritDoc} */
+ @Override public boolean hasSession() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public GridSelectorNioSessionImpl session() {
+ return this;
+ }
+
/**
* @return Worker.
*/
@@ -385,22 +395,6 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
return inRecovery;
}
- /** {@inheritDoc} */
- @Override public <T> T addMeta(int key, @Nullable T val) {
- if (!accepted() && val instanceof GridNioRecoveryDescriptor) {
- outRecovery = (GridNioRecoveryDescriptor)val;
-
- if (!outRecovery.pairedConnections())
- inRecovery = outRecovery;
-
- outRecovery.onConnected();
-
- return null;
- }
- else
- return super.addMeta(key, val);
- }
-
/**
*
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
index b4bd34a..f8a0dce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
@@ -69,9 +69,6 @@ public class GridNioSslFilter extends GridNioFilterAdapter {
/** Allocate direct buffer or heap buffer. */
private boolean directBuf;
- /** Whether SSLEngine should use client mode. */
- private boolean clientMode;
-
/** Whether direct mode is used. */
private boolean directMode;
@@ -93,13 +90,6 @@ public class GridNioSslFilter extends GridNioFilterAdapter {
}
/**
- * @param clientMode Flag indicating whether SSLEngine should use client mode..
- */
- public void clientMode(boolean clientMode) {
- this.clientMode = clientMode;
- }
-
- /**
*
* @param directMode Flag indicating whether direct mode is used.
*/
@@ -164,6 +154,8 @@ public class GridNioSslFilter extends GridNioFilterAdapter {
if (sslMeta == null) {
engine = sslCtx.createSSLEngine();
+ boolean clientMode = !ses.accepted();
+
engine.setUseClientMode(clientMode);
if (!clientMode) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index bab9cfa..7a54666 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2980,14 +2980,11 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
/**
- * Establish TCP connection to remote node and returns client.
- *
- * @param node Remote node.
- * @param connIdx Connection index.
- * @return Client.
- * @throws IgniteCheckedException If failed.
+ * @param node Node.
+ * @return Node addresses.
+ * @throws IgniteCheckedException If node does not have addresses.
*/
- protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
+ private LinkedHashSet<InetSocketAddress> nodeAddresses(ClusterNode node) throws IgniteCheckedException {
Collection<String> rmtAddrs0 = node.attribute(createSpiAttributeName(ATTR_ADDRS));
Collection<String> rmtHostNames0 = node.attribute(createSpiAttributeName(ATTR_HOST_NAMES));
Integer boundPort = node.attribute(createSpiAttributeName(ATTR_PORT));
@@ -3052,6 +3049,20 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
log.debug("Addresses to connect for node [rmtNode=" + node.id() + ", addrs=" + addrs.toString() + ']');
}
+ return addrs;
+ }
+
+ /**
+ * Establish TCP connection to remote node and returns client.
+ *
+ * @param node Remote node.
+ * @param connIdx Connection index.
+ * @return Client.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
+ LinkedHashSet<InetSocketAddress> addrs = nodeAddresses(node);
+
boolean conn = false;
GridCommunicationClient client = null;
IgniteCheckedException errs = null;
@@ -3162,10 +3173,10 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
if (recoveryDesc != null) {
recoveryDesc.onHandshake(rcvCnt);
- meta.put(-1, recoveryDesc);
+ meta.put(GridNioServer.RECOVERY_DESC_META_KEY, recoveryDesc);
}
- GridNioSession ses = nioSrvr.createSession(ch, meta).get();
+ GridNioSession ses = nioSrvr.createSession(ch, meta, false, null).get();
client = new GridTcpNioCommunicationClient(connIdx, ses, log);
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
index b3b5d1a..3ba319b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
@@ -50,8 +50,8 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
// Try provoke connection close on socket writeTimeout.
commSpi.setSharedMemoryPort(-1);
commSpi.setMessageQueueLimit(10);
- commSpi.setSocketReceiveBuffer(40);
- commSpi.setSocketSendBuffer(40);
+ commSpi.setSocketReceiveBuffer(64);
+ commSpi.setSocketSendBuffer(64);
commSpi.setSocketWriteTimeout(100);
commSpi.setUnacknowledgedMessagesBufferSize(1000);
commSpi.setConnectTimeout(10_000);
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/test/java/org/apache/ignite/internal/util/nio/GridNioSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/nio/GridNioSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/nio/GridNioSelfTest.java
index 4dbb7ce..e623467 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/nio/GridNioSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/nio/GridNioSelfTest.java
@@ -678,7 +678,7 @@ public class GridNioSelfTest extends GridCommonAbstractTest {
try {
SocketChannel ch = SocketChannel.open(new InetSocketAddress(U.getLocalHost(), srvr2.port()));
- GridNioFuture<GridNioSession> fut = srvr1.createSession(ch, null);
+ GridNioFuture<GridNioSession> fut = srvr1.createSession(ch, null, false, null);
ses = fut.get();
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java
index 1e25003..5d8e316 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java
@@ -30,6 +30,7 @@ import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.AddressResolver;
import org.apache.ignite.configuration.BasicAddressResolver;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.util.nio.GridCommunicationClient;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.lang.IgniteCallable;
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/test/java/org/apache/ignite/spi/communication/GridAbstractCommunicationSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridAbstractCommunicationSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridAbstractCommunicationSelfTest.java
index 88276c2..e51dac8 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridAbstractCommunicationSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridAbstractCommunicationSelfTest.java
@@ -30,6 +30,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.managers.communication.GridIoMessageFactory;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.util.typedef.CO;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -39,6 +40,7 @@ import org.apache.ignite.spi.IgniteSpiAdapter;
import org.apache.ignite.testframework.GridSpiTestContext;
import org.apache.ignite.testframework.GridTestNode;
import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
import org.apache.ignite.testframework.junits.IgniteMock;
import org.apache.ignite.testframework.junits.IgniteTestResources;
import org.apache.ignite.testframework.junits.spi.GridSpiAbstractTest;
@@ -70,6 +72,9 @@ public abstract class GridAbstractCommunicationSelfTest<T extends CommunicationS
private static final Object mux = new Object();
/** */
+ private static GridTimeoutProcessor timeoutProcessor;
+
+ /** */
protected boolean useSsl = false;
/**
@@ -289,6 +294,12 @@ public abstract class GridAbstractCommunicationSelfTest<T extends CommunicationS
Map<ClusterNode, GridSpiTestContext> ctxs = new HashMap<>();
+ timeoutProcessor = new GridTimeoutProcessor(new GridTestKernalContext(log));
+
+ timeoutProcessor.start();
+
+ timeoutProcessor.onKernalStart(true);
+
for (int i = 0; i < getSpiCount(); i++) {
CommunicationSpi<Message> spi = getSpi(i);
@@ -298,18 +309,20 @@ public abstract class GridAbstractCommunicationSelfTest<T extends CommunicationS
GridTestNode node = new GridTestNode(rsrcs.getNodeId());
- node.order(i);
-
GridSpiTestContext ctx = initSpiContext();
ctx.setLocalNode(node);
+ ctx.timeoutProcessor(timeoutProcessor);
+
info(">>> Initialized context: nodeId=" + ctx.localNode().id());
spiRsrcs.add(rsrcs);
rsrcs.inject(spi);
+ GridTestUtils.setFieldValue(spi, IgniteSpiAdapter.class, "igniteInstanceName", "grid-" + i);
+
if (useSsl) {
IgniteMock ignite = GridTestUtils.getFieldValue(spi, IgniteSpiAdapter.class, "ignite");
@@ -324,6 +337,8 @@ public abstract class GridAbstractCommunicationSelfTest<T extends CommunicationS
node.setAttributes(spi.getNodeAttributes());
node.setAttribute(ATTR_MACS, F.concat(U.allLocalMACs(), ", "));
+ node.order(i + 1);
+
nodes.add(node);
spi.spiStart(getTestIgniteInstanceName() + (i + 1));
@@ -346,6 +361,14 @@ public abstract class GridAbstractCommunicationSelfTest<T extends CommunicationS
/** {@inheritDoc} */
@Override protected void afterTestsStopped() throws Exception {
+ if (timeoutProcessor != null) {
+ timeoutProcessor.onKernalStop(true);
+
+ timeoutProcessor.stop(true);
+
+ timeoutProcessor = null;
+ }
+
for (CommunicationSpi<Message> spi : spis.values()) {
spi.onContextDestroyed();
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java
index 78bf869..ce96c55 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java
@@ -36,7 +36,9 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteNodeAttributes;
import org.apache.ignite.internal.managers.communication.GridIoMessageFactory;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.nio.GridCommunicationClient;
import org.apache.ignite.internal.util.nio.GridNioServer;
@@ -51,6 +53,7 @@ import org.apache.ignite.spi.communication.GridTestMessage;
import org.apache.ignite.testframework.GridSpiTestContext;
import org.apache.ignite.testframework.GridTestNode;
import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
import org.apache.ignite.testframework.junits.IgniteMock;
import org.apache.ignite.testframework.junits.IgniteTestResources;
import org.apache.ignite.testframework.junits.spi.GridSpiAbstractTest;
@@ -79,6 +82,9 @@ public class GridTcpCommunicationSpiConcurrentConnectSelfTest<T extends Communic
protected static final List<ClusterNode> nodes = new ArrayList<>();
/** */
+ private static GridTimeoutProcessor timeoutProcessor;
+
+ /** */
private static int port = 60_000;
/** Use ssl. */
@@ -407,27 +413,37 @@ public class GridTcpCommunicationSpiConcurrentConnectSelfTest<T extends Communic
Map<ClusterNode, GridSpiTestContext> ctxs = new HashMap<>();
+ timeoutProcessor = new GridTimeoutProcessor(new GridTestKernalContext(log));
+
+ timeoutProcessor.start();
+
+ timeoutProcessor.onKernalStart(true);
+
for (int i = 0; i < SPI_CNT; i++) {
CommunicationSpi<Message> spi = createSpi();
- GridTestUtils.setFieldValue(spi, IgniteSpiAdapter.class, "igniteInstanceName", "grid-" + i);
-
IgniteTestResources rsrcs = new IgniteTestResources();
GridTestNode node = new GridTestNode(rsrcs.getNodeId());
+ node.setAttribute(IgniteNodeAttributes.ATTR_CLIENT_MODE, false);
+
node.order(i + 1);
GridSpiTestContext ctx = initSpiContext();
ctx.setLocalNode(node);
+ ctx.timeoutProcessor(timeoutProcessor);
+
info(">>> Initialized context: nodeId=" + ctx.localNode().id());
spiRsrcs.add(rsrcs);
rsrcs.inject(spi);
+ GridTestUtils.setFieldValue(spi, IgniteSpiAdapter.class, "igniteInstanceName", "grid-" + i);
+
if (useSsl) {
IgniteMock ignite = GridTestUtils.getFieldValue(spi, IgniteSpiAdapter.class, "ignite");
@@ -494,6 +510,14 @@ public class GridTcpCommunicationSpiConcurrentConnectSelfTest<T extends Communic
* @throws Exception If failed.
*/
private void stopSpis() throws Exception {
+ if (timeoutProcessor != null) {
+ timeoutProcessor.onKernalStop(true);
+
+ timeoutProcessor.stop(true);
+
+ timeoutProcessor = null;
+ }
+
for (CommunicationSpi<Message> spi : spis) {
spi.onContextDestroyed();
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java
index feaae11..1467c29 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java
@@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.managers.communication.GridIoMessageFactory;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.nio.GridNioRecoveryDescriptor;
import org.apache.ignite.internal.util.nio.GridNioServer;
@@ -44,6 +45,7 @@ import org.apache.ignite.spi.communication.GridTestMessage;
import org.apache.ignite.testframework.GridSpiTestContext;
import org.apache.ignite.testframework.GridTestNode;
import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
import org.apache.ignite.testframework.junits.IgniteTestResources;
import org.apache.ignite.testframework.junits.spi.GridSpiAbstractTest;
import org.apache.ignite.testframework.junits.spi.GridSpiTest;
@@ -64,6 +66,9 @@ public class GridTcpCommunicationSpiRecoveryAckSelfTest<T extends CommunicationS
protected static final List<ClusterNode> nodes = new ArrayList<>();
/** */
+ private static GridTimeoutProcessor timeoutProcessor;
+
+ /** */
private static final int SPI_CNT = 2;
/**
@@ -159,6 +164,8 @@ public class GridTcpCommunicationSpiRecoveryAckSelfTest<T extends CommunicationS
spi1.sendMessage(node0, new GridTestMessage(node1.id(), ++msgId, 0));
}
+ U.sleep(500);
+
expMsgs += msgPerIter;
final long totAcked0 = totAcked;
@@ -166,9 +173,13 @@ public class GridTcpCommunicationSpiRecoveryAckSelfTest<T extends CommunicationS
for (TcpCommunicationSpi spi : spis) {
GridNioServer srv = U.field(spi, "nioSrvr");
- Collection<? extends GridNioSession> sessions = GridTestUtils.getFieldValue(srv, "sessions");
+ final Collection<? extends GridNioSession> sessions = GridTestUtils.getFieldValue(srv, "sessions");
- assertFalse(sessions.isEmpty());
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return !sessions.isEmpty();
+ }
+ }, 5_000);
boolean found = false;
@@ -282,7 +293,7 @@ public class GridTcpCommunicationSpiRecoveryAckSelfTest<T extends CommunicationS
int sentMsgs = 1;
- for (int i = 0; i < 150; i++) {
+ for (int i = 0; i < 1280; i++) {
try {
spi0.sendMessage(node1, new GridTestMessage(node0.id(), ++msgId, 0));
@@ -379,11 +390,15 @@ public class GridTcpCommunicationSpiRecoveryAckSelfTest<T extends CommunicationS
Map<ClusterNode, GridSpiTestContext> ctxs = new HashMap<>();
+ timeoutProcessor = new GridTimeoutProcessor(new GridTestKernalContext(log));
+
+ timeoutProcessor.start();
+
+ timeoutProcessor.onKernalStart(true);
+
for (int i = 0; i < SPI_CNT; i++) {
TcpCommunicationSpi spi = getSpi(ackCnt, idleTimeout, queueLimit);
- GridTestUtils.setFieldValue(spi, IgniteSpiAdapter.class, "igniteInstanceName", "grid-" + i);
-
IgniteTestResources rsrcs = new IgniteTestResources();
GridTestNode node = new GridTestNode(rsrcs.getNodeId());
@@ -392,14 +407,20 @@ public class GridTcpCommunicationSpiRecoveryAckSelfTest<T extends CommunicationS
ctx.setLocalNode(node);
+ ctx.timeoutProcessor(timeoutProcessor);
+
spiRsrcs.add(rsrcs);
rsrcs.inject(spi);
+ GridTestUtils.setFieldValue(spi, IgniteSpiAdapter.class, "igniteInstanceName", "grid-" + i);
+
spi.setListener(new TestListener());
node.setAttributes(spi.getNodeAttributes());
+ node.order(i);
+
nodes.add(node);
spi.spiStart(getTestIgniteInstanceName() + (i + 1));
@@ -455,6 +476,14 @@ public class GridTcpCommunicationSpiRecoveryAckSelfTest<T extends CommunicationS
* @throws Exception If failed.
*/
private void stopSpis() throws Exception {
+ if (timeoutProcessor != null) {
+ timeoutProcessor.onKernalStop(true);
+
+ timeoutProcessor.stop(true);
+
+ timeoutProcessor = null;
+ }
+
for (CommunicationSpi<Message> spi : spis) {
spi.onContextDestroyed();
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java
index 2a043ee..c722577 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java
@@ -32,6 +32,7 @@ import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.managers.communication.GridIoMessageFactory;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.nio.GridNioServer;
import org.apache.ignite.internal.util.nio.GridNioSession;
@@ -47,6 +48,7 @@ import org.apache.ignite.spi.communication.GridTestMessage;
import org.apache.ignite.testframework.GridSpiTestContext;
import org.apache.ignite.testframework.GridTestNode;
import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
import org.apache.ignite.testframework.junits.IgniteMock;
import org.apache.ignite.testframework.junits.IgniteTestResources;
import org.apache.ignite.testframework.junits.spi.GridSpiAbstractTest;
@@ -80,6 +82,9 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
/** Use ssl. */
protected boolean useSsl;
+ /** */
+ private static GridTimeoutProcessor timeoutProcessor;
+
/**
*
*/
@@ -186,7 +191,7 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
* @return Timeout.
*/
protected long awaitForSocketWriteTimeout() {
- return 8000;
+ return 20_000;
}
/**
@@ -298,6 +303,12 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
// Send message to establish connection.
spi0.sendMessage(node1, new GridTestMessage(node0.id(), msgId.incrementAndGet(), 0));
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return lsnr1.rcvCnt.get() >= 1;
+ }
+ }, 1000);
+
final AtomicInteger sentCnt = new AtomicInteger(1);
int errCnt = 0;
@@ -413,6 +424,12 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
// Send message to establish connection.
spi0.sendMessage(node1, new GridTestMessage(node0.id(), msgId.incrementAndGet(), 0));
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return lsnr1.rcvCnt.get() >= 1;
+ }
+ }, 1000);
+
expCnt1.incrementAndGet();
int errCnt = 0;
@@ -451,7 +468,7 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
ses1.resumeReads().get();
}
catch (IgniteCheckedException ignore) {
- // Can fail is ses1 was closed.
+ // Can fail if ses1 was closed.
}
// Wait when session is closed, then try to open new connection from node1.
@@ -534,6 +551,12 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
// Send message to establish connection.
spi0.sendMessage(node1, new GridTestMessage(node0.id(), msgId.incrementAndGet(), 0));
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return lsnr1.rcvCnt.get() >= 1;
+ }
+ }, 1000);
+
final AtomicInteger sentCnt = new AtomicInteger(1);
int errCnt = 0;
@@ -686,11 +709,15 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
Map<ClusterNode, GridSpiTestContext> ctxs = new HashMap<>();
+ timeoutProcessor = new GridTimeoutProcessor(new GridTestKernalContext(log));
+
+ timeoutProcessor.start();
+
+ timeoutProcessor.onKernalStart(true);
+
for (int i = 0; i < SPI_CNT; i++) {
TcpCommunicationSpi spi = getSpi(i);
- GridTestUtils.setFieldValue(spi, IgniteSpiAdapter.class, "igniteInstanceName", "grid-" + i);
-
IgniteTestResources rsrcs = new IgniteTestResources();
GridTestNode node = new GridTestNode(rsrcs.getNodeId());
@@ -701,10 +728,14 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
ctx.setLocalNode(node);
+ ctx.timeoutProcessor(timeoutProcessor);
+
spiRsrcs.add(rsrcs);
rsrcs.inject(spi);
+ GridTestUtils.setFieldValue(spi, IgniteSpiAdapter.class, "igniteInstanceName", "grid-" + i);
+
if (useSsl) {
IgniteMock ignite = GridTestUtils.getFieldValue(spi, IgniteSpiAdapter.class, "ignite");
@@ -770,6 +801,14 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
* @throws Exception If failed.
*/
private void stopSpis() throws Exception {
+ if (timeoutProcessor != null) {
+ timeoutProcessor.onKernalStop(true);
+
+ timeoutProcessor.stop(true);
+
+ timeoutProcessor = null;
+ }
+
for (CommunicationSpi<Message> spi : spis) {
spi.onContextDestroyed();
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java
index 3f58055..d1689bd 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java
@@ -29,6 +29,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.managers.communication.GridIoMessageFactory;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.nio.GridNioRecoveryDescriptor;
import org.apache.ignite.internal.util.nio.GridNioServer;
@@ -47,6 +48,7 @@ import org.apache.ignite.spi.communication.GridTestMessage;
import org.apache.ignite.testframework.GridSpiTestContext;
import org.apache.ignite.testframework.GridTestNode;
import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
import org.apache.ignite.testframework.junits.IgniteTestResources;
import org.apache.ignite.testframework.junits.spi.GridSpiAbstractTest;
import org.apache.ignite.testframework.junits.spi.GridSpiTest;
@@ -70,6 +72,9 @@ public class IgniteTcpCommunicationRecoveryAckClosureSelfTest<T extends Communic
/** */
private static final int SPI_CNT = 2;
+ /** */
+ private static GridTimeoutProcessor timeoutProcessor;
+
/**
*
*/
@@ -98,8 +103,6 @@ public class IgniteTcpCommunicationRecoveryAckClosureSelfTest<T extends Communic
/** {@inheritDoc} */
@Override public void onMessage(UUID nodeId, Message msg, IgniteRunnable msgC) {
- info("Test listener received message: " + msg);
-
assertTrue("Unexpected message: " + msg, msg instanceof GridTestMessage);
GridTestMessage msg0 = (GridTestMessage)msg;
@@ -171,6 +174,17 @@ public class IgniteTcpCommunicationRecoveryAckClosureSelfTest<T extends Communic
spi0.sendMessage(node1, new GridTestMessage(node0.id(), ++msgId, 0), ackC);
spi1.sendMessage(node0, new GridTestMessage(node1.id(), ++msgId, 0), ackC);
+
+ if (j == 0) {
+ final TestListener lsnr0 = (TestListener)spi0.getListener();
+ final TestListener lsnr1 = (TestListener)spi1.getListener();
+
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return lsnr0.rcvCnt.get() >= 1 && lsnr1.rcvCnt.get() >= 1;
+ }
+ }, 1000);
+ }
}
expMsgs += msgPerIter;
@@ -415,6 +429,12 @@ public class IgniteTcpCommunicationRecoveryAckClosureSelfTest<T extends Communic
Map<ClusterNode, GridSpiTestContext> ctxs = new HashMap<>();
+ timeoutProcessor = new GridTimeoutProcessor(new GridTestKernalContext(log));
+
+ timeoutProcessor.start();
+
+ timeoutProcessor.onKernalStart(true);
+
for (int i = 0; i < SPI_CNT; i++) {
TcpCommunicationSpi spi = getSpi(ackCnt, idleTimeout, queueLimit);
@@ -428,6 +448,8 @@ public class IgniteTcpCommunicationRecoveryAckClosureSelfTest<T extends Communic
ctx.setLocalNode(node);
+ ctx.timeoutProcessor(timeoutProcessor);
+
spiRsrcs.add(rsrcs);
rsrcs.inject(spi);
@@ -436,6 +458,8 @@ public class IgniteTcpCommunicationRecoveryAckClosureSelfTest<T extends Communic
node.setAttributes(spi.getNodeAttributes());
+ node.order(i);
+
nodes.add(node);
spi.spiStart(getTestIgniteInstanceName() + (i + 1));
@@ -491,6 +515,14 @@ public class IgniteTcpCommunicationRecoveryAckClosureSelfTest<T extends Communic
* @throws Exception If failed.
*/
private void stopSpis() throws Exception {
+ if (timeoutProcessor != null) {
+ timeoutProcessor.onKernalStop(true);
+
+ timeoutProcessor.stop(true);
+
+ timeoutProcessor = null;
+ }
+
for (CommunicationSpi<Message> spi : spis) {
spi.onContextDestroyed();
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
index e215a34..842f283 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
@@ -202,8 +202,7 @@ public class TcpCommunicationSpiDropNodesTest extends GridCommonAbstractTest {
final CountDownLatch latch = new CountDownLatch(1);
grid(0).events().localListen(new IgnitePredicate<Event>() {
- @Override
- public boolean apply(Event event) {
+ @Override public boolean apply(Event evt) {
latch.countDown();
return true;
http://git-wip-us.apache.org/repos/asf/ignite/blob/79d47f87/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
index 8a20eec..a241a04 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
@@ -1004,7 +1004,10 @@ public class HadoopExternalCommunication {
HandshakeFinish fin = new HandshakeFinish();
- GridNioSession ses = nioSrvr.createSession(ch, F.asMap(HANDSHAKE_FINISH_META, fin)).get();
+ GridNioFuture<GridNioSession> sesFut =
+ nioSrvr.createSession(ch, F.<Integer, Object>asMap(HANDSHAKE_FINISH_META, fin), false, null);
+
+ GridNioSession ses = sesFut.get();
client = new HadoopTcpNioCommunicationClient(ses);
[02/18] ignite git commit: IGNITE-6154 also fixed check for WAL record
Posted by sb...@apache.org.
IGNITE-6154 also fixed check for WAL record
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/fa42218e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/fa42218e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/fa42218e
Branch: refs/heads/ignite-6149
Commit: fa42218e52d07f8869ed7da6874525b2722f3fcf
Parents: 474ecb8
Author: Ivan Rakov <iv...@gmail.com>
Authored: Tue Aug 22 17:03:42 2017 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Aug 22 20:49:07 2017 +0300
----------------------------------------------------------------------
.../cache/persistence/GridCacheDatabaseSharedManager.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/fa42218e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 0f0fb88..9f2e4a0 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -2129,6 +2129,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
tracker.onLockWaitStart();
+ boolean hasPages;
+
checkpointLock.writeLock().lock();
try {
@@ -2193,7 +2195,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
cpPagesTuple = beginAllCheckpoints();
- if (!F.isEmpty(cpPagesTuple.get1())) {
+ hasPages = hasPageForWrite(cpPagesTuple.get1());
+
+ if (hasPages) {
// No page updates for this checkpoint are allowed from now on.
cpPtr = cctx.wal().log(cpRec);
@@ -2209,7 +2213,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
curr.cpBeginFut.onDone();
- if (hasPageForWrite(cpPagesTuple.get1())) {
+ if (hasPages) {
assert cpPtr != null;
// Sync log outside the checkpoint write lock.
[08/18] ignite git commit: ignite-6124 Added missed initialization of
merged join exchanges in GridDhtPartitionsExchangeFuture.onBecomeCoordinator
Posted by sb...@apache.org.
ignite-6124 Added missed initialization of merged join exchanges in GridDhtPartitionsExchangeFuture.onBecomeCoordinator
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0c5dca9a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0c5dca9a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0c5dca9a
Branch: refs/heads/ignite-6149
Commit: 0c5dca9a807c8f024e3477c1b18ddb3f237124b2
Parents: d22631e
Author: sboikov <sb...@gridgain.com>
Authored: Wed Aug 23 16:44:04 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Wed Aug 23 16:44:04 2017 +0300
----------------------------------------------------------------------
.../GridCachePartitionExchangeManager.java | 71 +++++++++++++-------
.../GridDhtPartitionsExchangeFuture.java | 7 +-
.../dht/preloader/InitNewCoordinatorFuture.java | 22 +++++-
3 files changed, 71 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c5dca9a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 984721b..bd34a5f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -309,9 +309,11 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
if (!crdInitFut.isDone() && !msg.restoreState()) {
GridDhtPartitionExchangeId exchId = msg.exchangeId();
- log.info("Waiting for coordinator initialization [node=" + node.id() +
- ", nodeOrder=" + node.order() +
- ", ver=" + (exchId != null ? exchId.topologyVersion() : null) + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Waiting for coordinator initialization [node=" + node.id() +
+ ", nodeOrder=" + node.order() +
+ ", ver=" + (exchId != null ? exchId.topologyVersion() : null) + ']');
+ }
crdInitFut.listen(new CI1<IgniteInternalFuture>() {
@Override public void apply(IgniteInternalFuture fut) {
@@ -1821,18 +1823,22 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
GridDhtPartitionsExchangeFuture fut = (GridDhtPartitionsExchangeFuture) task;
if (fut.initialVersion().compareTo(resVer) > 0) {
- log.info("Merge exchange future on finish stop [curFut=" + curFut.initialVersion() +
- ", resVer=" + resVer +
- ", nextFutVer=" + fut.initialVersion() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Merge exchange future on finish stop [curFut=" + curFut.initialVersion() +
+ ", resVer=" + resVer +
+ ", nextFutVer=" + fut.initialVersion() + ']');
+ }
break;
}
- log.info("Merge exchange future on finish [curFut=" + curFut.initialVersion() +
- ", mergedFut=" + fut.initialVersion() +
- ", evt=" + IgniteUtils.gridEventName(fut.firstEvent().type()) +
- ", evtNode=" + fut.firstEvent().eventNode().id()+
- ", evtNodeClient=" + CU.clientNode(fut.firstEvent().eventNode())+ ']');
+ if (log.isInfoEnabled()) {
+ log.info("Merge exchange future on finish [curFut=" + curFut.initialVersion() +
+ ", mergedFut=" + fut.initialVersion() +
+ ", evt=" + IgniteUtils.gridEventName(fut.firstEvent().type()) +
+ ", evtNode=" + fut.firstEvent().eventNode().id()+
+ ", evtNodeClient=" + CU.clientNode(fut.firstEvent().eventNode())+ ']');
+ }
DiscoveryEvent evt = fut.firstEvent();
@@ -1843,8 +1849,16 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
if (evt.type() == EVT_NODE_JOINED) {
final GridDhtPartitionsSingleMessage pendingMsg = fut.mergeJoinExchangeOnDone(curFut);
- if (pendingMsg != null)
+ if (pendingMsg != null) {
+ if (log.isInfoEnabled()) {
+ log.info("Merged join exchange future on finish, will reply to node [" +
+ "curFut=" + curFut.initialVersion() +
+ ", mergedFut=" + fut.initialVersion() +
+ ", evtNode=" + evt.eventNode().id() + ']');
+ }
+
curFut.waitAndReplyToNode(evt.eventNode().id(), pendingMsg);
+ }
}
}
}
@@ -1876,8 +1890,10 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
AffinityTopologyVersion exchMergeTestWaitVer = this.exchMergeTestWaitVer;
if (exchMergeTestWaitVer != null) {
- log.info("Exchange merge test, waiting for version [exch=" + curFut.initialVersion() +
- ", waitVer=" + exchMergeTestWaitVer + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Exchange merge test, waiting for version [exch=" + curFut.initialVersion() +
+ ", waitVer=" + exchMergeTestWaitVer + ']');
+ }
long end = U.currentTimeMillis() + 10_000;
@@ -1889,7 +1905,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
GridDhtPartitionsExchangeFuture fut = (GridDhtPartitionsExchangeFuture)task;
if (exchMergeTestWaitVer.equals(fut.initialVersion())) {
- log.info("Exchange merge test, found awaited version: " + exchMergeTestWaitVer);
+ if (log.isInfoEnabled())
+ log.info("Exchange merge test, found awaited version: " + exchMergeTestWaitVer);
found = true;
@@ -1923,7 +1940,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
DiscoveryEvent evt = fut.firstEvent();
if (evt.type() == EVT_DISCOVERY_CUSTOM_EVT) {
- log.info("Stop merge, custom event found: " + evt);
+ if (log.isInfoEnabled())
+ log.info("Stop merge, custom event found: " + evt);
break;
}
@@ -1931,21 +1949,25 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
ClusterNode node = evt.eventNode();
if (!curFut.context().supportsMergeExchanges(node)) {
- log.info("Stop merge, node does not support merge: " + node);
+ if (log.isInfoEnabled())
+ log.info("Stop merge, node does not support merge: " + node);
break;
}
if (evt.type() == EVT_NODE_JOINED && cctx.cache().hasCachesReceivedFromJoin(node)) {
- log.info("Stop merge, received caches from node: " + node);
+ if (log.isInfoEnabled())
+ log.info("Stop merge, received caches from node: " + node);
break;
}
- log.info("Merge exchange future [curFut=" + curFut.initialVersion() +
- ", mergedFut=" + fut.initialVersion() +
- ", evt=" + IgniteUtils.gridEventName(fut.firstEvent().type()) +
- ", evtNode=" + fut.firstEvent().eventNode().id() +
- ", evtNodeClient=" + CU.clientNode(fut.firstEvent().eventNode())+ ']');
+ if (log.isInfoEnabled()) {
+ log.info("Merge exchange future [curFut=" + curFut.initialVersion() +
+ ", mergedFut=" + fut.initialVersion() +
+ ", evt=" + IgniteUtils.gridEventName(fut.firstEvent().type()) +
+ ", evtNode=" + fut.firstEvent().eventNode().id() +
+ ", evtNodeClient=" + CU.clientNode(fut.firstEvent().eventNode())+ ']');
+ }
curFut.context().events().addEvent(fut.initialVersion(),
fut.firstEvent(),
@@ -1958,7 +1980,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
}
else {
if (!task.skipForExchangeMerge()) {
- log.info("Stop merge, custom task found: " + task);
+ if (log.isInfoEnabled())
+ log.info("Stop merge, custom task found: " + task);
break;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c5dca9a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 0dd6e4a..6decb44 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -3204,13 +3204,16 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
}
+ Map<UUID, GridDhtPartitionsSingleMessage> mergedJoins = newCrdFut.mergedJoinExchangeMessages();
+
if (log.isInfoEnabled()) {
log.info("New coordinator sends full message [ver=" + initialVersion() +
", resVer=" + fullMsg.resultTopologyVersion() +
- ", nodes=" + F.nodeIds(msgs.keySet()) + ']');
+ ", nodes=" + F.nodeIds(msgs.keySet()) +
+ ", mergedJoins=" + (mergedJoins != null ? mergedJoins.keySet() : null) + ']');
}
- sendAllPartitions(fullMsg, msgs.keySet(), null, joinedNodeAff);
+ sendAllPartitions(fullMsg, msgs.keySet(), mergedJoins, joinedNodeAff);
}
return;
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c5dca9a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
index d0e619b..b5acd4b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
@@ -196,6 +196,13 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
}
/**
+ * @return Messages for merged join exchanges.
+ */
+ Map<UUID, GridDhtPartitionsSingleMessage> mergedJoinExchangeMessages() {
+ return joinExchMsgs;
+ }
+
+ /**
* @return Full message is some of nodes received it from previous coordinator.
*/
GridDhtPartitionsFullMessage fullMessage() {
@@ -247,7 +254,8 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
if (fullMsg != null) {
AffinityTopologyVersion resVer = fullMsg.resultTopologyVersion();
- for (Iterator<Map.Entry<ClusterNode, GridDhtPartitionsSingleMessage>> it = msgs.entrySet().iterator(); it.hasNext();) {
+ for (Iterator<Map.Entry<ClusterNode, GridDhtPartitionsSingleMessage>> it = msgs.entrySet().iterator();
+ it.hasNext();) {
Map.Entry<ClusterNode, GridDhtPartitionsSingleMessage> e = it.next();
GridDhtPartitionExchangeId msgVer = joinedNodes.get(e.getKey().id());
@@ -263,8 +271,16 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
if (msgVer.topologyVersion().compareTo(resVer) > 0)
it.remove();
- else
- e.getValue().exchangeId(msgVer);
+ else {
+ GridDhtPartitionsSingleMessage msg = e.getValue();
+
+ msg.exchangeId(msgVer);
+
+ if (joinExchMsgs == null)
+ joinExchMsgs = new HashMap<>();
+
+ joinExchMsgs.put(e.getKey().id(), msg);
+ }
}
}
}
[10/18] ignite git commit: ignite-6174 Temporary changed test until
issue not fixed
Posted by sb...@apache.org.
ignite-6174 Temporary changed test until issue not fixed
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4fe8f762
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4fe8f762
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4fe8f762
Branch: refs/heads/ignite-6149
Commit: 4fe8f762c2dc1b62bdcddd606b66dc67215b50a6
Parents: eee638d
Author: sboikov <sb...@gridgain.com>
Authored: Wed Aug 23 18:29:52 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Wed Aug 23 18:29:52 2017 +0300
----------------------------------------------------------------------
...ePrimaryNodeFailureRecoveryAbstractTest.java | 111 ++++++++++---------
1 file changed, 60 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4fe8f762/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePrimaryNodeFailureRecoveryAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePrimaryNodeFailureRecoveryAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePrimaryNodeFailureRecoveryAbstractTest.java
index 0d0cda4..d147d31 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePrimaryNodeFailureRecoveryAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePrimaryNodeFailureRecoveryAbstractTest.java
@@ -56,6 +56,7 @@ import org.apache.ignite.transactions.Transaction;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.internal.processors.cache.ExchangeContext.IGNITE_EXCHANGE_COMPATIBILITY_VER_1;
import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
@@ -328,91 +329,99 @@ public abstract class IgniteCachePrimaryNodeFailureRecoveryAbstractTest extends
boolean optimistic)
throws Exception
{
- IgniteCache<Integer, Integer> cache0 = jcache(0);
- IgniteCache<Integer, Integer> cache2 = jcache(2);
+ // TODO IGNITE-6174: when exchanges can be merged test fails because of IGNITE-6174.
+ System.setProperty(IGNITE_EXCHANGE_COMPATIBILITY_VER_1, "true");
- Affinity<Integer> aff = ignite(0).affinity(DEFAULT_CACHE_NAME);
+ try {
+ IgniteCache<Integer, Integer> cache0 = jcache(0);
+ IgniteCache<Integer, Integer> cache2 = jcache(2);
- Integer key0 = null;
+ Affinity<Integer> aff = ignite(0).affinity(DEFAULT_CACHE_NAME);
- for (int key = 0; key < 10_000; key++) {
- if (aff.isPrimary(ignite(1).cluster().localNode(), key)) {
- if (locBackupKey == aff.isBackup(ignite(0).cluster().localNode(), key)) {
- key0 = key;
+ Integer key0 = null;
- break;
+ for (int key = 0; key < 10_000; key++) {
+ if (aff.isPrimary(ignite(1).cluster().localNode(), key)) {
+ if (locBackupKey == aff.isBackup(ignite(0).cluster().localNode(), key)) {
+ key0 = key;
+
+ break;
+ }
}
}
- }
- assertNotNull(key0);
+ assertNotNull(key0);
- final Integer key1 = key0;
- final Integer key2 = primaryKey(cache2);
+ final Integer key1 = key0;
+ final Integer key2 = primaryKey(cache2);
- int backups = cache0.getConfiguration(CacheConfiguration.class).getBackups();
+ int backups = cache0.getConfiguration(CacheConfiguration.class).getBackups();
- final Collection<ClusterNode> key1Nodes =
- (locBackupKey && backups < 2) ? null : aff.mapKeyToPrimaryAndBackups(key1);
- final Collection<ClusterNode> key2Nodes = aff.mapKeyToPrimaryAndBackups(key2);
+ final Collection<ClusterNode> key1Nodes =
+ (locBackupKey && backups < 2) ? null : aff.mapKeyToPrimaryAndBackups(key1);
+ final Collection<ClusterNode> key2Nodes = aff.mapKeyToPrimaryAndBackups(key2);
- TestCommunicationSpi commSpi = (TestCommunicationSpi)ignite(0).configuration().getCommunicationSpi();
+ TestCommunicationSpi commSpi = (TestCommunicationSpi)ignite(0).configuration().getCommunicationSpi();
- IgniteTransactions txs = ignite(0).transactions();
+ IgniteTransactions txs = ignite(0).transactions();
- Transaction tx = txs.txStart(optimistic ? OPTIMISTIC : PESSIMISTIC, REPEATABLE_READ);
+ Transaction tx = txs.txStart(optimistic ? OPTIMISTIC : PESSIMISTIC, REPEATABLE_READ);
- log.info("Put key1 [key1=" + key1 + ", nodes=" + U.nodeIds(aff.mapKeyToPrimaryAndBackups(key1)) + ']');
+ log.info("Put key1 [key1=" + key1 + ", nodes=" + U.nodeIds(aff.mapKeyToPrimaryAndBackups(key1)) + ']');
- cache0.put(key1, key1);
+ cache0.put(key1, key1);
- log.info("Put key2 [key2=" + key2 + ", nodes=" + U.nodeIds(aff.mapKeyToPrimaryAndBackups(key2)) + ']');
+ log.info("Put key2 [key2=" + key2 + ", nodes=" + U.nodeIds(aff.mapKeyToPrimaryAndBackups(key2)) + ']');
- cache0.put(key2, key2);
+ cache0.put(key2, key2);
- log.info("Start prepare.");
+ log.info("Start prepare.");
- GridNearTxLocal txEx = ((TransactionProxyImpl)tx).tx();
+ GridNearTxLocal txEx = ((TransactionProxyImpl)tx).tx();
- commSpi.blockMessages(ignite(2).cluster().localNode().id()); // Do not allow to finish prepare for key2.
+ commSpi.blockMessages(ignite(2).cluster().localNode().id()); // Do not allow to finish prepare for key2.
- IgniteInternalFuture<?> prepFut = txEx.prepareNearTxLocal();
+ IgniteInternalFuture<?> prepFut = txEx.prepareNearTxLocal();
- waitPrepared(ignite(1));
+ waitPrepared(ignite(1));
- log.info("Stop one primary node.");
+ log.info("Stop one primary node.");
- stopGrid(1);
+ stopGrid(1);
- U.sleep(1000); // Wait some time to catch possible issues in tx recovery.
+ U.sleep(1000); // Wait some time to catch possible issues in tx recovery.
- if (!rollback) {
- commSpi.stopBlock();
+ if (!rollback) {
+ commSpi.stopBlock();
- prepFut.get(10_000);
- }
+ prepFut.get(10_000);
+ }
- log.info("Stop originating node.");
+ log.info("Stop originating node.");
- stopGrid(0);
+ stopGrid(0);
- GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- try {
- checkKey(key1, rollback ? null : key1Nodes);
- checkKey(key2, rollback ? null : key2Nodes);
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ try {
+ checkKey(key1, rollback ? null : key1Nodes);
+ checkKey(key2, rollback ? null : key2Nodes);
- return true;
- } catch (AssertionError e) {
- log.info("Check failed: " + e);
+ return true;
+ } catch (AssertionError e) {
+ log.info("Check failed: " + e);
- return false;
+ return false;
+ }
}
- }
- }, 5000);
+ }, 5000);
- checkKey(key1, rollback ? null : key1Nodes);
- checkKey(key2, rollback ? null : key2Nodes);
+ checkKey(key1, rollback ? null : key1Nodes);
+ checkKey(key2, rollback ? null : key2Nodes);
+ }
+ finally {
+ System.clearProperty(IGNITE_EXCHANGE_COMPATIBILITY_VER_1);
+ }
}
/**
[09/18] ignite git commit: Merge remote-tracking branch
'origin/master'
Posted by sb...@apache.org.
Merge remote-tracking branch 'origin/master'
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/eee638d3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/eee638d3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/eee638d3
Branch: refs/heads/ignite-6149
Commit: eee638d3da9a5dfbc7b43fd56eb4f72328c13a31
Parents: 0c5dca9 ac94426
Author: sboikov <sb...@gridgain.com>
Authored: Wed Aug 23 16:44:27 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Wed Aug 23 16:44:27 2017 +0300
----------------------------------------------------------------------
.../internal/jdbc2/JdbcMetadataSelfTest.java | 17 +++++++++++++++++
.../internal/jdbc2/JdbcDatabaseMetadata.java | 2 ++
2 files changed, 19 insertions(+)
----------------------------------------------------------------------
[07/18] ignite git commit: IGNITE-4643: Fixed NPE in
JdbcDatabaseMetadata.getIndexInfo(). This closes #2481.
Posted by sb...@apache.org.
IGNITE-4643: Fixed NPE in JdbcDatabaseMetadata.getIndexInfo(). This closes #2481.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ac94426c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ac94426c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ac94426c
Branch: refs/heads/ignite-6149
Commit: ac94426ced2cee5df2c993029e2a17ea5a9aa398
Parents: d22631e
Author: Ilya Kasnacheev <il...@gmail.com>
Authored: Wed Aug 23 16:26:58 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Aug 23 16:26:58 2017 +0300
----------------------------------------------------------------------
.../internal/jdbc2/JdbcMetadataSelfTest.java | 17 +++++++++++++++++
.../internal/jdbc2/JdbcDatabaseMetadata.java | 2 ++
2 files changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac94426c/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMetadataSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMetadataSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMetadataSelfTest.java
index 52e68c9..f2ef769 100755
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMetadataSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMetadataSelfTest.java
@@ -278,6 +278,23 @@ public class JdbcMetadataSelfTest extends GridCommonAbstractTest {
}
/**
+ * @throws Exception If failed.
+ */
+ public void testIndexMetadata() throws Exception {
+ try (Connection conn = DriverManager.getConnection(BASE_URL);
+ ResultSet rs = conn.getMetaData().getIndexInfo(null, "pers", "PERSON", false, false)) {
+
+ int cnt = 0;
+
+ while (rs.next()) {
+ cnt++;
+ }
+
+ assertEquals(0, cnt);
+ }
+ }
+
+ /**
* Person.
*/
@SuppressWarnings("UnusedDeclaration")
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac94426c/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcDatabaseMetadata.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcDatabaseMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcDatabaseMetadata.java
index b369b0b..e9a5fde 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcDatabaseMetadata.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcDatabaseMetadata.java
@@ -965,6 +965,8 @@ public class JdbcDatabaseMetadata implements DatabaseMetaData {
/** {@inheritDoc} */
@Override public ResultSet getIndexInfo(String catalog, String schema, String tbl, boolean unique,
boolean approximate) throws SQLException {
+ updateMetaData();
+
Collection<List<?>> rows = new ArrayList<>(indexes.size());
for (List<Object> idx : indexes) {
[18/18] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-6149
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-6149
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c13877f2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c13877f2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c13877f2
Branch: refs/heads/ignite-6149
Commit: c13877f2f28fd04fda3809f50103ea3df1b2700c
Parents: 42d5d63 79d47f8
Author: sboikov <sb...@gridgain.com>
Authored: Fri Aug 25 11:39:25 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Aug 25 11:39:25 2017 +0300
----------------------------------------------------------------------
.../ml/math/matrix/CacheMatrixExample.java | 4 +-
.../ml/math/vector/CacheVectorExample.java | 4 +-
.../internal/jdbc2/JdbcMetadataSelfTest.java | 17 +
.../GridClientConnectionManagerAdapter.java | 1 -
.../connection/GridClientNioTcpConnection.java | 2 +-
.../internal/jdbc/thin/JdbcThinTcpIo.java | 37 +-
.../internal/jdbc2/JdbcDatabaseMetadata.java | 2 +
.../GridCachePartitionExchangeManager.java | 118 ++--
.../dht/GridClientPartitionTopology.java | 9 +
.../dht/GridDhtPartitionTopology.java | 5 +
.../dht/GridDhtPartitionTopologyImpl.java | 5 +
.../CachePartitionFullCountersMap.java | 36 ++
.../CachePartitionPartialCountersMap.java | 23 +
.../GridDhtPartitionsExchangeFuture.java | 249 ++++++---
.../preloader/GridDhtPartitionsFullMessage.java | 84 ++-
.../GridDhtPartitionsSingleMessage.java | 23 +-
.../IgniteDhtPartitionCountersMap.java | 14 +-
.../IgniteDhtPartitionCountersMap2.java | 69 +++
.../dht/preloader/InitNewCoordinatorFuture.java | 60 +-
.../GridCacheDatabaseSharedManager.java | 29 +-
.../cluster/GridClusterStateProcessor.java | 6 +-
.../datastreamer/DataStreamerImpl.java | 14 +-
.../internal/util/GridSpinReadWriteLock.java | 2 +-
.../internal/util/nio/GridNioKeyAttachment.java | 33 ++
.../util/nio/GridNioRecoveryDescriptor.java | 3 +-
.../ignite/internal/util/nio/GridNioServer.java | 248 ++++++--
.../util/nio/GridSelectorNioSessionImpl.java | 28 +-
.../internal/util/nio/ssl/GridNioSslFilter.java | 12 +-
.../communication/tcp/TcpCommunicationSpi.java | 29 +-
.../IgniteCacheMessageWriteTimeoutTest.java | 4 +-
...ePrimaryNodeFailureRecoveryAbstractTest.java | 111 ++--
.../processors/database/BPlusTreeSelfTest.java | 39 +-
.../internal/util/nio/GridNioSelfTest.java | 2 +-
.../spi/GridTcpSpiForwardingSelfTest.java | 1 +
.../GridAbstractCommunicationSelfTest.java | 27 +-
...mmunicationSpiConcurrentConnectSelfTest.java | 28 +-
...dTcpCommunicationSpiRecoveryAckSelfTest.java | 39 +-
...GridTcpCommunicationSpiRecoverySelfTest.java | 47 +-
...CommunicationRecoveryAckClosureSelfTest.java | 36 +-
.../tcp/TcpCommunicationSpiDropNodesTest.java | 3 +-
.../HadoopExternalCommunication.java | 5 +-
modules/ml/pom.xml | 1 -
.../clustering/KMeansDistributedClusterer.java | 55 +-
.../ignite/ml/math/IdentityValueMapper.java | 2 +
.../apache/ignite/ml/math/MatrixKeyMapper.java | 30 -
.../apache/ignite/ml/math/MatrixStorage.java | 7 +
.../org/apache/ignite/ml/math/ValueMapper.java | 37 --
.../apache/ignite/ml/math/VectorKeyMapper.java | 29 -
.../ignite/ml/math/distributed/CacheUtils.java | 546 ++++++++++++++++++
.../ml/math/distributed/DistributedStorage.java | 35 ++
.../ml/math/distributed/MatrixKeyMapper.java | 33 ++
.../ignite/ml/math/distributed/ValueMapper.java | 37 ++
.../ml/math/distributed/VectorKeyMapper.java | 32 ++
.../math/distributed/keys/BlockMatrixKey.java | 30 +
.../math/distributed/keys/MatrixCacheKey.java | 35 ++
.../math/distributed/keys/RowColMatrixKey.java | 30 +
.../distributed/keys/impl/BlockMatrixKey.java | 144 +++++
.../distributed/keys/impl/SparseMatrixKey.java | 142 +++++
.../distributed/keys/impl/package-info.java | 22 +
.../ml/math/distributed/keys/package-info.java | 22 +
.../ml/math/distributed/package-info.java | 22 +
.../apache/ignite/ml/math/impls/CacheUtils.java | 559 -------------------
.../ml/math/impls/matrix/CacheMatrix.java | 6 +-
.../impls/matrix/DenseLocalOnHeapMatrix.java | 4 +-
.../matrix/SparseBlockDistributedMatrix.java | 16 +-
.../impls/matrix/SparseDistributedMatrix.java | 83 ++-
.../storage/matrix/BaseBlockMatrixKey.java | 41 --
.../impls/storage/matrix/BlockMatrixKey.java | 144 -----
.../storage/matrix/BlockMatrixStorage.java | 38 +-
.../storage/matrix/CacheMatrixStorage.java | 9 +-
.../matrix/DenseOffHeapMatrixStorage.java | 5 +
.../storage/matrix/DiagonalMatrixStorage.java | 5 +
.../storage/matrix/FunctionMatrixStorage.java | 5 +
.../storage/matrix/MatrixDelegateStorage.java | 5 +
.../storage/matrix/PivotedMatrixStorage.java | 5 +
.../storage/matrix/RandomMatrixStorage.java | 5 +
.../matrix/SparseDistributedMatrixStorage.java | 54 +-
.../matrix/SparseLocalOnHeapMatrixStorage.java | 6 +-
.../storage/vector/CacheVectorStorage.java | 4 +-
.../ml/math/impls/vector/CacheVector.java | 6 +-
.../ml/math/impls/matrix/CacheMatrixTest.java | 2 +-
.../impls/matrix/MatrixKeyMapperForTests.java | 2 +-
.../SparseDistributedBlockMatrixTest.java | 7 +-
.../matrix/SparseDistributedMatrixTest.java | 40 +-
.../ml/math/impls/vector/CacheVectorTest.java | 2 +-
.../app/modules/agent/AgentManager.service.js | 2 +-
.../frontend/webpack/webpack.common.js | 1 -
87 files changed, 2525 insertions(+), 1330 deletions(-)
----------------------------------------------------------------------
[11/18] ignite git commit: ignite-5280 SparseDistributedMatrix
refactoring
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
deleted file mode 100644
index 2c519f0..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
+++ /dev/null
@@ -1,559 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.math.impls;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.function.BinaryOperator;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.query.ScanQuery;
-import org.apache.ignite.cluster.ClusterGroup;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
-import org.apache.ignite.internal.util.typedef.internal.A;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.lang.IgniteCallable;
-import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.lang.IgniteRunnable;
-import org.apache.ignite.lang.IgniteUuid;
-import org.apache.ignite.ml.math.KeyMapper;
-import org.apache.ignite.ml.math.ValueMapper;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteConsumer;
-import org.apache.ignite.ml.math.functions.IgniteDoubleFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.BlockEntry;
-import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey;
-
-/**
- * Distribution-related misc. support.
- *
- * TODO: IGNITE-5102, fix sparse key filters.
- */
-public class CacheUtils {
- /**
- * Cache entry support.
- *
- * @param <K>
- * @param <V>
- */
- public static class CacheEntry<K, V> {
- /** */
- private Cache.Entry<K, V> entry;
- /** */
- private IgniteCache<K, V> cache;
-
- /**
- * @param entry Original cache entry.
- * @param cache Cache instance.
- */
- CacheEntry(Cache.Entry<K, V> entry, IgniteCache<K, V> cache) {
- this.entry = entry;
- this.cache = cache;
- }
-
- /**
- *
- *
- */
- public Cache.Entry<K, V> entry() {
- return entry;
- }
-
- /**
- *
- *
- */
- public IgniteCache<K, V> cache() {
- return cache;
- }
- }
-
- /**
- * Gets local Ignite instance.
- */
- public static Ignite ignite() {
- return Ignition.localIgnite();
- }
-
- /**
- * @param cacheName Cache name.
- * @param k Key into the cache.
- * @param <K> Key type.
- * @return Cluster group for given key.
- */
- public static <K> ClusterGroup groupForKey(String cacheName, K k) {
- return ignite().cluster().forNode(ignite().affinity(cacheName).mapKeyToNode(k));
- }
-
- /**
- * @param cacheName Cache name.
- * @param keyMapper {@link KeyMapper} to validate cache key.
- * @param valMapper {@link ValueMapper} to obtain double value for given cache key.
- * @param <K> Cache key object type.
- * @param <V> Cache value object type.
- * @return Sum of the values obtained for valid keys.
- */
- public static <K, V> double sum(String cacheName, KeyMapper<K> keyMapper, ValueMapper<V> valMapper) {
- Collection<Double> subSums = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
- if (keyMapper.isValid(ce.entry().getKey())) {
- double v = valMapper.toDouble(ce.entry().getValue());
-
- return acc == null ? v : acc + v;
- }
- else
- return acc;
- });
-
- return sum(subSums);
- }
-
- /**
- * @param matrixUuid Matrix UUID.
- * @return Sum obtained using sparse logic.
- */
- @SuppressWarnings("unchecked")
- public static <K, V> double sparseSum(IgniteUuid matrixUuid, String cacheName) {
- A.notNull(matrixUuid, "matrixUuid");
- A.notNull(cacheName, "cacheName");
-
- Collection<Double> subSums = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
- V v = ce.entry().getValue();
-
- double sum = 0.0;
-
- if (v instanceof Map) {
- Map<Integer, Double> map = (Map<Integer, Double>)v;
-
- sum = sum(map.values());
- }
- else if (v instanceof BlockEntry) {
- BlockEntry be = (BlockEntry)v;
-
- sum = be.sum();
- }
- else
- throw new UnsupportedOperationException();
-
- return acc == null ? sum : acc + sum;
- }, key -> {
- if (key instanceof BlockMatrixKey)
- return ((BlockMatrixKey)key).matrixId().equals(matrixUuid);
- else if (key instanceof IgniteBiTuple)
- return ((IgniteBiTuple<Integer, IgniteUuid>)key).get2().equals(matrixUuid);
- else
- throw new UnsupportedOperationException();
- });
-
- return sum(subSums);
- }
-
- /**
- * @param c {@link Collection} of double values to sum.
- * @return Sum of the values.
- */
- private static double sum(Collection<Double> c) {
- double sum = 0.0;
-
- for (double d : c)
- sum += d;
-
- return sum;
- }
-
- /**
- * @param cacheName Cache name.
- * @param keyMapper {@link KeyMapper} to validate cache key.
- * @param valMapper {@link ValueMapper} to obtain double value for given cache key.
- * @param <K> Cache key object type.
- * @param <V> Cache value object type.
- * @return Minimum value for valid keys.
- */
- public static <K, V> double min(String cacheName, KeyMapper<K> keyMapper, ValueMapper<V> valMapper) {
- Collection<Double> mins = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
- if (keyMapper.isValid(ce.entry().getKey())) {
- double v = valMapper.toDouble(ce.entry().getValue());
-
- if (acc == null)
- return v;
- else
- return Math.min(acc, v);
- }
- else
- return acc;
- });
-
- return Collections.min(mins);
- }
-
- /**
- * @param matrixUuid Matrix UUID.
- * @return Minimum value obtained using sparse logic.
- */
- @SuppressWarnings("unchecked")
- public static <K, V> double sparseMin(IgniteUuid matrixUuid, String cacheName) {
- A.notNull(matrixUuid, "matrixUuid");
- A.notNull(cacheName, "cacheName");
-
- Collection<Double> mins = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
- V v = ce.entry().getValue();
-
- double min;
-
- if (v instanceof Map) {
- Map<Integer, Double> map = (Map<Integer, Double>)v;
-
- min = Collections.min(map.values());
- }
- else if (v instanceof BlockEntry) {
- BlockEntry be = (BlockEntry)v;
-
- min = be.minValue();
- }
- else
- throw new UnsupportedOperationException();
-
- if (acc == null)
- return min;
- else
- return Math.min(acc, min);
-
- }, key -> {
- if (key instanceof BlockMatrixKey)
- return ((BlockMatrixKey)key).matrixId().equals(matrixUuid);
- else if (key instanceof IgniteBiTuple)
- return ((IgniteBiTuple<Integer, IgniteUuid>)key).get2().equals(matrixUuid);
- else
- throw new UnsupportedOperationException();
- });
-
- return Collections.min(mins);
- }
-
- /**
- * @param matrixUuid Matrix UUID.
- * @return Maximum value obtained using sparse logic.
- */
- @SuppressWarnings("unchecked")
- public static <K, V> double sparseMax(IgniteUuid matrixUuid, String cacheName) {
- A.notNull(matrixUuid, "matrixUuid");
- A.notNull(cacheName, "cacheName");
-
- Collection<Double> maxes = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
- V v = ce.entry().getValue();
-
- double max;
-
- if (v instanceof Map) {
- Map<Integer, Double> map = (Map<Integer, Double>)v;
-
- max = Collections.max(map.values());
- }
- else if (v instanceof BlockEntry) {
- BlockEntry be = (BlockEntry)v;
-
- max = be.maxValue();
- }
- else
- throw new UnsupportedOperationException();
-
- if (acc == null)
- return max;
- else
- return Math.max(acc, max);
-
- }, key -> {
- if (key instanceof BlockMatrixKey)
- return ((BlockMatrixKey)key).matrixId().equals(matrixUuid);
- else if (key instanceof IgniteBiTuple)
- return ((IgniteBiTuple<Integer, IgniteUuid>)key).get2().equals(matrixUuid);
- else
- throw new UnsupportedOperationException();
- });
-
- return Collections.max(maxes);
- }
-
- /**
- * @param cacheName Cache name.
- * @param keyMapper {@link KeyMapper} to validate cache key.
- * @param valMapper {@link ValueMapper} to obtain double value for given cache key.
- * @param <K> Cache key object type.
- * @param <V> Cache value object type.
- * @return Maximum value for valid keys.
- */
- public static <K, V> double max(String cacheName, KeyMapper<K> keyMapper, ValueMapper<V> valMapper) {
- Collection<Double> maxes = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
- if (keyMapper.isValid(ce.entry().getKey())) {
- double v = valMapper.toDouble(ce.entry().getValue());
-
- if (acc == null)
- return v;
- else
- return Math.max(acc, v);
- }
- else
- return acc;
- });
-
- return Collections.max(maxes);
- }
-
- /**
- * @param cacheName Cache name.
- * @param keyMapper {@link KeyMapper} to validate cache key.
- * @param valMapper {@link ValueMapper} to obtain double value for given cache key.
- * @param mapper Mapping {@link IgniteFunction}.
- * @param <K> Cache key object type.
- * @param <V> Cache value object type.
- */
- public static <K, V> void map(String cacheName, KeyMapper<K> keyMapper, ValueMapper<V> valMapper,
- IgniteFunction<Double, Double> mapper) {
- foreach(cacheName, (CacheEntry<K, V> ce) -> {
- K k = ce.entry().getKey();
-
- if (keyMapper.isValid(k))
- // Actual assignment.
- ce.cache().put(k, valMapper.fromDouble(mapper.apply(valMapper.toDouble(ce.entry().getValue()))));
- });
- }
-
- /**
- * @param matrixUuid Matrix UUID.
- * @param mapper Mapping {@link IgniteFunction}.
- */
- @SuppressWarnings("unchecked")
- public static <K, V> void sparseMap(IgniteUuid matrixUuid, IgniteDoubleFunction<Double> mapper, String cacheName) {
- A.notNull(matrixUuid, "matrixUuid");
- A.notNull(cacheName, "cacheName");
- A.notNull(mapper, "mapper");
-
- foreach(cacheName, (CacheEntry<K, V> ce) -> {
- K k = ce.entry().getKey();
-
- V v = ce.entry().getValue();
-
- if (v instanceof Map) {
- Map<Integer, Double> map = (Map<Integer, Double>)v;
-
- for (Map.Entry<Integer, Double> e : (map.entrySet()))
- e.setValue(mapper.apply(e.getValue()));
-
- }
- else if (v instanceof BlockEntry) {
- BlockEntry be = (BlockEntry)v;
-
- be.map(mapper);
- }
- else
- throw new UnsupportedOperationException();
-
- ce.cache().put(k, v);
- }, key -> {
- if (key instanceof BlockMatrixKey)
- return ((BlockMatrixKey)key).matrixId().equals(matrixUuid);
- else if (key instanceof IgniteBiTuple)
- return ((IgniteBiTuple<Integer, IgniteUuid>)key).get2().equals(matrixUuid);
- else
- throw new UnsupportedOperationException();
- });
- }
-
- /**
- * @param cacheName Cache name.
- * @param fun An operation that accepts a cache entry and processes it.
- * @param <K> Cache key object type.
- * @param <V> Cache value object type.
- */
- public static <K, V> void foreach(String cacheName, IgniteConsumer<CacheEntry<K, V>> fun) {
- foreach(cacheName, fun, null);
- }
-
- /**
- * @param cacheName Cache name.
- * @param fun An operation that accepts a cache entry and processes it.
- * @param keyFilter Cache keys filter.
- * @param <K> Cache key object type.
- * @param <V> Cache value object type.
- */
- public static <K, V> void foreach(String cacheName, IgniteConsumer<CacheEntry<K, V>> fun,
- IgnitePredicate<K> keyFilter) {
- bcast(cacheName, () -> {
- Ignite ignite = Ignition.localIgnite();
- IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
-
- int partsCnt = ignite.affinity(cacheName).partitions();
-
- // Use affinity in filter for scan query. Otherwise we accept consumer in each node which is wrong.
- Affinity affinity = ignite.affinity(cacheName);
- ClusterNode locNode = ignite.cluster().localNode();
-
- // Iterate over all partitions. Some of them will be stored on that local node.
- for (int part = 0; part < partsCnt; part++) {
- int p = part;
-
- // Iterate over given partition.
- // Query returns an empty cursor if this partition is not stored on this node.
- for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part, (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k)))))
- fun.accept(new CacheEntry<>(entry, cache));
- }
- });
- }
-
- /**
- * <b>Currently fold supports only commutative operations.<b/>
- *
- * @param cacheName Cache name.
- * @param folder Fold function operating over cache entries.
- * @param <K> Cache key object type.
- * @param <V> Cache value object type.
- * @param <A> Fold result type.
- * @return Fold operation result.
- */
- public static <K, V, A> Collection<A> fold(String cacheName, IgniteBiFunction<CacheEntry<K, V>, A, A> folder) {
- return fold(cacheName, folder, null);
- }
-
- /**
- * <b>Currently fold supports only commutative operations.<b/>
- *
- * @param cacheName Cache name.
- * @param folder Fold function operating over cache entries.
- * @param <K> Cache key object type.
- * @param <V> Cache value object type.
- * @param <A> Fold result type.
- * @return Fold operation result.
- */
- public static <K, V, A> Collection<A> fold(String cacheName, IgniteBiFunction<CacheEntry<K, V>, A, A> folder,
- IgnitePredicate<K> keyFilter) {
- return bcast(cacheName, () -> {
- Ignite ignite = Ignition.localIgnite();
- IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
-
- int partsCnt = ignite.affinity(cacheName).partitions();
-
- // Use affinity in filter for ScanQuery. Otherwise we accept consumer in each node which is wrong.
- Affinity affinity = ignite.affinity(cacheName);
- ClusterNode locNode = ignite.cluster().localNode();
-
- A a = null;
-
- // Iterate over all partitions. Some of them will be stored on that local node.
- for (int part = 0; part < partsCnt; part++) {
- int p = part;
-
- // Iterate over given partition.
- // Query returns an empty cursor if this partition is not stored on this node.
- for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part,
- (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k)))))
- a = folder.apply(new CacheEntry<>(entry, cache), a);
- }
-
- return a;
- });
- }
-
- /**
- * Distributed version of fold operation.
- *
- * @param cacheName Cache name.
- * @param folder Folder.
- * @param keyFilter Key filter.
- * @param accumulator Accumulator.
- * @param zeroVal Zero value.
- */
- public static <K, V, A> A distributedFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder,
- IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, A zeroVal) {
- return sparseFold(cacheName, folder, keyFilter, accumulator, zeroVal, null, null, 0,
- false);
- }
-
- /**
- * Sparse version of fold. This method also applicable to sparse zeroes.
- *
- * @param cacheName Cache name.
- * @param folder Folder.
- * @param keyFilter Key filter.
- * @param accumulator Accumulator.
- * @param zeroVal Zero value.
- * @param defVal Def value.
- * @param defKey Def key.
- * @param defValCnt Def value count.
- * @param isNilpotent Is nilpotent.
- */
- private static <K, V, A> A sparseFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder,
- IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, A zeroVal, V defVal, K defKey, long defValCnt,
- boolean isNilpotent) {
-
- A defRes = zeroVal;
-
- if (!isNilpotent)
- for (int i = 0; i < defValCnt; i++)
- defRes = folder.apply(new CacheEntryImpl<>(defKey, defVal), defRes);
-
- Collection<A> totalRes = bcast(cacheName, () -> {
- Ignite ignite = Ignition.localIgnite();
- IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
-
- int partsCnt = ignite.affinity(cacheName).partitions();
-
- // Use affinity in filter for ScanQuery. Otherwise we accept consumer in each node which is wrong.
- Affinity affinity = ignite.affinity(cacheName);
- ClusterNode locNode = ignite.cluster().localNode();
-
- A a = zeroVal;
-
- // Iterate over all partitions. Some of them will be stored on that local node.
- for (int part = 0; part < partsCnt; part++) {
- int p = part;
-
- // Iterate over given partition.
- // Query returns an empty cursor if this partition is not stored on this node.
- for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part,
- (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k)))))
- a = folder.apply(entry, a);
- }
-
- return a;
- });
- totalRes.add(defRes);
- return totalRes.stream().reduce(zeroVal, accumulator);
- }
-
- /**
- * @param cacheName Cache name.
- * @param run {@link Runnable} to broadcast to cache nodes for given cache name.
- */
- public static void bcast(String cacheName, IgniteRunnable run) {
- ignite().compute(ignite().cluster().forCacheNodes(cacheName)).broadcast(run);
- }
-
- /**
- * @param cacheName Cache name.
- * @param call {@link IgniteCallable} to broadcast to cache nodes for given cache name.
- * @param <A> Type returned by the callable.
- */
- public static <A> Collection<A> bcast(String cacheName, IgniteCallable<A> call) {
- return ignite().compute(ignite().cluster().forCacheNodes(cacheName)).broadcast(call);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java
index 7f00bcb..1f832bc 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java
@@ -19,13 +19,13 @@ package org.apache.ignite.ml.math.impls.matrix;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.MatrixKeyMapper;
-import org.apache.ignite.ml.math.ValueMapper;
import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.distributed.CacheUtils;
+import org.apache.ignite.ml.math.distributed.MatrixKeyMapper;
+import org.apache.ignite.ml.math.distributed.ValueMapper;
import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
import org.apache.ignite.ml.math.functions.IgniteDoubleFunction;
import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.CacheUtils;
import org.apache.ignite.ml.math.impls.storage.matrix.CacheMatrixStorage;
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/DenseLocalOnHeapMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/DenseLocalOnHeapMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/DenseLocalOnHeapMatrix.java
index 393fff6..2376cbd 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/DenseLocalOnHeapMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/DenseLocalOnHeapMatrix.java
@@ -126,8 +126,8 @@ public class DenseLocalOnHeapMatrix extends AbstractMatrix implements OrderedMat
return new DenseLocalOnHeapVector(crd);
}
- /** */
+ /** {@inheritDoc} */
@Override public int accessMode() {
- return ((ArrayMatrixStorage)getStorage()).accessMode();
+ return getStorage().accessMode();
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java
index b3481f9..1dcf1d8 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java
@@ -29,11 +29,11 @@ import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.StorageConstants;
import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.distributed.CacheUtils;
+import org.apache.ignite.ml.math.distributed.keys.impl.BlockMatrixKey;
import org.apache.ignite.ml.math.exceptions.CardinalityException;
import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
import org.apache.ignite.ml.math.functions.IgniteDoubleFunction;
-import org.apache.ignite.ml.math.impls.CacheUtils;
-import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey;
import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixStorage;
/**
@@ -101,10 +101,10 @@ public class SparseBlockDistributedMatrix extends AbstractMatrix implements Stor
SparseBlockDistributedMatrix matrixA = this;
SparseBlockDistributedMatrix matrixB = (SparseBlockDistributedMatrix)mtx;
- String cacheName = BlockMatrixStorage.ML_BLOCK_CACHE_NAME;
+ String cacheName = this.storage().cacheName();
SparseBlockDistributedMatrix matrixC = new SparseBlockDistributedMatrix(matrixA.rowSize(), matrixB.columnSize());
- CacheUtils.bcast(BlockMatrixStorage.ML_BLOCK_CACHE_NAME, () -> {
+ CacheUtils.bcast(cacheName, () -> {
Ignite ignite = Ignition.localIgnite();
Affinity affinity = ignite.affinity(cacheName);
@@ -156,17 +156,17 @@ public class SparseBlockDistributedMatrix extends AbstractMatrix implements Stor
/** {@inheritDoc} */
@Override public double sum() {
- return CacheUtils.sparseSum(getUUID(), BlockMatrixStorage.ML_BLOCK_CACHE_NAME);
+ return CacheUtils.sparseSum(getUUID(), this.storage().cacheName());
}
/** {@inheritDoc} */
@Override public double maxValue() {
- return CacheUtils.sparseMax(getUUID(), BlockMatrixStorage.ML_BLOCK_CACHE_NAME);
+ return CacheUtils.sparseMax(getUUID(), this.storage().cacheName());
}
/** {@inheritDoc} */
@Override public double minValue() {
- return CacheUtils.sparseMin(getUUID(), BlockMatrixStorage.ML_BLOCK_CACHE_NAME);
+ return CacheUtils.sparseMin(getUUID(), this.storage().cacheName());
}
/** {@inheritDoc} */
@@ -194,7 +194,7 @@ public class SparseBlockDistributedMatrix extends AbstractMatrix implements Stor
* @return Matrix with mapped values.
*/
private Matrix mapOverValues(IgniteDoubleFunction<Double> mapper) {
- CacheUtils.sparseMap(getUUID(), mapper, BlockMatrixStorage.ML_BLOCK_CACHE_NAME);
+ CacheUtils.sparseMap(getUUID(), mapper, this.storage().cacheName());
return this;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
index a3a7df4..92d7c39 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
@@ -17,14 +17,22 @@
package org.apache.ignite.ml.math.impls.matrix;
+import java.util.Collection;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.StorageConstants;
import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.distributed.CacheUtils;
+import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey;
import org.apache.ignite.ml.math.exceptions.CardinalityException;
import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
import org.apache.ignite.ml.math.functions.IgniteDoubleFunction;
-import org.apache.ignite.ml.math.impls.CacheUtils;
import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
/**
@@ -93,32 +101,63 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo
return mapOverValues(v -> v * x);
}
- /**
- * TODO: IGNITE-5114, tmp naive implementation, WIP.
- */
+
+ /** {@inheritDoc} */
@Override public Matrix times(Matrix mtx) {
- int cols = columnSize();
+ if (mtx == null)
+ throw new IllegalArgumentException("The matrix should be not null.");
+
+ if (columnSize() != mtx.rowSize())
+ throw new CardinalityException(columnSize(), mtx.rowSize());
+
+ SparseDistributedMatrix matrixA = this;
+ SparseDistributedMatrix matrixB = (SparseDistributedMatrix)mtx;
+
+ String cacheName = storage().cacheName();
+ SparseDistributedMatrix matrixC = new SparseDistributedMatrix(matrixA.rowSize(), matrixB.columnSize()
+ , getStorage().storageMode(), getStorage().isRandomAccess() ? RANDOM_ACCESS_MODE : SEQUENTIAL_ACCESS_MODE);
+
+ CacheUtils.bcast(cacheName, () -> {
+ Ignite ignite = Ignition.localIgnite();
+ Affinity affinity = ignite.affinity(cacheName);
+
+ IgniteCache<RowColMatrixKey, BlockEntry> cache = ignite.getOrCreateCache(cacheName);
+ ClusterNode locNode = ignite.cluster().localNode();
- if (cols != mtx.rowSize())
- throw new CardinalityException(cols, mtx.rowSize());
+ SparseDistributedMatrixStorage storageC = matrixC.storage();
- int rows = rowSize();
+ Map<ClusterNode, Collection<RowColMatrixKey>> keysCToNodes = affinity.mapKeysToNodes(storageC.getAllKeys());
+ Collection<RowColMatrixKey> locKeys = keysCToNodes.get(locNode);
- int mtxCols = mtx.columnSize();
+ boolean isRowMode = storageC.storageMode() == ROW_STORAGE_MODE;
- Matrix res = like(rows, mtxCols);
+ if (locKeys == null)
+ return;
- for (int x = 0; x < rows; x++)
- for (int y = 0; y < mtxCols; y++) {
- double sum = 0.0;
+ // compute Cij locally on each node
+ // TODO: IGNITE:5114, exec in parallel
+ locKeys.forEach(key -> {
+ int idx = key.index();
+
+ if (isRowMode){
+ Vector Aik = matrixA.getCol(idx);
- for (int k = 0; k < cols; k++)
- sum += getX(x, k) * mtx.getX(k, y);
+ for (int i = 0; i < columnSize(); i++) {
+ Vector Bkj = matrixB.getRow(i);
+ matrixC.set(idx, i, Aik.times(Bkj).sum());
+ }
+ } else {
+ Vector Bkj = matrixB.getRow(idx);
- res.setX(x, y, sum);
- }
+ for (int i = 0; i < rowSize(); i++) {
+ Vector Aik = matrixA.getCol(i);
+ matrixC.set(idx, i, Aik.times(Bkj).sum());
+ }
+ }
+ });
+ });
- return res;
+ return matrixC;
}
/** {@inheritDoc} */
@@ -136,24 +175,24 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo
* @return Matrix with mapped values.
*/
private Matrix mapOverValues(IgniteDoubleFunction<Double> mapper) {
- CacheUtils.sparseMap(getUUID(), mapper, SparseDistributedMatrixStorage.ML_CACHE_NAME);
+ CacheUtils.sparseMap(getUUID(), mapper, storage().cacheName());
return this;
}
/** {@inheritDoc} */
@Override public double sum() {
- return CacheUtils.sparseSum(getUUID(), SparseDistributedMatrixStorage.ML_CACHE_NAME);
+ return CacheUtils.sparseSum(getUUID(), storage().cacheName());
}
/** {@inheritDoc} */
@Override public double maxValue() {
- return CacheUtils.sparseMax(getUUID(), SparseDistributedMatrixStorage.ML_CACHE_NAME);
+ return CacheUtils.sparseMax(getUUID(), storage().cacheName());
}
/** {@inheritDoc} */
@Override public double minValue() {
- return CacheUtils.sparseMin(getUUID(), SparseDistributedMatrixStorage.ML_CACHE_NAME);
+ return CacheUtils.sparseMin(getUUID(), storage().cacheName());
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java
deleted file mode 100644
index 74ddfe5..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.math.impls.storage.matrix;
-
-import org.apache.ignite.lang.IgniteUuid;
-import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix;
-
-/**
- * Cache key for blocks in {@link SparseBlockDistributedMatrix}.
- */
-public interface BaseBlockMatrixKey {
- /**
- * @return block id.
- */
- public long blockId();
-
- /**
- * @return matrix id.
- */
- public IgniteUuid matrixId();
-
- /**
- * @return key affinity key.
- */
- public IgniteUuid affinityKey();
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java
deleted file mode 100644
index 3749f44..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.math.impls.storage.matrix;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import org.apache.ignite.binary.BinaryObjectException;
-import org.apache.ignite.binary.BinaryRawReader;
-import org.apache.ignite.binary.BinaryRawWriter;
-import org.apache.ignite.binary.BinaryReader;
-import org.apache.ignite.binary.BinaryWriter;
-import org.apache.ignite.binary.Binarylizable;
-import org.apache.ignite.internal.binary.BinaryUtils;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteUuid;
-import org.apache.ignite.ml.math.impls.matrix.BlockEntry;
-import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * Key implementation for {@link BlockEntry} using for {@link SparseBlockDistributedMatrix}.
- */
-public class BlockMatrixKey implements BaseBlockMatrixKey, Externalizable, Binarylizable {
- /** */
- private static final long serialVersionUID = 0L;
- /** Block ID */
- private long blockId;
- /** Matrix ID */
- private IgniteUuid matrixUuid;
- /** Block affinity key. */
- private IgniteUuid affinityKey;
-
- /**
- * Empty constructor required for {@link Externalizable}.
- */
- public BlockMatrixKey() {
- // No-op.
- }
-
- /**
- * Construct matrix block key.
- *
- * @param blockId Block id.
- * @param matrixUuid Matrix uuid.
- * @param affinityKey Affinity key.
- */
- public BlockMatrixKey(long blockId, IgniteUuid matrixUuid, @Nullable IgniteUuid affinityKey) {
- assert blockId >= 0;
- assert matrixUuid != null;
-
- this.blockId = blockId;
- this.matrixUuid = matrixUuid;
- this.affinityKey = affinityKey;
- }
-
- /** {@inheritDoc} */
- @Override public long blockId() {
- return blockId;
- }
-
- /** {@inheritDoc} */
- @Override public IgniteUuid matrixId() {
- return matrixUuid;
- }
-
- /** {@inheritDoc} */
- @Override public IgniteUuid affinityKey() {
- return affinityKey;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- U.writeGridUuid(out, matrixUuid);
- U.writeGridUuid(out, affinityKey);
- out.writeLong(blockId);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- matrixUuid = U.readGridUuid(in);
- affinityKey = U.readGridUuid(in);
- blockId = in.readLong();
- }
-
- /** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
- BinaryRawWriter out = writer.rawWriter();
-
- BinaryUtils.writeIgniteUuid(out, matrixUuid);
- BinaryUtils.writeIgniteUuid(out, affinityKey);
- out.writeLong(blockId);
- }
-
- /** {@inheritDoc} */
- @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
- BinaryRawReader in = reader.rawReader();
-
- matrixUuid = BinaryUtils.readIgniteUuid(in);
- affinityKey = BinaryUtils.readIgniteUuid(in);
- blockId = in.readLong();
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return matrixUuid.hashCode() + (int)(blockId ^ (blockId >>> 32));
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object obj) {
- if (obj == this)
- return true;
-
- if (obj == null || obj.getClass() != getClass())
- return false;
-
- BlockMatrixKey that = (BlockMatrixKey)obj;
-
- return blockId == that.blockId && matrixUuid.equals(that.matrixUuid) && F.eq(affinityKey, that.affinityKey);
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(BlockMatrixKey.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java
index 979f223..0f285c2 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java
@@ -20,7 +20,7 @@ package org.apache.ignite.ml.math.impls.storage.matrix;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
-import java.util.Collection;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
@@ -37,7 +37,9 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.ml.math.MatrixStorage;
import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.impls.CacheUtils;
+import org.apache.ignite.ml.math.distributed.CacheUtils;
+import org.apache.ignite.ml.math.distributed.DistributedStorage;
+import org.apache.ignite.ml.math.distributed.keys.impl.BlockMatrixKey;
import org.apache.ignite.ml.math.impls.matrix.BlockEntry;
import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix;
@@ -46,9 +48,9 @@ import static org.apache.ignite.ml.math.impls.matrix.BlockEntry.MAX_BLOCK_SIZE;
/**
* Storage for {@link SparseBlockDistributedMatrix}.
*/
-public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, StorageConstants {
+public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, StorageConstants, DistributedStorage<BlockMatrixKey> {
/** Cache name used for all instances of {@link BlockMatrixStorage}. */
- public static final String ML_BLOCK_CACHE_NAME = "ML_BLOCK_SPARSE_MATRICES_CONTAINER";
+ private static final String CACHE_NAME = "ML_BLOCK_SPARSE_MATRICES_CONTAINER";
/** */
private int blocksInCol;
/** */
@@ -129,6 +131,11 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto
}
/** {@inheritDoc} */
+ @Override public int accessMode() {
+ return RANDOM_ACCESS_MODE;
+ }
+
+ /** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
out.writeInt(rows);
out.writeInt(cols);
@@ -256,12 +263,10 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto
return getColForBlock(locBlock);
}
- /**
- * Build a keyset for this matrix storage.
- */
- public Collection<BlockMatrixKey> getAllKeys() {
+ /** {@inheritDoc} */
+ @Override public Set<BlockMatrixKey> getAllKeys() {
long maxBlockId = numberOfBlocks();
- Collection<BlockMatrixKey> keys = new LinkedList<>();
+ Set<BlockMatrixKey> keys = new HashSet<>();
for (long id = 0; id < maxBlockId; id++)
keys.add(getCacheKey(id));
@@ -269,6 +274,11 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto
return keys;
}
+ /** {@inheritDoc} */
+ @Override public String cacheName() {
+ return CACHE_NAME;
+ }
+
/** */
private List<BlockEntry> getRowForBlock(long blockId) {
List<BlockEntry> res = new LinkedList<>();
@@ -348,8 +358,8 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto
private void matrixSet(int a, int b, double v) {
long id = getBlockId(a, b);
// Remote set on the primary node (where given row or column is stored locally).
- ignite().compute(groupForKey(ML_BLOCK_CACHE_NAME, id)).run(() -> {
- IgniteCache<BlockMatrixKey, BlockEntry> cache = Ignition.localIgnite().getOrCreateCache(ML_BLOCK_CACHE_NAME);
+ ignite().compute(groupForKey(CACHE_NAME, id)).run(() -> {
+ IgniteCache<BlockMatrixKey, BlockEntry> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME);
BlockMatrixKey key = getCacheKey(getBlockId(a, b));
@@ -396,8 +406,8 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto
*/
private double matrixGet(int a, int b) {
// Remote get from the primary node (where given row or column is stored locally).
- return ignite().compute(groupForKey(ML_BLOCK_CACHE_NAME, getBlockId(a, b))).call(() -> {
- IgniteCache<BlockMatrixKey, BlockEntry> cache = Ignition.localIgnite().getOrCreateCache(ML_BLOCK_CACHE_NAME);
+ return ignite().compute(groupForKey(CACHE_NAME, getBlockId(a, b))).call(() -> {
+ IgniteCache<BlockMatrixKey, BlockEntry> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME);
BlockMatrixKey key = getCacheKey(getBlockId(a, b));
@@ -433,7 +443,7 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto
cfg.setCacheMode(CacheMode.PARTITIONED);
// Random cache name.
- cfg.setName(ML_BLOCK_CACHE_NAME);
+ cfg.setName(CACHE_NAME);
return Ignition.localIgnite().getOrCreateCache(cfg);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/CacheMatrixStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/CacheMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/CacheMatrixStorage.java
index 05f3c21..fbad957 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/CacheMatrixStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/CacheMatrixStorage.java
@@ -22,10 +22,10 @@ import java.io.ObjectInput;
import java.io.ObjectOutput;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
-import org.apache.ignite.ml.math.MatrixKeyMapper;
import org.apache.ignite.ml.math.MatrixStorage;
import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.ValueMapper;
+import org.apache.ignite.ml.math.distributed.MatrixKeyMapper;
+import org.apache.ignite.ml.math.distributed.ValueMapper;
/**
* Matrix storage based on arbitrary cache and key and value mapping functions.
@@ -118,6 +118,11 @@ public class CacheMatrixStorage<K, V> implements MatrixStorage {
}
/** {@inheritDoc} */
+ @Override public int accessMode() {
+ return StorageConstants.RANDOM_ACCESS_MODE;
+ }
+
+ /** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
out.writeInt(rows);
out.writeInt(cols);
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java
index 921544e..6749488 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java
@@ -123,6 +123,11 @@ public class DenseOffHeapMatrixStorage implements MatrixStorage {
}
/** {@inheritDoc} */
+ @Override public int accessMode() {
+ return StorageConstants.RANDOM_ACCESS_MODE;
+ }
+
+ /** {@inheritDoc} */
@Override public boolean isArrayBased() {
return false;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DiagonalMatrixStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DiagonalMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DiagonalMatrixStorage.java
index d313c45..a0f102a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DiagonalMatrixStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DiagonalMatrixStorage.java
@@ -84,6 +84,11 @@ public class DiagonalMatrixStorage implements MatrixStorage {
}
/** {@inheritDoc} */
+ @Override public int accessMode() {
+ return StorageConstants.RANDOM_ACCESS_MODE;
+ }
+
+ /** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
out.writeObject(diagonal);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/FunctionMatrixStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/FunctionMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/FunctionMatrixStorage.java
index ac7fa51..18cc108 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/FunctionMatrixStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/FunctionMatrixStorage.java
@@ -158,6 +158,11 @@ public class FunctionMatrixStorage implements MatrixStorage {
}
/** {@inheritDoc} */
+ @Override public int accessMode() {
+ return StorageConstants.RANDOM_ACCESS_MODE;
+ }
+
+ /** {@inheritDoc} */
@Override public boolean equals(Object o) {
if (this == o)
return true;
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixDelegateStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixDelegateStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixDelegateStorage.java
index 7a9c2e5..7dc37cd 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixDelegateStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixDelegateStorage.java
@@ -130,6 +130,11 @@ public class MatrixDelegateStorage implements MatrixStorage {
}
/** {@inheritDoc} */
+ @Override public int accessMode() {
+ return dlg.accessMode();
+ }
+
+ /** {@inheritDoc} */
@Override public boolean isArrayBased() {
return dlg.isArrayBased() && rowOff == 0 && colOff == 0;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/PivotedMatrixStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/PivotedMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/PivotedMatrixStorage.java
index 749a508..387b347 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/PivotedMatrixStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/PivotedMatrixStorage.java
@@ -165,6 +165,11 @@ public class PivotedMatrixStorage implements MatrixStorage {
}
/** {@inheritDoc} */
+ @Override public int accessMode() {
+ return sto.accessMode();
+ }
+
+ /** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
out.writeObject(sto);
out.writeObject(rowPivot);
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/RandomMatrixStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/RandomMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/RandomMatrixStorage.java
index 1435629..56bd871 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/RandomMatrixStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/RandomMatrixStorage.java
@@ -156,6 +156,11 @@ public class RandomMatrixStorage implements MatrixStorage {
}
/** {@inheritDoc} */
+ @Override public int accessMode() {
+ return StorageConstants.RANDOM_ACCESS_MODE;
+ }
+
+ /** {@inheritDoc} */
@Override public int hashCode() {
int res = 1;
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java
index 5716a1a..95852b7 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java
@@ -33,19 +33,21 @@ import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.ml.math.MatrixStorage;
import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.impls.CacheUtils;
+import org.apache.ignite.ml.math.distributed.CacheUtils;
+import org.apache.ignite.ml.math.distributed.DistributedStorage;
+import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey;
+import org.apache.ignite.ml.math.distributed.keys.impl.SparseMatrixKey;
import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
/**
* {@link MatrixStorage} implementation for {@link SparseDistributedMatrix}.
*/
-public class SparseDistributedMatrixStorage extends CacheUtils implements MatrixStorage, StorageConstants {
+public class SparseDistributedMatrixStorage extends CacheUtils implements MatrixStorage, StorageConstants, DistributedStorage<RowColMatrixKey> {
/** Cache name used for all instances of {@link SparseDistributedMatrixStorage}. */
- public static final String ML_CACHE_NAME = "ML_SPARSE_MATRICES_CONTAINER";
+ private static final String CACHE_NAME = "ML_SPARSE_MATRICES_CONTAINER";
/** Amount of rows in the matrix. */
private int rows;
/** Amount of columns in the matrix. */
@@ -59,7 +61,7 @@ public class SparseDistributedMatrixStorage extends CacheUtils implements Matrix
/** Actual distributed storage. */
private IgniteCache<
- IgniteBiTuple<Integer, IgniteUuid> /* Row or column index with matrix uuid. */,
+ RowColMatrixKey /* Row or column index with matrix uuid. */,
Map<Integer, Double> /* Map-based row or column. */
> cache = null;
@@ -95,8 +97,8 @@ public class SparseDistributedMatrixStorage extends CacheUtils implements Matrix
/**
* Create new ML cache if needed.
*/
- private IgniteCache<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> newCache() {
- CacheConfiguration<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> cfg = new CacheConfiguration<>();
+ private IgniteCache<RowColMatrixKey, Map<Integer, Double>> newCache() {
+ CacheConfiguration<RowColMatrixKey, Map<Integer, Double>> cfg = new CacheConfiguration<>();
// Write to primary.
cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
@@ -114,24 +116,20 @@ public class SparseDistributedMatrixStorage extends CacheUtils implements Matrix
cfg.setCacheMode(CacheMode.PARTITIONED);
// Random cache name.
- cfg.setName(ML_CACHE_NAME);
+ cfg.setName(CACHE_NAME);
return Ignition.localIgnite().getOrCreateCache(cfg);
}
/**
*
- *
*/
- public IgniteCache<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> cache() {
+ public IgniteCache<RowColMatrixKey, Map<Integer, Double>> cache() {
return cache;
}
- /**
- *
- *
- */
- public int accessMode() {
+ /** {@inheritDoc} */
+ @Override public int accessMode() {
return acsMode;
}
@@ -160,8 +158,8 @@ public class SparseDistributedMatrixStorage extends CacheUtils implements Matrix
*/
private double matrixGet(int a, int b) {
// Remote get from the primary node (where given row or column is stored locally).
- return ignite().compute(groupForKey(ML_CACHE_NAME, a)).call(() -> {
- IgniteCache<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> cache = Ignition.localIgnite().getOrCreateCache(ML_CACHE_NAME);
+ return ignite().compute(groupForKey(CACHE_NAME, a)).call(() -> {
+ IgniteCache<RowColMatrixKey, Map<Integer, Double>> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME);
// Local get.
Map<Integer, Double> map = cache.localPeek(getCacheKey(a), CachePeekMode.PRIMARY);
@@ -182,8 +180,8 @@ public class SparseDistributedMatrixStorage extends CacheUtils implements Matrix
*/
private void matrixSet(int a, int b, double v) {
// Remote set on the primary node (where given row or column is stored locally).
- ignite().compute(groupForKey(ML_CACHE_NAME, a)).run(() -> {
- IgniteCache<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> cache = Ignition.localIgnite().getOrCreateCache(ML_CACHE_NAME);
+ ignite().compute(groupForKey(CACHE_NAME, a)).run(() -> {
+ IgniteCache<RowColMatrixKey, Map<Integer, Double>> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME);
// Local get.
Map<Integer, Double> map = cache.localPeek(getCacheKey(a), CachePeekMode.PRIMARY);
@@ -206,8 +204,8 @@ public class SparseDistributedMatrixStorage extends CacheUtils implements Matrix
}
/** Build cache key for row/column. */
- private IgniteBiTuple<Integer, IgniteUuid> getCacheKey(int idx) {
- return new IgniteBiTuple<>(idx, uuid);
+ public RowColMatrixKey getCacheKey(int idx) {
+ return new SparseMatrixKey(idx, uuid, null);
}
/** {@inheritDoc} */
@@ -272,7 +270,7 @@ public class SparseDistributedMatrixStorage extends CacheUtils implements Matrix
/** Delete all data from cache. */
@Override public void destroy() {
- Set<IgniteBiTuple<Integer, IgniteUuid>> keyset = IntStream.range(0, rows).mapToObj(this::getCacheKey).collect(Collectors.toSet());
+ Set<RowColMatrixKey> keyset = IntStream.range(0, rows).mapToObj(this::getCacheKey).collect(Collectors.toSet());
cache.clearAll(keyset);
}
@@ -309,4 +307,16 @@ public class SparseDistributedMatrixStorage extends CacheUtils implements Matrix
public IgniteUuid getUUID() {
return uuid;
}
+
+ /** {@inheritDoc} */
+ @Override public Set<RowColMatrixKey> getAllKeys() {
+ int range = stoMode == ROW_STORAGE_MODE ? rows : cols;
+
+ return IntStream.range(0, range).mapToObj(i -> new SparseMatrixKey(i, getUUID(), null)).collect(Collectors.toSet());
+ }
+
+ /** {@inheritDoc} */
+ @Override public String cacheName() {
+ return CACHE_NAME;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseLocalOnHeapMatrixStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseLocalOnHeapMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseLocalOnHeapMatrixStorage.java
index 99ef6fc..5e68b6c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseLocalOnHeapMatrixStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseLocalOnHeapMatrixStorage.java
@@ -75,10 +75,8 @@ public class SparseLocalOnHeapMatrixStorage implements MatrixStorage, StorageCon
return stoMode;
}
- /**
- * @return Matrix elements access mode.
- */
- public int getAccessMode() {
+ /** {@inheritDoc} */
+ @Override public int accessMode() {
return acsMode;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/CacheVectorStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/CacheVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/CacheVectorStorage.java
index c0c7152..c4bb995 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/CacheVectorStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/CacheVectorStorage.java
@@ -22,9 +22,9 @@ import java.io.ObjectInput;
import java.io.ObjectOutput;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
-import org.apache.ignite.ml.math.ValueMapper;
-import org.apache.ignite.ml.math.VectorKeyMapper;
import org.apache.ignite.ml.math.VectorStorage;
+import org.apache.ignite.ml.math.distributed.ValueMapper;
+import org.apache.ignite.ml.math.distributed.VectorKeyMapper;
/**
* Vector storage based on existing cache and index and value mapping functions.
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/CacheVector.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/CacheVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/CacheVector.java
index e0a1a9d..676f271 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/CacheVector.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/CacheVector.java
@@ -19,14 +19,14 @@ package org.apache.ignite.ml.math.impls.vector;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.ValueMapper;
import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.VectorKeyMapper;
+import org.apache.ignite.ml.math.distributed.CacheUtils;
+import org.apache.ignite.ml.math.distributed.ValueMapper;
+import org.apache.ignite.ml.math.distributed.VectorKeyMapper;
import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.functions.IgniteDoubleFunction;
import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.CacheUtils;
import org.apache.ignite.ml.math.impls.storage.vector.CacheVectorStorage;
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrixTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrixTest.java
index a7e9488..c6f6f86 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrixTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrixTest.java
@@ -24,7 +24,7 @@ import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.ml.math.ExternalizeTest;
import org.apache.ignite.ml.math.IdentityValueMapper;
import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.MatrixKeyMapper;
+import org.apache.ignite.ml.math.distributed.MatrixKeyMapper;
import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
import org.apache.ignite.ml.math.impls.MathTestConstants;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixKeyMapperForTests.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixKeyMapperForTests.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixKeyMapperForTests.java
index cfdd0f3..10e6e3f 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixKeyMapperForTests.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixKeyMapperForTests.java
@@ -16,7 +16,7 @@
*/
package org.apache.ignite.ml.math.impls.matrix;
-import org.apache.ignite.ml.math.MatrixKeyMapper;
+import org.apache.ignite.ml.math.distributed.MatrixKeyMapper;
/** */
public class MatrixKeyMapperForTests implements MatrixKeyMapper<Integer> {
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java
index 1228f05..56ff638 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java
@@ -30,9 +30,10 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.distributed.DistributedStorage;
+import org.apache.ignite.ml.math.distributed.keys.impl.BlockMatrixKey;
import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
import org.apache.ignite.ml.math.impls.MathTestConstants;
-import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey;
import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixStorage;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.testframework.junits.common.GridCommonTest;
@@ -229,9 +230,9 @@ public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest {
Collection<String> cacheNames = ignite.cacheNames();
- assert cacheNames.contains(BlockMatrixStorage.ML_BLOCK_CACHE_NAME);
+ assert cacheNames.contains(((DistributedStorage)cacheMatrix1.getStorage()).cacheName());
- IgniteCache<BlockMatrixKey, Object> cache = ignite.getOrCreateCache(BlockMatrixStorage.ML_BLOCK_CACHE_NAME);
+ IgniteCache<BlockMatrixKey, Object> cache = ignite.getOrCreateCache(((DistributedStorage)cacheMatrix1.getStorage()).cacheName());
Set<BlockMatrixKey> keySet1 = buildKeySet(cacheMatrix1);
Set<BlockMatrixKey> keySet2 = buildKeySet(cacheMatrix2);
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java
index 3fec83c..1955588 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java
@@ -23,16 +23,15 @@ import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.Collection;
-import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.StorageConstants;
+import org.apache.ignite.ml.math.distributed.DistributedStorage;
+import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey;
import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
import org.apache.ignite.ml.math.impls.MathTestConstants;
import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
@@ -235,12 +234,12 @@ public class SparseDistributedMatrixTest extends GridCommonAbstractTest {
Collection<String> cacheNames = ignite.cacheNames();
- assert cacheNames.contains(SparseDistributedMatrixStorage.ML_CACHE_NAME);
+ assert cacheNames.contains(((DistributedStorage)cacheMatrix1.getStorage()).cacheName());
- IgniteCache<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> cache = ignite.getOrCreateCache(SparseDistributedMatrixStorage.ML_CACHE_NAME);
+ IgniteCache<RowColMatrixKey, Map<Integer, Double>> cache = ignite.getOrCreateCache(((DistributedStorage)cacheMatrix1.getStorage()).cacheName());
- Set<IgniteBiTuple<Integer, IgniteUuid>> keySet1 = buildKeySet(cacheMatrix1);
- Set<IgniteBiTuple<Integer, IgniteUuid>> keySet2 = buildKeySet(cacheMatrix2);
+ Set<RowColMatrixKey> keySet1 = ((SparseDistributedMatrixStorage)cacheMatrix1.getStorage()).getAllKeys();
+ Set<RowColMatrixKey> keySet2 = ((SparseDistributedMatrixStorage)cacheMatrix2.getStorage()).getAllKeys();
assert cache.containsKeys(keySet1) ||
keySet1.stream().allMatch(k -> cache.invoke(k, (entry, arguments) -> entry.getKey().equals(k) && entry.getValue().size() == 100));
@@ -289,18 +288,20 @@ public class SparseDistributedMatrixTest extends GridCommonAbstractTest {
public void testMatrixTimes(){
IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- SparseDistributedMatrix cacheMatrix1 = new SparseDistributedMatrix(MATRIX_SIZE, MATRIX_SIZE, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
- SparseDistributedMatrix cacheMatrix2 = new SparseDistributedMatrix(MATRIX_SIZE, MATRIX_SIZE, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
+ int size = MATRIX_SIZE;
- for (int i = 0; i < MATRIX_SIZE; i++) {
+ SparseDistributedMatrix cacheMatrix1 = new SparseDistributedMatrix(size, size, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
+ SparseDistributedMatrix cacheMatrix2 = new SparseDistributedMatrix(size, size, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
+
+ for (int i = 0; i < size; i++) {
cacheMatrix1.setX(i, i, i);
cacheMatrix2.setX(i, i, i);
}
Matrix res = cacheMatrix1.times(cacheMatrix2);
- for(int i = 0; i < MATRIX_SIZE; i++)
- for(int j = 0; j < MATRIX_SIZE; j++)
+ for(int i = 0; i < size; i++)
+ for(int j = 0; j < size; j++)
if (i == j)
assertEquals(UNEXPECTED_VAL, i * i, res.get(i, j), PRECISION);
else
@@ -313,19 +314,4 @@ public class SparseDistributedMatrixTest extends GridCommonAbstractTest {
for (int j = 0; j < m.columnSize(); j++)
m.set(i, j, 1.0);
}
-
- /** Build key set for SparseDistributedMatrix. */
- private Set<IgniteBiTuple<Integer, IgniteUuid>> buildKeySet(SparseDistributedMatrix m) {
- Set<IgniteBiTuple<Integer, IgniteUuid>> set = new HashSet<>();
-
- SparseDistributedMatrixStorage storage = (SparseDistributedMatrixStorage)m.getStorage();
-
- IgniteUuid uuid = storage.getUUID();
- int size = storage.storageMode() == StorageConstants.ROW_STORAGE_MODE ? storage.rowSize() : storage.columnSize();
-
- for (int i = 0; i < size; i++)
- set.add(new IgniteBiTuple<>(i, uuid));
-
- return set;
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/CacheVectorTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/CacheVectorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/CacheVectorTest.java
index b5813d7..a6cdd4c 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/CacheVectorTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/CacheVectorTest.java
@@ -30,7 +30,7 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.ml.math.IdentityValueMapper;
import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.VectorKeyMapper;
+import org.apache.ignite.ml.math.distributed.VectorKeyMapper;
import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
import org.apache.ignite.ml.math.functions.Functions;
import org.apache.ignite.ml.math.impls.MathTestConstants;
[14/18] ignite git commit: ignite-6124 DataStreamerImpl: do not wait
for exchange future inside cache gateway.
Posted by sb...@apache.org.
ignite-6124 DataStreamerImpl: do not wait for exchange future inside cache gateway.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3ab523cd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3ab523cd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3ab523cd
Branch: refs/heads/ignite-6149
Commit: 3ab523cd0f9c8bf962e6069596002204bcc49886
Parents: e1bf8d7
Author: sboikov <sb...@gridgain.com>
Authored: Thu Aug 24 14:09:12 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Aug 24 14:09:12 2017 +0300
----------------------------------------------------------------------
.../processors/datastreamer/DataStreamerImpl.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ab523cd/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index aee0d26..6ed552a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -764,6 +764,13 @@ public class DataStreamerImpl<K, V> implements IgniteDataStreamer<K, V>, Delayed
GridCacheGateway gate = null;
+ AffinityTopologyVersion topVer;
+
+ if (!cctx.isLocal())
+ topVer = ctx.cache().context().exchange().lastTopologyFuture().get();
+ else
+ topVer = ctx.cache().context().exchange().readyAffinityVersion();
+
if (!allowOverwrite() && !cctx.isLocal()) { // Cases where cctx required.
gate = cctx.gate();
@@ -771,13 +778,6 @@ public class DataStreamerImpl<K, V> implements IgniteDataStreamer<K, V>, Delayed
}
try {
- AffinityTopologyVersion topVer;
-
- if (!cctx.isLocal())
- topVer = ctx.cache().context().exchange().lastTopologyFuture().get();
- else
- topVer = ctx.cache().context().exchange().readyAffinityVersion();
-
for (DataStreamerEntry entry : entries) {
List<ClusterNode> nodes;
[16/18] ignite git commit: IGNITE-5200 Web Console: Don't cache
generated chunks in production.
Posted by sb...@apache.org.
IGNITE-5200 Web Console: Don't cache generated chunks in production.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e1eb1b97
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e1eb1b97
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e1eb1b97
Branch: refs/heads/ignite-6149
Commit: e1eb1b97f541f944492f4860d33ad7899aafef87
Parents: 24306ba
Author: Andrey Novikov <an...@gridgain.com>
Authored: Fri Aug 25 10:48:15 2017 +0700
Committer: Andrey Novikov <an...@gridgain.com>
Committed: Fri Aug 25 10:48:15 2017 +0700
----------------------------------------------------------------------
modules/web-console/frontend/webpack/webpack.common.js | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e1eb1b97/modules/web-console/frontend/webpack/webpack.common.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/webpack/webpack.common.js b/modules/web-console/frontend/webpack/webpack.common.js
index 5a3763e..48e1e9b 100644
--- a/modules/web-console/frontend/webpack/webpack.common.js
+++ b/modules/web-console/frontend/webpack/webpack.common.js
@@ -37,7 +37,6 @@ const app = path.resolve('app');
const IgniteModules = process.env.IGNITE_MODULES ? path.join(process.env.IGNITE_MODULES, 'frontend') : path.resolve('ignite_modules');
export default {
- cache: true,
node: {
fs: 'empty'
},
[06/18] ignite git commit: IGNITE-6169: Fixed thin JDBC driver
compatibility problem.
Posted by sb...@apache.org.
IGNITE-6169: Fixed thin JDBC driver compatibility problem.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d22631e6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d22631e6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d22631e6
Branch: refs/heads/ignite-6149
Commit: d22631e647280eff9f661dd7cfd492a6ba7460af
Parents: bebe4d8
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Wed Aug 23 15:46:23 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Aug 23 15:46:23 2017 +0300
----------------------------------------------------------------------
.../internal/jdbc/thin/JdbcThinTcpIo.java | 37 +++++++++++---------
1 file changed, 20 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d22631e6/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
index e124921..fdec77e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
@@ -219,30 +219,33 @@ public class JdbcThinTcpIo {
boolean accepted = reader.readBoolean();
if (accepted) {
- byte maj = reader.readByte();
- byte min = reader.readByte();
- byte maintenance = reader.readByte();
+ if (reader.available() > 0) {
+ byte maj = reader.readByte();
+ byte min = reader.readByte();
+ byte maintenance = reader.readByte();
- String stage = reader.readString();
+ String stage = reader.readString();
- long ts = reader.readLong();
- byte[] hash = reader.readByteArray();
+ long ts = reader.readLong();
+ byte[] hash = reader.readByteArray();
- igniteVer = new IgniteProductVersion(maj, min, maintenance, stage, ts, hash);
-
- return;
+ igniteVer = new IgniteProductVersion(maj, min, maintenance, stage, ts, hash);
+ }
+ else
+ igniteVer = new IgniteProductVersion((byte)2, (byte)0, (byte)0, "Unknown", 0L, null);
}
+ else {
+ short maj = reader.readShort();
+ short min = reader.readShort();
+ short maintenance = reader.readShort();
- short maj = reader.readShort();
- short min = reader.readShort();
- short maintenance = reader.readShort();
+ String err = reader.readString();
- String err = reader.readString();
+ SqlListenerProtocolVersion ver = SqlListenerProtocolVersion.create(maj, min, maintenance);
- SqlListenerProtocolVersion ver = SqlListenerProtocolVersion.create(maj, min, maintenance);
-
- throw new IgniteCheckedException("Handshake failed [driverProtocolVer=" + CURRENT_VER +
- ", remoteNodeProtocolVer=" + ver + ", err=" + err + ']');
+ throw new IgniteCheckedException("Handshake failed [driverProtocolVer=" + CURRENT_VER +
+ ", remoteNodeProtocolVer=" + ver + ", err=" + err + ']');
+ }
}
/**
[05/18] ignite git commit: ignite-6124 Guard logging with
isInfoEnabled
Posted by sb...@apache.org.
ignite-6124 Guard logging with isInfoEnabled
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bebe4d87
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bebe4d87
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bebe4d87
Branch: refs/heads/ignite-6149
Commit: bebe4d872cd687f793596ee1b2067f777e9ce46e
Parents: 129be29
Author: sboikov <sb...@gridgain.com>
Authored: Wed Aug 23 15:19:52 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Wed Aug 23 15:19:52 2017 +0300
----------------------------------------------------------------------
.../GridDhtPartitionsExchangeFuture.java | 203 ++++++++++++-------
.../dht/preloader/InitNewCoordinatorFuture.java | 38 +++-
2 files changed, 160 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bebe4d87/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 8e0deb9..0dd6e4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -677,7 +677,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
if (cctx.localNode().isClient())
tryToPerformLocalSnapshotOperation();
- exchLog.info("Finished exchange init [topVer=" + topVer + ", crd=" + crdNode + ']');
+ if (exchLog.isInfoEnabled())
+ exchLog.info("Finished exchange init [topVer=" + topVer + ", crd=" + crdNode + ']');
}
catch (IgniteInterruptedCheckedException e) {
onDone(e);
@@ -1397,9 +1398,11 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
if (isDone() || !done.compareAndSet(false, true))
return false;
- log.info("Finish exchange future [startVer=" + initialVersion() +
- ", resVer=" + res +
- ", err=" + err + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Finish exchange future [startVer=" + initialVersion() +
+ ", resVer=" + res +
+ ", err=" + err + ']');
+ }
assert res != null || err != null;
@@ -1588,15 +1591,19 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
if (msg != null) {
assert msg.exchangeId().topologyVersion().equals(new AffinityTopologyVersion(node.order()));
- log.info("Merge server join exchange, message received [curFut=" + initialVersion() +
- ", node=" + nodeId + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Merge server join exchange, message received [curFut=" + initialVersion() +
+ ", node=" + nodeId + ']');
+ }
mergedJoinExchMsgs.put(nodeId, msg);
}
else {
if (cctx.discovery().alive(nodeId)) {
- log.info("Merge server join exchange, wait for message [curFut=" + initialVersion() +
- ", node=" + nodeId + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Merge server join exchange, wait for message [curFut=" + initialVersion() +
+ ", node=" + nodeId + ']');
+ }
wait = true;
@@ -1605,8 +1612,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
awaitMergedMsgs++;
}
else {
- log.info("Merge server join exchange, awaited node left [curFut=" + initialVersion() +
- ", node=" + nodeId + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Merge server join exchange, awaited node left [curFut=" + initialVersion() +
+ ", node=" + nodeId + ']');
+ }
}
}
}
@@ -1684,11 +1693,13 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
mergedJoinExchMsgs.containsKey(node.id()) &&
mergedJoinExchMsgs.get(node.id()) == null;
- log.info("Merge server join exchange, received message [curFut=" + initialVersion() +
- ", node=" + node.id() +
- ", msgVer=" + msg.exchangeId().topologyVersion() +
- ", process=" + process +
- ", awaited=" + awaitMergedMsgs + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Merge server join exchange, received message [curFut=" + initialVersion() +
+ ", node=" + node.id() +
+ ", msgVer=" + msg.exchangeId().topologyVersion() +
+ ", process=" + process +
+ ", awaited=" + awaitMergedMsgs + ']');
+ }
if (process) {
mergedJoinExchMsgs.put(node.id(), msg);
@@ -1842,8 +1853,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
switch (state) {
case DONE: {
- log.info("Received single message, already done [ver=" + initialVersion() +
- ", node=" + nodeId + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Received single message, already done [ver=" + initialVersion() +
+ ", node=" + nodeId + ']');
+ }
assert finishState != null;
@@ -1865,9 +1878,11 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
allReceived = remaining.isEmpty();
- log.info("Coordinator received single message [ver=" + initialVersion() +
- ", node=" + nodeId +
- ", allReceived=" + allReceived + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Coordinator received single message [ver=" + initialVersion() +
+ ", node=" + nodeId +
+ ", allReceived=" + allReceived + ']');
+ }
}
break;
@@ -1875,8 +1890,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
case SRV:
case BECOME_CRD: {
- log.info("Non-coordinator received single message [ver=" + initialVersion() +
- ", node=" + nodeId + ", state=" + state + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Non-coordinator received single message [ver=" + initialVersion() +
+ ", node=" + nodeId + ", state=" + state + ']');
+ }
pendingSingleMsgs.put(nodeId, msg);
@@ -2170,7 +2187,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
if (exchCtx.mergeExchanges()) {
- log.info("Coordinator received all messages, try merge [ver=" + initialVersion() + ']');
+ if (log.isInfoEnabled())
+ log.info("Coordinator received all messages, try merge [ver=" + initialVersion() + ']');
boolean finish = cctx.exchange().mergeExchangesOnCoordinator(this);
@@ -2195,8 +2213,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
try {
AffinityTopologyVersion resTopVer = exchCtx.events().topologyVersion();
- log.info("finishExchangeOnCoordinator [topVer=" + initialVersion() +
- ", resVer=" + resTopVer + ']');
+ if (log.isInfoEnabled()) {
+ log.info("finishExchangeOnCoordinator [topVer=" + initialVersion() +
+ ", resVer=" + resTopVer + ']');
+ }
Map<Integer, CacheGroupAffinityMessage> idealAffDiff = null;
@@ -2394,9 +2414,11 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
onDone(exchCtx.events().topologyVersion(), err);
for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> e : pendingSingleMsgs.entrySet()) {
- log.info("Process pending message on coordinator [node=" + e.getKey() +
- ", ver=" + initialVersion() +
- ", resVer=" + resTopVer + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Process pending message on coordinator [node=" + e.getKey() +
+ ", ver=" + initialVersion() +
+ ", resVer=" + resTopVer + ']');
+ }
processSingleMessage(e.getKey(), e.getValue());
}
@@ -2519,7 +2541,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
synchronized (mux) {
if (crd == null) {
- log.info("Ignore partitions request, no coordinator [node=" + node.id() + ']');
+ if (log.isInfoEnabled())
+ log.info("Ignore partitions request, no coordinator [node=" + node.id() + ']');
return;
}
@@ -2529,7 +2552,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
assert finishState != null;
if (node.id().equals(finishState.crdId)) {
- log.info("Ignore partitions request, finished exchange with this coordinator: " + msg);
+ if (log.isInfoEnabled())
+ log.info("Ignore partitions request, finished exchange with this coordinator: " + msg);
return;
}
@@ -2541,7 +2565,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
case CRD:
case BECOME_CRD: {
- log.info("Ignore partitions request, node is coordinator: " + msg);
+ if (log.isInfoEnabled())
+ log.info("Ignore partitions request, node is coordinator: " + msg);
return;
}
@@ -2549,7 +2574,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
case CLIENT:
case SRV: {
if (!cctx.discovery().alive(node)) {
- log.info("Ignore partitions request, node is not alive [node=" + node.id() + ']');
+ if (log.isInfoEnabled())
+ log.info("Ignore partitions request, node is not alive [node=" + node.id() + ']');
return;
}
@@ -2557,14 +2583,18 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
if (msg.restoreState()) {
if (!node.equals(crd)) {
if (node.order() > crd.order()) {
- log.info("Received partitions request, change coordinator [oldCrd=" + crd.id() +
- ", newCrd=" + node.id() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Received partitions request, change coordinator [oldCrd=" + crd.id() +
+ ", newCrd=" + node.id() + ']');
+ }
crd = node; // Do not allow to process FullMessage from old coordinator.
}
else {
- log.info("Ignore restore state request, coordinator changed [oldCrd=" + crd.id() +
- ", newCrd=" + node.id() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Ignore restore state request, coordinator changed [oldCrd=" + crd.id() +
+ ", newCrd=" + node.id() + ']');
+ }
return;
}
@@ -2595,10 +2625,12 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
res.restoreState(true);
- log.info("Send restore state response [node=" + node.id() +
- ", exchVer=" + msg.restoreExchangeId().topologyVersion() +
- ", hasState=" + (finishState0 != null) +
- ", affReq=" + !F.isEmpty(res.cacheGroupsAffinityRequest()) + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Send restore state response [node=" + node.id() +
+ ", exchVer=" + msg.restoreExchangeId().topologyVersion() +
+ ", hasState=" + (finishState0 != null) +
+ ", affReq=" + !F.isEmpty(res.cacheGroupsAffinityRequest()) + ']');
+ }
res.finishMessage(finishState0 != null ? finishState0.msg : null);
@@ -2624,6 +2656,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
/**
+ * @param checkCrd If {@code true} checks that local node is exchange coordinator.
* @param node Sender node.
* @param msg Message.
*/
@@ -2637,7 +2670,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
synchronized (mux) {
if (crd == null) {
- log.info("Ignore full message, all server nodes left: " + msg);
+ if (log.isInfoEnabled())
+ log.info("Ignore full message, all server nodes left: " + msg);
return;
}
@@ -2645,13 +2679,15 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
switch (state) {
case CRD:
case BECOME_CRD: {
- log.info("Ignore full message, node is coordinator: " + msg);
+ if (log.isInfoEnabled())
+ log.info("Ignore full message, node is coordinator: " + msg);
return;
}
case DONE: {
- log.info("Ignore full message, future is done: " + msg);
+ if (log.isInfoEnabled())
+ log.info("Ignore full message, future is done: " + msg);
return;
}
@@ -2659,10 +2695,12 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
case SRV:
case CLIENT: {
if (!crd.equals(node)) {
- log.info("Received full message from non-coordinator [node=" + node.id() +
- ", nodeOrder=" + node.order() +
- ", crd=" + crd.id() +
- ", crdOrder=" + crd.order() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Received full message from non-coordinator [node=" + node.id() +
+ ", nodeOrder=" + node.order() +
+ ", crd=" + crd.id() +
+ ", crdOrder=" + crd.order() + ']');
+ }
if (node.order() > crd.order())
fullMsgs.put(node, msg);
@@ -2672,8 +2710,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
else {
AffinityTopologyVersion resVer = msg.resultTopologyVersion() != null ? msg.resultTopologyVersion() : initialVersion();
- log.info("Received full message, will finish exchange [node=" + node.id() +
- ", resVer=" + resVer + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Received full message, will finish exchange [node=" + node.id() +
+ ", resVer=" + resVer + ']');
+ }
finishState = new FinishState(crd.id(), resVer, msg);
@@ -2692,8 +2732,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
if (exchCtx.mergeExchanges()) {
if (msg.resultTopologyVersion() != null && !initialVersion().equals(msg.resultTopologyVersion())) {
- log.info("Received full message, need merge [curFut=" + initialVersion() +
- ", resVer=" + msg.resultTopologyVersion() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Received full message, need merge [curFut=" + initialVersion() +
+ ", resVer=" + msg.resultTopologyVersion() + ']');
+ }
resTopVer = msg.resultTopologyVersion();
@@ -3024,8 +3066,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
changeGlobalStateExceptions.put(crd0.id(), changeGlobalStateE);
if (crdChanged) {
- log.info("Coordinator failed, node is new coordinator [ver=" + initialVersion() +
- ", prev=" + node.id() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Coordinator failed, node is new coordinator [ver=" + initialVersion() +
+ ", prev=" + node.id() + ']');
+ }
assert newCrdFut != null;
@@ -3069,10 +3113,12 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
if (crdChanged) {
for (Map.Entry<ClusterNode, GridDhtPartitionsFullMessage> m : fullMsgs.entrySet()) {
if (crd0.equals(m.getKey())) {
- log.info("Coordinator changed, process pending full message [" +
- "ver=" + initialVersion() +
- ", crd=" + node.id() +
- ", pendingMsgNode=" + m.getKey() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Coordinator changed, process pending full message [" +
+ "ver=" + initialVersion() +
+ ", crd=" + node.id() +
+ ", pendingMsgNode=" + m.getKey() + ']');
+ }
processFullMessage(true, m.getKey(), m.getValue());
@@ -3081,10 +3127,12 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
}
- log.info("Coordinator changed, send partitions to new coordinator [" +
- "ver=" + initialVersion() +
- ", crd=" + node.id() +
- ", newCrd=" + crd0.id() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Coordinator changed, send partitions to new coordinator [" +
+ "ver=" + initialVersion() +
+ ", crd=" + node.id() +
+ ", newCrd=" + crd0.id() + ']');
+ }
sendPartitions(crd0);
}
@@ -3121,8 +3169,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
assert msgs.isEmpty() : msgs;
if (fullMsg != null) {
- log.info("New coordinator restored state [ver=" + initialVersion() +
- ", resVer=" + fullMsg.resultTopologyVersion() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("New coordinator restored state [ver=" + initialVersion() +
+ ", resVer=" + fullMsg.resultTopologyVersion() + ']');
+ }
synchronized (mux) {
state = ExchangeLocalState.DONE;
@@ -3154,13 +3204,20 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
}
+ if (log.isInfoEnabled()) {
+ log.info("New coordinator sends full message [ver=" + initialVersion() +
+ ", resVer=" + fullMsg.resultTopologyVersion() +
+ ", nodes=" + F.nodeIds(msgs.keySet()) + ']');
+ }
+
sendAllPartitions(fullMsg, msgs.keySet(), null, joinedNodeAff);
}
return;
}
else {
- log.info("New coordinator restore state finished [ver=" + initialVersion() + ']');
+ if (log.isInfoEnabled())
+ log.info("New coordinator restore state finished [ver=" + initialVersion() + ']');
for (Map.Entry<ClusterNode, GridDhtPartitionsSingleMessage> e : newCrdFut.messages().entrySet()) {
GridDhtPartitionsSingleMessage msg = e.getValue();
@@ -3195,8 +3252,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
assert mergedJoinExchMsgs == null;
- log.info("New coordinator initialization finished [ver=" + initialVersion() +
- ", remaining=" + remaining + ']');
+ if (log.isInfoEnabled()) {
+ log.info("New coordinator initialization finished [ver=" + initialVersion() +
+ ", remaining=" + remaining + ']');
+ }
if (!remaining.isEmpty())
remaining0 = new HashSet<>(remaining);
@@ -3209,8 +3268,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
for (UUID nodeId : remaining0) {
try {
if (!pendingSingleMsgs.containsKey(nodeId)) {
- log.info("New coordinator sends request [ver=" + initialVersion() +
- ", node=" + nodeId + ']');
+ if (log.isInfoEnabled()) {
+ log.info("New coordinator sends request [ver=" + initialVersion() +
+ ", node=" + nodeId + ']');
+ }
cctx.io().send(nodeId, req, SYSTEM_POOL);
}
@@ -3226,8 +3287,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> m : pendingSingleMsgs.entrySet()) {
- log.info("New coordinator process pending message [ver=" + initialVersion() +
- ", node=" + m.getKey() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("New coordinator process pending message [ver=" + initialVersion() +
+ ", node=" + m.getKey() + ']');
+ }
processSingleMessage(m.getKey(), m.getValue());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/bebe4d87/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
index 42ce9b9..d0e619b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
@@ -147,8 +147,10 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
}
}
- log.info("Try restore exchange result [allNodes=" + awaited +
- ", joined=" + joinedNodes.keySet() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Try restore exchange result [allNodes=" + awaited +
+ ", joined=" + joinedNodes.keySet() + ']');
+ }
if (!nodes.isEmpty()) {
GridDhtPartitionsSingleRequest req = GridDhtPartitionsSingleRequest.restoreStateRequest(exchFut.exchangeId(),
@@ -179,6 +181,9 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
markInitialized();
}
+ /**
+ * @return {@code True} if new coordinator tried to restore exchange state.
+ */
boolean restoreState() {
return restoreState;
}
@@ -202,9 +207,11 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
* @param msg Message.
*/
public void onMessage(ClusterNode node, GridDhtPartitionsSingleMessage msg) {
- log.info("Init new coordinator, received response [node=" + node.id() +
- ", fullMsg=" + (msg.finishMessage() != null) +
- ", affReq=" + !F.isEmpty(msg.cacheGroupsAffinityRequest()) + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Init new coordinator, received response [node=" + node.id() +
+ ", fullMsg=" + (msg.finishMessage() != null) +
+ ", affReq=" + !F.isEmpty(msg.cacheGroupsAffinityRequest()) + ']');
+ }
assert msg.restoreState() : msg;
@@ -248,9 +255,11 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
if (msgVer != null) {
assert msgVer.topologyVersion().compareTo(initTopVer) > 0 : msgVer;
- log.info("Process joined node message [resVer=" + resVer +
- ", initTopVer=" + initTopVer +
- ", msgVer=" + msgVer.topologyVersion() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Process joined node message [resVer=" + resVer +
+ ", initTopVer=" + initTopVer +
+ ", msgVer=" + msgVer.topologyVersion() + ']');
+ }
if (msgVer.topologyVersion().compareTo(resVer) > 0)
it.remove();
@@ -270,8 +279,10 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
assert msgVer.topologyVersion().compareTo(initTopVer) > 0 : msgVer;
- log.info("Process joined node message [initTopVer=" + initTopVer +
- ", msgVer=" + msgVer.topologyVersion() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Process joined node message [initTopVer=" + initTopVer +
+ ", msgVer=" + msgVer.topologyVersion() + ']');
+ }
if (joinExchMsgs == null)
joinExchMsgs = new HashMap<>();
@@ -285,6 +296,10 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
}
}
+ /**
+ * @param nodeId Node ID.
+ * @return Single message for node joined after exchange start.
+ */
@Nullable GridDhtPartitionsSingleMessage joinExchangeMessage(UUID nodeId) {
return joinExchMsgs != null ? joinExchMsgs.get(nodeId) : null;
}
@@ -293,7 +308,8 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
* @param nodeId Failed node ID.
*/
public void onNodeLeft(UUID nodeId) {
- log.info("Init new coordinator, node left [node=" + nodeId + ']');
+ if (log.isInfoEnabled())
+ log.info("Init new coordinator, node left [node=" + nodeId + ']');
boolean done;
[04/18] ignite git commit: ignite-5872 Fixed backward compatibility
Posted by sb...@apache.org.
ignite-5872 Fixed backward compatibility
(cherry picked from commit cca9117)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/129be29e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/129be29e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/129be29e
Branch: refs/heads/ignite-6149
Commit: 129be29e96cfaba3bb7645c66981de62646f820c
Parents: fa42218
Author: sboikov <sb...@gridgain.com>
Authored: Mon Aug 21 18:39:12 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Wed Aug 23 12:22:15 2017 +0300
----------------------------------------------------------------------
.../GridCachePartitionExchangeManager.java | 47 ++++++++---
.../dht/GridClientPartitionTopology.java | 9 +++
.../dht/GridDhtPartitionTopology.java | 5 ++
.../dht/GridDhtPartitionTopologyImpl.java | 5 ++
.../CachePartitionFullCountersMap.java | 36 +++++++++
.../CachePartitionPartialCountersMap.java | 23 ++++++
.../GridDhtPartitionsExchangeFuture.java | 41 +++++++---
.../preloader/GridDhtPartitionsFullMessage.java | 84 +++++++++++++++++---
.../GridDhtPartitionsSingleMessage.java | 23 ++++--
.../IgniteDhtPartitionCountersMap.java | 14 ++--
.../IgniteDhtPartitionCountersMap2.java | 69 ++++++++++++++++
11 files changed, 315 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/129be29e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 200f677..984721b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -65,6 +65,8 @@ import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCach
import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionFullCountersMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.ForceRebalanceExchangeTask;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
@@ -973,7 +975,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
*/
private void sendAllPartitions(Collection<ClusterNode> nodes,
AffinityTopologyVersion msgTopVer) {
- GridDhtPartitionsFullMessage m = createPartitionsFullMessage(true, null, null, null, null);
+ GridDhtPartitionsFullMessage m = createPartitionsFullMessage(true, false, null, null, null, null);
m.topologyVersion(msgTopVer);
@@ -1000,6 +1002,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
/**
* @param compress {@code True} if possible to compress message (properly work only if prepareMarshall/
* finishUnmarshall methods are called).
+ * @param newCntrMap {@code True} if possible to use {@link CachePartitionFullCountersMap}.
* @param exchId Non-null exchange ID if message is created for exchange.
* @param lastVer Last version.
* @param partHistSuppliers Partition history suppliers map.
@@ -1008,6 +1011,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
*/
public GridDhtPartitionsFullMessage createPartitionsFullMessage(
boolean compress,
+ boolean newCntrMap,
@Nullable final GridDhtPartitionExchangeId exchId,
@Nullable GridCacheVersion lastVer,
@Nullable IgniteDhtPartitionHistorySuppliersMap partHistSuppliers,
@@ -1046,8 +1050,16 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
affCache.similarAffinityKey());
}
- if (exchId != null)
- m.addPartitionUpdateCounters(grp.groupId(), grp.topology().fullUpdateCounters());
+ if (exchId != null) {
+ CachePartitionFullCountersMap cntrsMap = grp.topology().fullUpdateCounters();
+
+ if (newCntrMap)
+ m.addPartitionUpdateCounters(grp.groupId(), cntrsMap);
+ else {
+ m.addPartitionUpdateCounters(grp.groupId(),
+ CachePartitionFullCountersMap.toCountersMap(cntrsMap));
+ }
+ }
}
}
@@ -1064,8 +1076,14 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
top.similarAffinityKey());
}
- if (exchId != null)
- m.addPartitionUpdateCounters(top.groupId(), top.fullUpdateCounters());
+ if (exchId != null) {
+ CachePartitionFullCountersMap cntrsMap = top.fullUpdateCounters();
+
+ if (newCntrMap)
+ m.addPartitionUpdateCounters(top.groupId(), cntrsMap);
+ else
+ m.addPartitionUpdateCounters(top.groupId(), CachePartitionFullCountersMap.toCountersMap(cntrsMap));
+ }
}
return m;
@@ -1119,6 +1137,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
GridDhtPartitionsSingleMessage m = createPartitionsSingleMessage(id,
cctx.kernalContext().clientNode(),
false,
+ false,
null);
if (log.isDebugEnabled())
@@ -1141,12 +1160,14 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
* @param exchangeId Exchange ID.
* @param clientOnlyExchange Client exchange flag.
* @param sndCounters {@code True} if need send partition update counters.
+ * @param newCntrMap {@code True} if possible to use {@link CachePartitionPartialCountersMap}.
* @return Message.
*/
public GridDhtPartitionsSingleMessage createPartitionsSingleMessage(
@Nullable GridDhtPartitionExchangeId exchangeId,
boolean clientOnlyExchange,
boolean sndCounters,
+ boolean newCntrMap,
ExchangeActions exchActions
) {
GridDhtPartitionsSingleMessage m = new GridDhtPartitionsSingleMessage(exchangeId,
@@ -1167,8 +1188,12 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
locMap,
grp.affinity().similarAffinityKey());
- if (sndCounters)
- m.partitionUpdateCounters(grp.groupId(), grp.topology().localUpdateCounters(true));
+ if (sndCounters) {
+ CachePartitionPartialCountersMap cntrsMap = grp.topology().localUpdateCounters(true);
+
+ m.addPartitionUpdateCounters(grp.groupId(),
+ newCntrMap ? cntrsMap : CachePartitionPartialCountersMap.toCountersMap(cntrsMap));
+ }
}
}
@@ -1185,8 +1210,12 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
locMap,
top.similarAffinityKey());
- if (sndCounters)
- m.partitionUpdateCounters(top.groupId(), top.localUpdateCounters(true));
+ if (sndCounters) {
+ CachePartitionPartialCountersMap cntrsMap = top.localUpdateCounters(true);
+
+ m.addPartitionUpdateCounters(top.groupId(),
+ newCntrMap ? cntrsMap : CachePartitionPartialCountersMap.toCountersMap(cntrsMap));
+ }
}
return m;
http://git-wip-us.apache.org/repos/asf/ignite/blob/129be29e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index 77792c7..c8856fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -115,6 +115,9 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
/** */
private volatile DiscoCache discoCache;
+ /** */
+ private final int parts;
+
/**
* @param cctx Context.
* @param grpId Group ID.
@@ -130,6 +133,7 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
this.cctx = cctx;
this.grpId = grpId;
this.similarAffKey = similarAffKey;
+ this.parts = parts;
topVer = AffinityTopologyVersion.NONE;
@@ -142,6 +146,11 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
cntrMap = new CachePartitionFullCountersMap(parts);
}
+ /** {@inheritDoc} */
+ @Override public int partitions() {
+ return parts;
+ }
+
/**
* @return Key to find caches with similar affinity.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/129be29e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index 22205ea..4ae68ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -43,6 +43,11 @@ import org.jetbrains.annotations.Nullable;
@GridToStringExclude
public interface GridDhtPartitionTopology {
/**
+ * @return Total cache partitions.
+ */
+ public int partitions();
+
+ /**
* Locks the topology, usually during mapping on locks or transactions.
*/
public void readLock();
http://git-wip-us.apache.org/repos/asf/ignite/blob/129be29e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 16fe012..f25ae21 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -161,6 +161,11 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
+ @Override public int partitions() {
+ return grp.affinityFunction().partitions();
+ }
+
+ /** {@inheritDoc} */
@Override public int groupId() {
return grp.groupId();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/129be29e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java
index 1384a55..ebc993c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java
@@ -19,6 +19,9 @@ package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
import java.io.Serializable;
import java.util.Arrays;
+import java.util.Map;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
/**
*
@@ -96,4 +99,37 @@ public class CachePartitionFullCountersMap implements Serializable {
Arrays.fill(initialUpdCntrs, 0);
Arrays.fill(updCntrs, 0);
}
+
+ /**
+ * @param map Full counters map.
+ * @return Regular java map with counters.
+ */
+ public static Map<Integer, T2<Long, Long>> toCountersMap(CachePartitionFullCountersMap map) {
+ int partsCnt = map.updCntrs.length;
+
+ Map<Integer, T2<Long, Long>> map0 = U.newHashMap(partsCnt);
+
+ for (int p = 0; p < partsCnt; p++)
+ map0.put(p, new T2<>(map.initialUpdCntrs[p], map.updCntrs[p]));
+
+ return map0;
+ }
+
+ /**
+ * @param map Regular java map with counters.
+ * @param partsCnt Total cache partitions.
+ * @return Full counters map.
+ */
+ static CachePartitionFullCountersMap fromCountersMap(Map<Integer, T2<Long, Long>> map, int partsCnt) {
+ CachePartitionFullCountersMap map0 = new CachePartitionFullCountersMap(partsCnt);
+
+ for (Map.Entry<Integer, T2<Long, Long>> e : map.entrySet()) {
+ T2<Long, Long> cntrs = e.getValue();
+
+ map0.initialUpdCntrs[e.getKey()] = cntrs.get1();
+ map0.updCntrs[e.getKey()] = cntrs.get2();
+ }
+
+ return map0;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/129be29e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
index 851ffed..83c0231 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
@@ -21,8 +21,10 @@ import java.io.Serializable;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
+import java.util.TreeMap;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteProductVersion;
/**
*
@@ -32,6 +34,9 @@ public class CachePartitionPartialCountersMap implements Serializable {
private static final long serialVersionUID = 0L;
/** */
+ static final IgniteProductVersion PARTIAL_COUNTERS_MAP_SINCE = IgniteProductVersion.fromString("2.1.4");
+
+ /** */
public static final CachePartitionPartialCountersMap EMPTY = new CachePartitionPartialCountersMap();
/** */
@@ -158,4 +163,22 @@ public class CachePartitionPartialCountersMap implements Serializable {
return res;
}
+
+ /**
+ * @param map Partition ID to partition counters map.
+ * @param partsCnt Total cache partitions.
+ * @return Partial local counters map.
+ */
+ static CachePartitionPartialCountersMap fromCountersMap(Map<Integer, T2<Long, Long>> map, int partsCnt) {
+ CachePartitionPartialCountersMap map0 = new CachePartitionPartialCountersMap(partsCnt);
+
+ TreeMap<Integer, T2<Long, Long>> sorted = new TreeMap<>(map);
+
+ for (Map.Entry<Integer, T2<Long, Long>> e : sorted.entrySet())
+ map0.add(e.getKey(), e.getValue().get1(), e.getValue().get2());
+
+ map0.trim();
+
+ return map0;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/129be29e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index ceb5abc..8e0deb9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -92,6 +92,7 @@ import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteProductVersion;
import org.apache.ignite.lang.IgniteRunnable;
import org.jetbrains.annotations.Nullable;
import org.jsr166.ConcurrentHashMap8;
@@ -106,6 +107,7 @@ import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap.PARTIAL_COUNTERS_MAP_SINCE;
/**
* Future for exchanging partition maps.
@@ -1231,6 +1233,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
msg = cctx.exchange().createPartitionsSingleMessage(exchangeId(),
false,
true,
+ node.version().compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0,
exchActions);
Map<Integer, Map<Integer, Long>> partHistReserved0 = partHistReserved;
@@ -1258,13 +1261,16 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
/**
* @param compress Message compress flag.
+ * @param newCntrMap {@code True} if possible to use {@link CachePartitionFullCountersMap}.
* @return Message.
*/
- private GridDhtPartitionsFullMessage createPartitionsMessage(boolean compress) {
+ private GridDhtPartitionsFullMessage createPartitionsMessage(boolean compress,
+ boolean newCntrMap) {
GridCacheVersion last = lastVer.get();
GridDhtPartitionsFullMessage m = cctx.exchange().createPartitionsFullMessage(
compress,
+ newCntrMap,
exchangeId(),
last != null ? last : cctx.versions().last(),
partHistSuppliers,
@@ -1797,9 +1803,14 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
if (finishState0 == null) {
assert firstDiscoEvt.type() == EVT_NODE_JOINED && CU.clientNode(firstDiscoEvt.eventNode()) : this;
+ ClusterNode node = cctx.node(nodeId);
+
+ if (node == null)
+ return;
+
finishState0 = new FinishState(cctx.localNodeId(),
initialVersion(),
- createPartitionsMessage(true));
+ createPartitionsMessage(true, node.version().compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0));
}
sendAllPartitionsToNode(finishState0, msg, nodeId);
@@ -1937,7 +1948,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
Map<Integer, Map<Integer, List<UUID>>> assignmentChange = fut.get();
- GridDhtPartitionsFullMessage m = createPartitionsMessage(false);
+ GridDhtPartitionsFullMessage m = createPartitionsMessage(false, false);
CacheAffinityChangeMessage msg = new CacheAffinityChangeMessage(exchId, m, assignmentChange);
@@ -1959,7 +1970,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
Map<Integer, Long> minCntrs = new HashMap<>();
for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> e : msgs.entrySet()) {
- CachePartitionPartialCountersMap nodeCntrs = e.getValue().partitionUpdateCounters(top.groupId());
+ CachePartitionPartialCountersMap nodeCntrs = e.getValue().partitionUpdateCounters(top.groupId(),
+ top.partitions());
assert nodeCntrs != null;
@@ -2235,7 +2247,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
GridDhtPartitionTopology top = grp != null ? grp.topology() :
cctx.exchange().clientTopology(grpId);
- CachePartitionPartialCountersMap cntrs = msg.partitionUpdateCounters(grpId);
+ CachePartitionPartialCountersMap cntrs = msg.partitionUpdateCounters(grpId,
+ top.partitions());
if (cntrs != null)
top.collectUpdateCounters(cntrs);
@@ -2283,7 +2296,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
cctx.versions().onExchange(lastVer.get().order());
- GridDhtPartitionsFullMessage msg = createPartitionsMessage(true);
+ IgniteProductVersion minVer = exchCtx.events().discoveryCache().minimumNodeVersion();
+
+ GridDhtPartitionsFullMessage msg = createPartitionsMessage(true,
+ minVer.compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0);
if (exchCtx.mergeExchanges()) {
assert !centralizedAff;
@@ -2571,6 +2587,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
msg.restoreExchangeId(),
cctx.kernalContext().clientNode(),
true,
+ node.version().compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0,
exchActions);
if (localJoinExchange() && finishState0 == null)
@@ -2745,11 +2762,12 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
for (Map.Entry<Integer, GridDhtPartitionFullMap> entry : msg.partitions().entrySet()) {
Integer grpId = entry.getKey();
- CachePartitionFullCountersMap cntrMap = msg.partitionUpdateCounters(grpId);
-
CacheGroupContext grp = cctx.cache().cacheGroup(grpId);
if (grp != null) {
+ CachePartitionFullCountersMap cntrMap = msg.partitionUpdateCounters(grpId,
+ grp.topology().partitions());
+
grp.topology().update(resTopVer,
entry.getValue(),
cntrMap,
@@ -2760,7 +2778,12 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(AffinityTopologyVersion.NONE);
if (oldest != null && oldest.isLocal()) {
- cctx.exchange().clientTopology(grpId).update(resTopVer,
+ GridDhtPartitionTopology top = cctx.exchange().clientTopology(grpId);
+
+ CachePartitionFullCountersMap cntrMap = msg.partitionUpdateCounters(grpId,
+ top.partitions());
+
+ top.update(resTopVer,
entry.getValue(),
cntrMap,
Collections.<Integer>emptySet(),
http://git-wip-us.apache.org/repos/asf/ignite/blob/129be29e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index 2bb19cd..edbfc23 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -33,6 +33,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartit
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
@@ -68,6 +69,14 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
/** Serialized partitions counters. */
private byte[] partCntrsBytes;
+ /** Partitions update counters. */
+ @GridToStringInclude
+ @GridDirectTransient
+ private IgniteDhtPartitionCountersMap2 partCntrs2;
+
+ /** Serialized partitions counters. */
+ private byte[] partCntrsBytes2;
+
/** Partitions history suppliers. */
@GridToStringInclude
@GridDirectTransient
@@ -149,6 +158,8 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
cp.partsBytes = partsBytes;
cp.partCntrs = partCntrs;
cp.partCntrsBytes = partCntrsBytes;
+ cp.partCntrs2 = partCntrs2;
+ cp.partCntrsBytes2 = partCntrsBytes2;
cp.partHistSuppliers = partHistSuppliers;
cp.partHistSuppliersBytes = partHistSuppliersBytes;
cp.partsToReload = partsToReload;
@@ -275,7 +286,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
* @param grpId Cache group ID.
* @param cntrMap Partition update counters.
*/
- public void addPartitionUpdateCounters(int grpId, CachePartitionFullCountersMap cntrMap) {
+ public void addPartitionUpdateCounters(int grpId, Map<Integer, T2<Long, Long>> cntrMap) {
if (partCntrs == null)
partCntrs = new IgniteDhtPartitionCountersMap();
@@ -284,10 +295,30 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
/**
* @param grpId Cache group ID.
+ * @param cntrMap Partition update counters.
+ */
+ public void addPartitionUpdateCounters(int grpId, CachePartitionFullCountersMap cntrMap) {
+ if (partCntrs2 == null)
+ partCntrs2 = new IgniteDhtPartitionCountersMap2();
+
+ partCntrs2.putIfAbsent(grpId, cntrMap);
+ }
+
+ /**
+ * @param grpId Cache group ID.
+ * @param partsCnt Total cache partitions.
* @return Partition update counters.
*/
- public CachePartitionFullCountersMap partitionUpdateCounters(int grpId) {
- return partCntrs == null ? null : partCntrs.get(grpId);
+ public CachePartitionFullCountersMap partitionUpdateCounters(int grpId, int partsCnt) {
+ if (partCntrs2 != null)
+ return partCntrs2.get(grpId);
+
+ if (partCntrs == null)
+ return null;
+
+ Map<Integer, T2<Long, Long>> map = partCntrs.get(grpId);
+
+ return map != null ? CachePartitionFullCountersMap.fromCountersMap(map, partsCnt) : null;
}
/**
@@ -327,6 +358,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
boolean marshal = (!F.isEmpty(parts) && partsBytes == null) ||
(partCntrs != null && !partCntrs.empty() && partCntrsBytes == null) ||
+ (partCntrs2 != null && !partCntrs2.empty() && partCntrsBytes2 == null) ||
(partHistSuppliers != null && partHistSuppliersBytes == null) ||
(partsToReload != null && partsToReloadBytes == null) ||
(!F.isEmpty(errs) && errsBytes == null);
@@ -334,6 +366,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
if (marshal) {
byte[] partsBytes0 = null;
byte[] partCntrsBytes0 = null;
+ byte[] partCntrsBytes20 = null;
byte[] partHistSuppliersBytes0 = null;
byte[] partsToReloadBytes0 = null;
byte[] errsBytes0 = null;
@@ -344,6 +377,9 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
if (partCntrs != null && !partCntrs.empty() && partCntrsBytes == null)
partCntrsBytes0 = U.marshal(ctx, partCntrs);
+ if (partCntrs2 != null && !partCntrs2.empty() && partCntrsBytes2 == null)
+ partCntrsBytes20 = U.marshal(ctx, partCntrs2);
+
if (partHistSuppliers != null && partHistSuppliersBytes == null)
partHistSuppliersBytes0 = U.marshal(ctx, partHistSuppliers);
@@ -359,12 +395,14 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
try {
byte[] partsBytesZip = U.zip(partsBytes0);
byte[] partCntrsBytesZip = U.zip(partCntrsBytes0);
+ byte[] partCntrsBytes2Zip = U.zip(partCntrsBytes20);
byte[] partHistSuppliersBytesZip = U.zip(partHistSuppliersBytes0);
byte[] partsToReloadBytesZip = U.zip(partsToReloadBytes0);
byte[] exsBytesZip = U.zip(errsBytes0);
partsBytes0 = partsBytesZip;
partCntrsBytes0 = partCntrsBytesZip;
+ partCntrsBytes20 = partCntrsBytes2Zip;
partHistSuppliersBytes0 = partHistSuppliersBytesZip;
partsToReloadBytes0 = partsToReloadBytesZip;
errsBytes0 = exsBytesZip;
@@ -378,6 +416,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
partsBytes = partsBytes0;
partCntrsBytes = partCntrsBytes0;
+ partCntrsBytes2 = partCntrsBytes20;
partHistSuppliersBytes = partHistSuppliersBytes0;
partsToReloadBytes = partsToReloadBytes0;
errsBytes = errsBytes0;
@@ -446,6 +485,13 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
partCntrs = U.unmarshal(ctx, partCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
}
+ if (partCntrsBytes2 != null && partCntrs2 == null) {
+ if (compressed())
+ partCntrs2 = U.unmarshalZip(ctx.marshaller(), partCntrsBytes2, U.resolveClassLoader(ldr, ctx.gridConfig()));
+ else
+ partCntrs2 = U.unmarshal(ctx, partCntrsBytes2, U.resolveClassLoader(ldr, ctx.gridConfig()));
+ }
+
if (partHistSuppliersBytes != null && partHistSuppliers == null) {
if (compressed())
partHistSuppliers = U.unmarshalZip(ctx.marshaller(), partHistSuppliersBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
@@ -520,30 +566,36 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
writer.incrementState();
case 10:
- if (!writer.writeByteArray("partHistSuppliersBytes", partHistSuppliersBytes))
+ if (!writer.writeByteArray("partCntrsBytes2", partCntrsBytes2))
return false;
writer.incrementState();
case 11:
- if (!writer.writeByteArray("partsBytes", partsBytes))
+ if (!writer.writeByteArray("partHistSuppliersBytes", partHistSuppliersBytes))
return false;
writer.incrementState();
case 12:
- if (!writer.writeByteArray("partsToReloadBytes", partsToReloadBytes))
+ if (!writer.writeByteArray("partsBytes", partsBytes))
return false;
writer.incrementState();
case 13:
- if (!writer.writeMessage("resTopVer", resTopVer))
+ if (!writer.writeByteArray("partsToReloadBytes", partsToReloadBytes))
return false;
writer.incrementState();
case 14:
+ if (!writer.writeMessage("resTopVer", resTopVer))
+ return false;
+
+ writer.incrementState();
+
+ case 15:
if (!writer.writeMessage("topVer", topVer))
return false;
@@ -606,7 +658,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 10:
- partHistSuppliersBytes = reader.readByteArray("partHistSuppliersBytes");
+ partCntrsBytes2 = reader.readByteArray("partCntrsBytes2");
if (!reader.isLastRead())
return false;
@@ -614,7 +666,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 11:
- partsBytes = reader.readByteArray("partsBytes");
+ partHistSuppliersBytes = reader.readByteArray("partHistSuppliersBytes");
if (!reader.isLastRead())
return false;
@@ -622,7 +674,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 12:
- partsToReloadBytes = reader.readByteArray("partsToReloadBytes");
+ partsBytes = reader.readByteArray("partsBytes");
if (!reader.isLastRead())
return false;
@@ -630,7 +682,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 13:
- resTopVer = reader.readMessage("resTopVer");
+ partsToReloadBytes = reader.readByteArray("partsToReloadBytes");
if (!reader.isLastRead())
return false;
@@ -638,6 +690,14 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 14:
+ resTopVer = reader.readMessage("resTopVer");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 15:
topVer = reader.readMessage("topVer");
if (!reader.isLastRead())
@@ -657,7 +717,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 15;
+ return 16;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/129be29e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index 44815ca..215152d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -32,6 +32,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartit
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
@@ -61,7 +62,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
/** Partitions update counters. */
@GridToStringInclude
@GridDirectTransient
- private Map<Integer, CachePartitionPartialCountersMap> partCntrs;
+ private Map<Integer, Object> partCntrs;
/** Serialized partitions counters. */
private byte[] partCntrsBytes;
@@ -189,7 +190,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
* @param grpId Cache group ID.
* @param cntrMap Partition update counters.
*/
- public void partitionUpdateCounters(int grpId, CachePartitionPartialCountersMap cntrMap) {
+ public void addPartitionUpdateCounters(int grpId, Object cntrMap) {
if (partCntrs == null)
partCntrs = new HashMap<>();
@@ -198,12 +199,24 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
/**
* @param grpId Cache group ID.
+ * @param partsCnt Total cache partitions.
* @return Partition update counters.
*/
- public CachePartitionPartialCountersMap partitionUpdateCounters(int grpId) {
- CachePartitionPartialCountersMap res = partCntrs == null ? null : partCntrs.get(grpId);
+ @SuppressWarnings("unchecked")
+ public CachePartitionPartialCountersMap partitionUpdateCounters(int grpId, int partsCnt) {
+ Object res = partCntrs == null ? null : partCntrs.get(grpId);
- return res == null ? CachePartitionPartialCountersMap.EMPTY : res;
+ if (res == null)
+ return CachePartitionPartialCountersMap.EMPTY;
+
+ if (res instanceof CachePartitionPartialCountersMap)
+ return (CachePartitionPartialCountersMap)res;
+
+ assert res instanceof Map : res;
+
+ Map<Integer, T2<Long, Long>> map = (Map<Integer, T2<Long, Long>>)res;
+
+ return CachePartitionPartialCountersMap.fromCountersMap(map, partsCnt);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/129be29e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap.java
index e7954d9..dc2fbf8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap.java
@@ -19,8 +19,10 @@
package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
import java.io.Serializable;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import org.apache.ignite.internal.util.typedef.T2;
/**
* Partition counters map.
@@ -30,7 +32,7 @@ public class IgniteDhtPartitionCountersMap implements Serializable {
private static final long serialVersionUID = 0L;
/** */
- private Map<Integer, CachePartitionFullCountersMap> map;
+ private Map<Integer, Map<Integer, T2<Long, Long>>> map;
/**
* @return {@code True} if map is empty.
@@ -43,7 +45,7 @@ public class IgniteDhtPartitionCountersMap implements Serializable {
* @param cacheId Cache ID.
* @param cntrMap Counters map.
*/
- public synchronized void putIfAbsent(int cacheId, CachePartitionFullCountersMap cntrMap) {
+ public synchronized void putIfAbsent(int cacheId, Map<Integer, T2<Long, Long>> cntrMap) {
if (map == null)
map = new HashMap<>();
@@ -55,14 +57,14 @@ public class IgniteDhtPartitionCountersMap implements Serializable {
* @param cacheId Cache ID.
* @return Counters map.
*/
- public synchronized CachePartitionFullCountersMap get(int cacheId) {
+ public synchronized Map<Integer, T2<Long, Long>> get(int cacheId) {
if (map == null)
- return null;
+ map = new HashMap<>();
- CachePartitionFullCountersMap cntrMap = map.get(cacheId);
+ Map<Integer, T2<Long, Long>> cntrMap = map.get(cacheId);
if (cntrMap == null)
- return null;
+ return Collections.emptyMap();
return cntrMap;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/129be29e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap2.java
new file mode 100644
index 0000000..d1e6d99
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap2.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Partition counters map.
+ */
+public class IgniteDhtPartitionCountersMap2 implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private Map<Integer, CachePartitionFullCountersMap> map;
+
+ /**
+ * @return {@code True} if map is empty.
+ */
+ public synchronized boolean empty() {
+ return map == null || map.isEmpty();
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param cntrMap Counters map.
+ */
+ public synchronized void putIfAbsent(int cacheId, CachePartitionFullCountersMap cntrMap) {
+ if (map == null)
+ map = new HashMap<>();
+
+ if (!map.containsKey(cacheId))
+ map.put(cacheId, cntrMap);
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @return Counters map.
+ */
+ public synchronized CachePartitionFullCountersMap get(int cacheId) {
+ if (map == null)
+ return null;
+
+ CachePartitionFullCountersMap cntrMap = map.get(cacheId);
+
+ if (cntrMap == null)
+ return null;
+
+ return cntrMap;
+ }
+}
[03/18] ignite git commit: IGNITE-6154 fix incorrect check checkpoint
pages
Posted by sb...@apache.org.
IGNITE-6154 fix incorrect check checkpoint pages
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/474ecb87
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/474ecb87
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/474ecb87
Branch: refs/heads/ignite-6149
Commit: 474ecb877dfa951a620dc1351b3882cd3e558672
Parents: f39cae9
Author: Dmitriy Govorukhin <dm...@gmail.com>
Authored: Tue Aug 22 16:34:31 2017 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Aug 22 20:49:07 2017 +0300
----------------------------------------------------------------------
.../GridCacheDatabaseSharedManager.java | 23 ++++++++++++++++++--
1 file changed, 21 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/474ecb87/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 58e28de..0f0fb88 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -2209,7 +2209,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
curr.cpBeginFut.onDone();
- if (!F.isEmpty(cpPagesTuple.get1())) {
+ if (hasPageForWrite(cpPagesTuple.get1())) {
assert cpPtr != null;
// Sync log outside the checkpoint write lock.
@@ -2260,6 +2260,24 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
+ * Check that at least one collection is not empty.
+ *
+ * @param cpPagesCollWrapper Collection of {@link GridMultiCollectionWrapper} checkpoint pages.
+ */
+ private boolean hasPageForWrite(Collection<GridMultiCollectionWrapper<FullPageId>> cpPagesCollWrapper) {
+ boolean hasPages = false;
+
+ for (Collection c : cpPagesCollWrapper)
+ if (!c.isEmpty()) {
+ hasPages = true;
+
+ break;
+ }
+
+ return hasPages;
+ }
+
+ /**
* @return tuple with collections of FullPageIds obtained from each PageMemory and overall number of dirty pages.
*/
private IgniteBiTuple<Collection<GridMultiCollectionWrapper<FullPageId>>, Integer> beginAllCheckpoints() {
@@ -2334,7 +2352,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
* @param cpPagesTuple Checkpoint pages tuple.
*/
private GridMultiCollectionWrapper<FullPageId> splitAndSortCpPagesIfNeeded(
- IgniteBiTuple<Collection<GridMultiCollectionWrapper<FullPageId>>, Integer> cpPagesTuple) {
+ IgniteBiTuple<Collection<GridMultiCollectionWrapper<FullPageId>>, Integer> cpPagesTuple
+ ) {
List<FullPageId> cpPagesList = new ArrayList<>(cpPagesTuple.get2());
for (GridMultiCollectionWrapper<FullPageId> col : cpPagesTuple.get1()) {
[12/18] ignite git commit: ignite-5280 SparseDistributedMatrix
refactoring
Posted by sb...@apache.org.
ignite-5280 SparseDistributedMatrix refactoring
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/46ec148c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/46ec148c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/46ec148c
Branch: refs/heads/ignite-6149
Commit: 46ec148ca9ae7ecc668c2c0bb9547140d05d68e2
Parents: 4fe8f76
Author: Yury Babak <yb...@gridgain.com>
Authored: Wed Aug 23 20:53:08 2017 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Wed Aug 23 20:53:41 2017 +0300
----------------------------------------------------------------------
.../ml/math/matrix/CacheMatrixExample.java | 4 +-
.../ml/math/vector/CacheVectorExample.java | 4 +-
modules/ml/pom.xml | 1 -
.../clustering/KMeansDistributedClusterer.java | 55 +-
.../ignite/ml/math/IdentityValueMapper.java | 2 +
.../apache/ignite/ml/math/MatrixKeyMapper.java | 30 -
.../apache/ignite/ml/math/MatrixStorage.java | 7 +
.../org/apache/ignite/ml/math/ValueMapper.java | 37 --
.../apache/ignite/ml/math/VectorKeyMapper.java | 29 -
.../ignite/ml/math/distributed/CacheUtils.java | 546 ++++++++++++++++++
.../ml/math/distributed/DistributedStorage.java | 35 ++
.../ml/math/distributed/MatrixKeyMapper.java | 33 ++
.../ignite/ml/math/distributed/ValueMapper.java | 37 ++
.../ml/math/distributed/VectorKeyMapper.java | 32 ++
.../math/distributed/keys/BlockMatrixKey.java | 30 +
.../math/distributed/keys/MatrixCacheKey.java | 35 ++
.../math/distributed/keys/RowColMatrixKey.java | 30 +
.../distributed/keys/impl/BlockMatrixKey.java | 144 +++++
.../distributed/keys/impl/SparseMatrixKey.java | 142 +++++
.../distributed/keys/impl/package-info.java | 22 +
.../ml/math/distributed/keys/package-info.java | 22 +
.../ml/math/distributed/package-info.java | 22 +
.../apache/ignite/ml/math/impls/CacheUtils.java | 559 -------------------
.../ml/math/impls/matrix/CacheMatrix.java | 6 +-
.../impls/matrix/DenseLocalOnHeapMatrix.java | 4 +-
.../matrix/SparseBlockDistributedMatrix.java | 16 +-
.../impls/matrix/SparseDistributedMatrix.java | 83 ++-
.../storage/matrix/BaseBlockMatrixKey.java | 41 --
.../impls/storage/matrix/BlockMatrixKey.java | 144 -----
.../storage/matrix/BlockMatrixStorage.java | 38 +-
.../storage/matrix/CacheMatrixStorage.java | 9 +-
.../matrix/DenseOffHeapMatrixStorage.java | 5 +
.../storage/matrix/DiagonalMatrixStorage.java | 5 +
.../storage/matrix/FunctionMatrixStorage.java | 5 +
.../storage/matrix/MatrixDelegateStorage.java | 5 +
.../storage/matrix/PivotedMatrixStorage.java | 5 +
.../storage/matrix/RandomMatrixStorage.java | 5 +
.../matrix/SparseDistributedMatrixStorage.java | 54 +-
.../matrix/SparseLocalOnHeapMatrixStorage.java | 6 +-
.../storage/vector/CacheVectorStorage.java | 4 +-
.../ml/math/impls/vector/CacheVector.java | 6 +-
.../ml/math/impls/matrix/CacheMatrixTest.java | 2 +-
.../impls/matrix/MatrixKeyMapperForTests.java | 2 +-
.../SparseDistributedBlockMatrixTest.java | 7 +-
.../matrix/SparseDistributedMatrixTest.java | 40 +-
.../ml/math/impls/vector/CacheVectorTest.java | 2 +-
46 files changed, 1368 insertions(+), 984 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/examples/src/main/ml/org/apache/ignite/examples/ml/math/matrix/CacheMatrixExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/math/matrix/CacheMatrixExample.java b/examples/src/main/ml/org/apache/ignite/examples/ml/math/matrix/CacheMatrixExample.java
index d7bb8ae..a7cbaab 100644
--- a/examples/src/main/ml/org/apache/ignite/examples/ml/math/matrix/CacheMatrixExample.java
+++ b/examples/src/main/ml/org/apache/ignite/examples/ml/math/matrix/CacheMatrixExample.java
@@ -23,9 +23,9 @@ import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.examples.ml.math.vector.CacheVectorExample;
import org.apache.ignite.ml.math.IdentityValueMapper;
-import org.apache.ignite.ml.math.MatrixKeyMapper;
import org.apache.ignite.ml.math.Tracer;
-import org.apache.ignite.ml.math.ValueMapper;
+import org.apache.ignite.ml.math.distributed.MatrixKeyMapper;
+import org.apache.ignite.ml.math.distributed.ValueMapper;
import org.apache.ignite.ml.math.functions.Functions;
import org.apache.ignite.ml.math.impls.matrix.CacheMatrix;
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/examples/src/main/ml/org/apache/ignite/examples/ml/math/vector/CacheVectorExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/math/vector/CacheVectorExample.java b/examples/src/main/ml/org/apache/ignite/examples/ml/math/vector/CacheVectorExample.java
index 14ec43b..4253ac1 100644
--- a/examples/src/main/ml/org/apache/ignite/examples/ml/math/vector/CacheVectorExample.java
+++ b/examples/src/main/ml/org/apache/ignite/examples/ml/math/vector/CacheVectorExample.java
@@ -22,8 +22,8 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.ml.math.IdentityValueMapper;
-import org.apache.ignite.ml.math.ValueMapper;
-import org.apache.ignite.ml.math.VectorKeyMapper;
+import org.apache.ignite.ml.math.distributed.ValueMapper;
+import org.apache.ignite.ml.math.distributed.VectorKeyMapper;
import org.apache.ignite.ml.math.impls.vector.CacheVector;
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/pom.xml
----------------------------------------------------------------------
diff --git a/modules/ml/pom.xml b/modules/ml/pom.xml
index 8774157..7d5d64f 100644
--- a/modules/ml/pom.xml
+++ b/modules/ml/pom.xml
@@ -91,7 +91,6 @@
<groupId>com.github.fommil.netlib</groupId>
<artifactId>core</artifactId>
<version>${netlibjava.version}</version>
- <type>pom</type>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
index 2ef61ad..d6a3fc3 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
@@ -29,23 +29,26 @@ import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.ml.math.DistanceMeasure;
import org.apache.ignite.ml.math.Vector;
import org.apache.ignite.ml.math.VectorUtils;
+import org.apache.ignite.ml.math.distributed.CacheUtils;
+import org.apache.ignite.ml.math.distributed.keys.impl.SparseMatrixKey;
import org.apache.ignite.ml.math.exceptions.ConvergenceException;
import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
import org.apache.ignite.ml.math.functions.Functions;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.impls.CacheUtils;
import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
import org.apache.ignite.ml.math.util.MapUtil;
import org.apache.ignite.ml.math.util.MatrixUtil;
-import static org.apache.ignite.ml.math.impls.CacheUtils.distributedFold;
+import static org.apache.ignite.ml.math.distributed.CacheUtils.distributedFold;
import static org.apache.ignite.ml.math.util.MatrixUtil.localCopyOf;
/**
* Clustering algorithm based on Bahmani et al. paper and Apache Spark class with corresponding functionality.
*
+ * TODO: IGNITE-6059, add block matrix support.
+ *
* @see <a href="http://theory.stanford.edu/~sergei/papers/vldb12-kmpar.pdf">Scalable K-Means++(wikipedia)</a>
*/
public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistributedMatrix> {
@@ -80,6 +83,8 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
MathIllegalArgumentException, ConvergenceException {
SparseDistributedMatrix pointsCp = (SparseDistributedMatrix)points.like(points.rowSize(), points.columnSize());
+ String cacheName = ((SparseDistributedMatrixStorage)points.getStorage()).cacheName();
+
// TODO: IGNITE-5825, this copy is very ineffective, just for POC. Immutability of data should be guaranteed by other methods
// such as logical locks for example.
pointsCp.assign(points);
@@ -93,7 +98,7 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
// Execute iterations of Lloyd's algorithm until converged
while (iteration < maxIterations && !converged) {
- SumsAndCounts stats = getSumsAndCounts(centers, dim, uid);
+ SumsAndCounts stats = getSumsAndCounts(centers, dim, uid, cacheName);
converged = true;
@@ -119,6 +124,8 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
// Initialize empty centers and point costs.
int ptsCnt = points.rowSize();
+ String cacheName = ((SparseDistributedMatrixStorage)points.getStorage()).cacheName();
+
// Initialize the first center to a random point.
Vector sample = localCopyOf(points.viewRow(rnd.nextInt(ptsCnt)));
@@ -137,7 +144,7 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
while (step < initSteps) {
// We assume here that costs can fit into memory of one node.
- ConcurrentHashMap<Integer, Double> newCosts = getNewCosts(points, newCenters);
+ ConcurrentHashMap<Integer, Double> newCosts = getNewCosts(points, newCenters, cacheName);
// Merge costs with new costs.
for (Integer ind : newCosts.keySet())
@@ -145,7 +152,7 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
double sumCosts = costs.values().stream().mapToDouble(Double::valueOf).sum();
- newCenters = getNewCenters(k, costs, uid, sumCosts);
+ newCenters = getNewCenters(k, costs, uid, sumCosts, cacheName);
centers.addAll(newCenters);
step++;
@@ -159,7 +166,7 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
// Finally, we might have a set of more than k distinct candidate centers; weight each
// candidate by the number of points in the dataset mapping to it and run a local k-means++
// on the weighted centers to pick k of them
- ConcurrentHashMap<Integer, Integer> centerInd2Weight = weightCenters(uid, distinctCenters);
+ ConcurrentHashMap<Integer, Integer> centerInd2Weight = weightCenters(uid, distinctCenters, cacheName);
List<Double> weights = new ArrayList<>(centerInd2Weight.size());
@@ -174,12 +181,12 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
/** */
private List<Vector> getNewCenters(int k, ConcurrentHashMap<Integer, Double> costs, IgniteUuid uid,
- double sumCosts) {
- return distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME,
- (IgniteBiFunction<Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>>,
+ double sumCosts, String cacheName) {
+ return distributedFold(cacheName,
+ (IgniteBiFunction<Cache.Entry<SparseMatrixKey, Map<Integer, Double>>,
List<Vector>,
List<Vector>>)(vectorWithIndex, list) -> {
- Integer ind = vectorWithIndex.getKey().get1();
+ Integer ind = vectorWithIndex.getKey().index();
double prob = costs.get(ind) * 2.0 * k / sumCosts;
@@ -188,7 +195,7 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
return list;
},
- key -> key.get2().equals(uid),
+ key -> key.matrixId().equals(uid),
(list1, list2) -> {
list1.addAll(list2);
return list1;
@@ -198,17 +205,17 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
}
/** */
- private ConcurrentHashMap<Integer, Double> getNewCosts(SparseDistributedMatrix points, List<Vector> newCenters) {
- return distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME,
- (IgniteBiFunction<Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, ConcurrentHashMap<Integer, Double>>,
+ private ConcurrentHashMap<Integer, Double> getNewCosts(SparseDistributedMatrix points, List<Vector> newCenters, String cacheName) {
+ return distributedFold(cacheName,
+ (IgniteBiFunction<Cache.Entry<SparseMatrixKey, ConcurrentHashMap<Integer, Double>>,
ConcurrentHashMap<Integer, Double>,
ConcurrentHashMap<Integer, Double>>)(vectorWithIndex, map) -> {
for (Vector center : newCenters)
- map.merge(vectorWithIndex.getKey().get1(), distance(vectorWithIndex.getValue(), center), Functions.MIN);
+ map.merge(vectorWithIndex.getKey().index(), distance(vectorWithIndex.getValue(), center), Functions.MIN);
return map;
},
- key -> key.get2().equals(points.getUUID()),
+ key -> key.matrixId().equals(points.getUUID()),
(map1, map2) -> {
map1.putAll(map2);
return map1;
@@ -216,9 +223,9 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
}
/** */
- private ConcurrentHashMap<Integer, Integer> weightCenters(IgniteUuid uid, List<Vector> distinctCenters) {
- return distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME,
- (IgniteBiFunction<Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>>,
+ private ConcurrentHashMap<Integer, Integer> weightCenters(IgniteUuid uid, List<Vector> distinctCenters, String cacheName) {
+ return distributedFold(cacheName,
+ (IgniteBiFunction<Cache.Entry<SparseMatrixKey, Map<Integer, Double>>,
ConcurrentHashMap<Integer, Integer>,
ConcurrentHashMap<Integer, Integer>>)(vectorWithIndex, countMap) -> {
Integer resInd = -1;
@@ -239,7 +246,7 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
countMap.compute(resInd, (ind, v) -> v != null ? v + 1 : 1);
return countMap;
},
- key -> key.get2().equals(uid),
+ key -> key.matrixId().equals(uid),
(map1, map2) -> MapUtil.mergeMaps(map1, map2, (integer, integer2) -> integer2 + integer,
ConcurrentHashMap::new),
new ConcurrentHashMap<>());
@@ -251,9 +258,9 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
}
/** */
- private SumsAndCounts getSumsAndCounts(Vector[] centers, int dim, IgniteUuid uid) {
- return CacheUtils.distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME,
- (IgniteBiFunction<Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>>, SumsAndCounts, SumsAndCounts>)(entry, counts) -> {
+ private SumsAndCounts getSumsAndCounts(Vector[] centers, int dim, IgniteUuid uid, String cacheName) {
+ return CacheUtils.distributedFold(cacheName,
+ (IgniteBiFunction<Cache.Entry<SparseMatrixKey, Map<Integer, Double>>, SumsAndCounts, SumsAndCounts>)(entry, counts) -> {
Map<Integer, Double> vec = entry.getValue();
IgniteBiTuple<Integer, Double> closest = findClosest(centers, VectorUtils.fromMap(vec, false));
@@ -270,7 +277,7 @@ public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistri
return counts;
},
- key -> key.get2().equals(uid),
+ key -> key.matrixId().equals(uid),
SumsAndCounts::merge, new SumsAndCounts()
);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/IdentityValueMapper.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/IdentityValueMapper.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/IdentityValueMapper.java
index 3c94edd..615006e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/IdentityValueMapper.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/IdentityValueMapper.java
@@ -17,6 +17,8 @@
package org.apache.ignite.ml.math;
+import org.apache.ignite.ml.math.distributed.ValueMapper;
+
/**
* Identity value mapper.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixKeyMapper.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixKeyMapper.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixKeyMapper.java
deleted file mode 100644
index 54d2088..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixKeyMapper.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.math;
-
-/**
- * Maps {@link Matrix} row and column index to cache key.
- */
-public interface MatrixKeyMapper<K> extends KeyMapper<K> {
- /**
- * @param x Matrix row index.
- * @param y Matrix column index.
- * @return Cache key for given row and column.
- */
- public K apply(int x, int y);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixStorage.java
index a80e066..e4f9e40 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixStorage.java
@@ -55,6 +55,13 @@ public interface MatrixStorage extends Externalizable, StorageOpsMetrics, Destro
public int storageMode();
/**
+ * @return Matrix access mode.
+ *
+ * @see StorageConstants
+ */
+ public int accessMode();
+
+ /**
* Gets underlying data, if {@link StorageOpsMetrics#isArrayBased()} returns {@code false} this method return
* copy of data. The data must be adapted for {@link Blas}.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/ValueMapper.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/ValueMapper.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/ValueMapper.java
deleted file mode 100644
index f0776a3..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/ValueMapper.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.math;
-
-import java.io.Serializable;
-
-/**
- * Utility mapper that can be used to map arbitrary values types to and from double.
- */
-public interface ValueMapper<V> extends Serializable {
- /**
- * @param v Value to map from double.
- * @return Mapped value.
- */
- public V fromDouble(double v);
-
- /**
- * @param v Value to map to double.
- * @return Mapped value.
- */
- public double toDouble(V v);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorKeyMapper.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorKeyMapper.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorKeyMapper.java
deleted file mode 100644
index 4b8fadb..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorKeyMapper.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.math;
-
-/**
- * Maps {@link Vector} element index to cache key.
- */
-public interface VectorKeyMapper<K> extends KeyMapper<K> {
- /**
- * @param i Vector element index.
- * @return Cache key for given element index.
- */
- public K apply(int i);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/CacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/CacheUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/CacheUtils.java
new file mode 100644
index 0000000..9a73c5a
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/CacheUtils.java
@@ -0,0 +1,546 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.distributed;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.function.BinaryOperator;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.ml.math.KeyMapper;
+import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey;
+import org.apache.ignite.ml.math.distributed.keys.impl.BlockMatrixKey;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteConsumer;
+import org.apache.ignite.ml.math.functions.IgniteDoubleFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.impls.matrix.BlockEntry;
+
+/**
+ * Distribution-related misc. support.
+ *
+ * TODO: IGNITE-5102, fix sparse key filters.
+ */
+public class CacheUtils {
+ /**
+ * Cache entry support.
+ *
+ * @param <K>
+ * @param <V>
+ */
+ public static class CacheEntry<K, V> {
+ /** */
+ private Cache.Entry<K, V> entry;
+ /** */
+ private IgniteCache<K, V> cache;
+
+ /**
+ * @param entry Original cache entry.
+ * @param cache Cache instance.
+ */
+ CacheEntry(Cache.Entry<K, V> entry, IgniteCache<K, V> cache) {
+ this.entry = entry;
+ this.cache = cache;
+ }
+
+ /**
+ *
+ *
+ */
+ public Cache.Entry<K, V> entry() {
+ return entry;
+ }
+
+ /**
+ *
+ *
+ */
+ public IgniteCache<K, V> cache() {
+ return cache;
+ }
+ }
+
+ /**
+ * Gets local Ignite instance.
+ */
+ public static Ignite ignite() {
+ return Ignition.localIgnite();
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @param k Key into the cache.
+ * @param <K> Key type.
+ * @return Cluster group for given key.
+ */
+ public static <K> ClusterGroup groupForKey(String cacheName, K k) {
+ return ignite().cluster().forNode(ignite().affinity(cacheName).mapKeyToNode(k));
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @param keyMapper {@link KeyMapper} to validate cache key.
+ * @param valMapper {@link ValueMapper} to obtain double value for given cache key.
+ * @param <K> Cache key object type.
+ * @param <V> Cache value object type.
+ * @return Sum of the values obtained for valid keys.
+ */
+ public static <K, V> double sum(String cacheName, KeyMapper<K> keyMapper, ValueMapper<V> valMapper) {
+ Collection<Double> subSums = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
+ if (keyMapper.isValid(ce.entry().getKey())) {
+ double v = valMapper.toDouble(ce.entry().getValue());
+
+ return acc == null ? v : acc + v;
+ }
+ else
+ return acc;
+ });
+
+ return sum(subSums);
+ }
+
+ /**
+ * @param matrixUuid Matrix UUID.
+ * @return Sum obtained using sparse logic.
+ */
+ @SuppressWarnings("unchecked")
+ public static <K, V> double sparseSum(IgniteUuid matrixUuid, String cacheName) {
+ A.notNull(matrixUuid, "matrixUuid");
+ A.notNull(cacheName, "cacheName");
+
+ Collection<Double> subSums = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
+ V v = ce.entry().getValue();
+
+ double sum = 0.0;
+
+ if (v instanceof Map) {
+ Map<Integer, Double> map = (Map<Integer, Double>)v;
+
+ sum = sum(map.values());
+ }
+ else if (v instanceof BlockEntry) {
+ BlockEntry be = (BlockEntry)v;
+
+ sum = be.sum();
+ }
+ else
+ throw new UnsupportedOperationException();
+
+ return acc == null ? sum : acc + sum;
+ }, sparseKeyFilter(matrixUuid));
+
+ return sum(subSums);
+ }
+
+ /**
+ * @param c {@link Collection} of double values to sum.
+ * @return Sum of the values.
+ */
+ private static double sum(Collection<Double> c) {
+ double sum = 0.0;
+
+ for (double d : c)
+ sum += d;
+
+ return sum;
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @param keyMapper {@link KeyMapper} to validate cache key.
+ * @param valMapper {@link ValueMapper} to obtain double value for given cache key.
+ * @param <K> Cache key object type.
+ * @param <V> Cache value object type.
+ * @return Minimum value for valid keys.
+ */
+ public static <K, V> double min(String cacheName, KeyMapper<K> keyMapper, ValueMapper<V> valMapper) {
+ Collection<Double> mins = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
+ if (keyMapper.isValid(ce.entry().getKey())) {
+ double v = valMapper.toDouble(ce.entry().getValue());
+
+ if (acc == null)
+ return v;
+ else
+ return Math.min(acc, v);
+ }
+ else
+ return acc;
+ });
+
+ return Collections.min(mins);
+ }
+
+ /**
+ * @param matrixUuid Matrix UUID.
+ * @return Minimum value obtained using sparse logic.
+ */
+ @SuppressWarnings("unchecked")
+ public static <K, V> double sparseMin(IgniteUuid matrixUuid, String cacheName) {
+ A.notNull(matrixUuid, "matrixUuid");
+ A.notNull(cacheName, "cacheName");
+
+ Collection<Double> mins = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
+ V v = ce.entry().getValue();
+
+ double min;
+
+ if (v instanceof Map) {
+ Map<Integer, Double> map = (Map<Integer, Double>)v;
+
+ min = Collections.min(map.values());
+ }
+ else if (v instanceof BlockEntry) {
+ BlockEntry be = (BlockEntry)v;
+
+ min = be.minValue();
+ }
+ else
+ throw new UnsupportedOperationException();
+
+ if (acc == null)
+ return min;
+ else
+ return Math.min(acc, min);
+
+ }, sparseKeyFilter(matrixUuid));
+
+ return Collections.min(mins);
+ }
+
+ /**
+ * @param matrixUuid Matrix UUID.
+ * @return Maximum value obtained using sparse logic.
+ */
+ @SuppressWarnings("unchecked")
+ public static <K, V> double sparseMax(IgniteUuid matrixUuid, String cacheName) {
+ A.notNull(matrixUuid, "matrixUuid");
+ A.notNull(cacheName, "cacheName");
+
+ Collection<Double> maxes = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
+ V v = ce.entry().getValue();
+
+ double max;
+
+ if (v instanceof Map) {
+ Map<Integer, Double> map = (Map<Integer, Double>)v;
+
+ max = Collections.max(map.values());
+ }
+ else if (v instanceof BlockEntry) {
+ BlockEntry be = (BlockEntry)v;
+
+ max = be.maxValue();
+ }
+ else
+ throw new UnsupportedOperationException();
+
+ if (acc == null)
+ return max;
+ else
+ return Math.max(acc, max);
+
+ }, sparseKeyFilter(matrixUuid));
+
+ return Collections.max(maxes);
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @param keyMapper {@link KeyMapper} to validate cache key.
+ * @param valMapper {@link ValueMapper} to obtain double value for given cache key.
+ * @param <K> Cache key object type.
+ * @param <V> Cache value object type.
+ * @return Maximum value for valid keys.
+ */
+ public static <K, V> double max(String cacheName, KeyMapper<K> keyMapper, ValueMapper<V> valMapper) {
+ Collection<Double> maxes = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> {
+ if (keyMapper.isValid(ce.entry().getKey())) {
+ double v = valMapper.toDouble(ce.entry().getValue());
+
+ if (acc == null)
+ return v;
+ else
+ return Math.max(acc, v);
+ }
+ else
+ return acc;
+ });
+
+ return Collections.max(maxes);
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @param keyMapper {@link KeyMapper} to validate cache key.
+ * @param valMapper {@link ValueMapper} to obtain double value for given cache key.
+ * @param mapper Mapping {@link IgniteFunction}.
+ * @param <K> Cache key object type.
+ * @param <V> Cache value object type.
+ */
+ public static <K, V> void map(String cacheName, KeyMapper<K> keyMapper, ValueMapper<V> valMapper,
+ IgniteFunction<Double, Double> mapper) {
+ foreach(cacheName, (CacheEntry<K, V> ce) -> {
+ K k = ce.entry().getKey();
+
+ if (keyMapper.isValid(k))
+ // Actual assignment.
+ ce.cache().put(k, valMapper.fromDouble(mapper.apply(valMapper.toDouble(ce.entry().getValue()))));
+ });
+ }
+
+ /**
+ * @param matrixUuid Matrix UUID.
+ * @param mapper Mapping {@link IgniteFunction}.
+ */
+ @SuppressWarnings("unchecked")
+ public static <K, V> void sparseMap(IgniteUuid matrixUuid, IgniteDoubleFunction<Double> mapper, String cacheName) {
+ A.notNull(matrixUuid, "matrixUuid");
+ A.notNull(cacheName, "cacheName");
+ A.notNull(mapper, "mapper");
+
+ foreach(cacheName, (CacheEntry<K, V> ce) -> {
+ K k = ce.entry().getKey();
+
+ V v = ce.entry().getValue();
+
+ if (v instanceof Map) {
+ Map<Integer, Double> map = (Map<Integer, Double>)v;
+
+ for (Map.Entry<Integer, Double> e : (map.entrySet()))
+ e.setValue(mapper.apply(e.getValue()));
+
+ }
+ else if (v instanceof BlockEntry) {
+ BlockEntry be = (BlockEntry)v;
+
+ be.map(mapper);
+ }
+ else
+ throw new UnsupportedOperationException();
+
+ ce.cache().put(k, v);
+ }, sparseKeyFilter(matrixUuid));
+ }
+
+ /**
+ * Filter for distributed matrix keys.
+ *
+ * @param matrixUuid Matrix uuid.
+ */
+ private static <K> IgnitePredicate<K> sparseKeyFilter(IgniteUuid matrixUuid) {
+ return key -> {
+ if (key instanceof BlockMatrixKey)
+ return ((BlockMatrixKey)key).matrixId().equals(matrixUuid);
+ else if (key instanceof RowColMatrixKey)
+ return ((RowColMatrixKey)key).matrixId().equals(matrixUuid);
+ else
+ throw new UnsupportedOperationException();
+ };
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @param fun An operation that accepts a cache entry and processes it.
+ * @param <K> Cache key object type.
+ * @param <V> Cache value object type.
+ */
+ public static <K, V> void foreach(String cacheName, IgniteConsumer<CacheEntry<K, V>> fun) {
+ foreach(cacheName, fun, null);
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @param fun An operation that accepts a cache entry and processes it.
+ * @param keyFilter Cache keys filter.
+ * @param <K> Cache key object type.
+ * @param <V> Cache value object type.
+ */
+ public static <K, V> void foreach(String cacheName, IgniteConsumer<CacheEntry<K, V>> fun,
+ IgnitePredicate<K> keyFilter) {
+ bcast(cacheName, () -> {
+ Ignite ignite = Ignition.localIgnite();
+ IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
+
+ int partsCnt = ignite.affinity(cacheName).partitions();
+
+ // Use affinity in filter for scan query. Otherwise we accept consumer in each node which is wrong.
+ Affinity affinity = ignite.affinity(cacheName);
+ ClusterNode locNode = ignite.cluster().localNode();
+
+ // Iterate over all partitions. Some of them will be stored on that local node.
+ for (int part = 0; part < partsCnt; part++) {
+ int p = part;
+
+ // Iterate over given partition.
+ // Query returns an empty cursor if this partition is not stored on this node.
+ for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part, (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k)))))
+ fun.accept(new CacheEntry<>(entry, cache));
+ }
+ });
+ }
+
+ /**
+ * <b>Currently fold supports only commutative operations.<b/>
+ *
+ * @param cacheName Cache name.
+ * @param folder Fold function operating over cache entries.
+ * @param <K> Cache key object type.
+ * @param <V> Cache value object type.
+ * @param <A> Fold result type.
+ * @return Fold operation result.
+ */
+ public static <K, V, A> Collection<A> fold(String cacheName, IgniteBiFunction<CacheEntry<K, V>, A, A> folder) {
+ return fold(cacheName, folder, null);
+ }
+
+ /**
+ * <b>Currently fold supports only commutative operations.<b/>
+ *
+ * @param cacheName Cache name.
+ * @param folder Fold function operating over cache entries.
+ * @param <K> Cache key object type.
+ * @param <V> Cache value object type.
+ * @param <A> Fold result type.
+ * @return Fold operation result.
+ */
+ public static <K, V, A> Collection<A> fold(String cacheName, IgniteBiFunction<CacheEntry<K, V>, A, A> folder,
+ IgnitePredicate<K> keyFilter) {
+ return bcast(cacheName, () -> {
+ Ignite ignite = Ignition.localIgnite();
+ IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
+
+ int partsCnt = ignite.affinity(cacheName).partitions();
+
+ // Use affinity in filter for ScanQuery. Otherwise we accept consumer in each node which is wrong.
+ Affinity affinity = ignite.affinity(cacheName);
+ ClusterNode locNode = ignite.cluster().localNode();
+
+ A a = null;
+
+ // Iterate over all partitions. Some of them will be stored on that local node.
+ for (int part = 0; part < partsCnt; part++) {
+ int p = part;
+
+ // Iterate over given partition.
+ // Query returns an empty cursor if this partition is not stored on this node.
+ for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part,
+ (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k)))))
+ a = folder.apply(new CacheEntry<>(entry, cache), a);
+ }
+
+ return a;
+ });
+ }
+
+ /**
+ * Distributed version of fold operation.
+ *
+ * @param cacheName Cache name.
+ * @param folder Folder.
+ * @param keyFilter Key filter.
+ * @param accumulator Accumulator.
+ * @param zeroVal Zero value.
+ */
+ public static <K, V, A> A distributedFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder,
+ IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, A zeroVal) {
+ return sparseFold(cacheName, folder, keyFilter, accumulator, zeroVal, null, null, 0,
+ false);
+ }
+
+ /**
+ * Sparse version of fold. This method also applicable to sparse zeroes.
+ *
+ * @param cacheName Cache name.
+ * @param folder Folder.
+ * @param keyFilter Key filter.
+ * @param accumulator Accumulator.
+ * @param zeroVal Zero value.
+ * @param defVal Def value.
+ * @param defKey Def key.
+ * @param defValCnt Def value count.
+ * @param isNilpotent Is nilpotent.
+ */
+ private static <K, V, A> A sparseFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder,
+ IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, A zeroVal, V defVal, K defKey, long defValCnt,
+ boolean isNilpotent) {
+
+ A defRes = zeroVal;
+
+ if (!isNilpotent)
+ for (int i = 0; i < defValCnt; i++)
+ defRes = folder.apply(new CacheEntryImpl<>(defKey, defVal), defRes);
+
+ Collection<A> totalRes = bcast(cacheName, () -> {
+ Ignite ignite = Ignition.localIgnite();
+ IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
+
+ int partsCnt = ignite.affinity(cacheName).partitions();
+
+ // Use affinity in filter for ScanQuery. Otherwise we accept consumer in each node which is wrong.
+ Affinity affinity = ignite.affinity(cacheName);
+ ClusterNode locNode = ignite.cluster().localNode();
+
+ A a = zeroVal;
+
+ // Iterate over all partitions. Some of them will be stored on that local node.
+ for (int part = 0; part < partsCnt; part++) {
+ int p = part;
+
+ // Iterate over given partition.
+ // Query returns an empty cursor if this partition is not stored on this node.
+ for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part,
+ (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k)))))
+ a = folder.apply(entry, a);
+ }
+
+ return a;
+ });
+ totalRes.add(defRes);
+ return totalRes.stream().reduce(zeroVal, accumulator);
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @param run {@link Runnable} to broadcast to cache nodes for given cache name.
+ */
+ public static void bcast(String cacheName, IgniteRunnable run) {
+ ignite().compute(ignite().cluster().forCacheNodes(cacheName)).broadcast(run);
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @param call {@link IgniteCallable} to broadcast to cache nodes for given cache name.
+ * @param <A> Type returned by the callable.
+ */
+ public static <A> Collection<A> bcast(String cacheName, IgniteCallable<A> call) {
+ return ignite().compute(ignite().cluster().forCacheNodes(cacheName)).broadcast(call);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/DistributedStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/DistributedStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/DistributedStorage.java
new file mode 100644
index 0000000..7b58d1d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/DistributedStorage.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.distributed;
+
+import java.util.Set;
+
+/**
+ * Extension for any distributed storage.
+ */
+public interface DistributedStorage<K> {
+ /**
+ * Build a keyset for this storage.
+ */
+ public Set<K> getAllKeys();
+
+ /**
+ * @return The name of cache used in this storage.
+ */
+ public String cacheName();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/MatrixKeyMapper.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/MatrixKeyMapper.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/MatrixKeyMapper.java
new file mode 100644
index 0000000..2a93328
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/MatrixKeyMapper.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.distributed;
+
+import org.apache.ignite.ml.math.KeyMapper;
+import org.apache.ignite.ml.math.Matrix;
+
+/**
+ * Maps {@link Matrix} row and column index to cache key.
+ */
+public interface MatrixKeyMapper<K> extends KeyMapper<K> {
+ /**
+ * @param x Matrix row index.
+ * @param y Matrix column index.
+ * @return Cache key for given row and column.
+ */
+ public K apply(int x, int y);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/ValueMapper.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/ValueMapper.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/ValueMapper.java
new file mode 100644
index 0000000..c94cfb6
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/ValueMapper.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.distributed;
+
+import java.io.Serializable;
+
+/**
+ * Utility mapper that can be used to map arbitrary values types to and from double.
+ */
+public interface ValueMapper<V> extends Serializable {
+ /**
+ * @param v Value to map from double.
+ * @return Mapped value.
+ */
+ public V fromDouble(double v);
+
+ /**
+ * @param v Value to map to double.
+ * @return Mapped value.
+ */
+ public double toDouble(V v);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/VectorKeyMapper.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/VectorKeyMapper.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/VectorKeyMapper.java
new file mode 100644
index 0000000..de08d7b
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/VectorKeyMapper.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.distributed;
+
+import org.apache.ignite.ml.math.KeyMapper;
+import org.apache.ignite.ml.math.Vector;
+
+/**
+ * Maps {@link Vector} element index to cache key.
+ */
+public interface VectorKeyMapper<K> extends KeyMapper<K> {
+ /**
+ * @param i Vector element index.
+ * @return Cache key for given element index.
+ */
+ public K apply(int i);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/BlockMatrixKey.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/BlockMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/BlockMatrixKey.java
new file mode 100644
index 0000000..c55b950
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/BlockMatrixKey.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.distributed.keys;
+
+import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix;
+
+/**
+ * Cache key for blocks in {@link SparseBlockDistributedMatrix}.
+ */
+public interface BlockMatrixKey extends MatrixCacheKey {
+ /**
+ * @return block id.
+ */
+ public long blockId();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/MatrixCacheKey.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/MatrixCacheKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/MatrixCacheKey.java
new file mode 100644
index 0000000..669e9a4
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/MatrixCacheKey.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.distributed.keys;
+
+import org.apache.ignite.lang.IgniteUuid;
+
+/**
+ * Base matrix cache key.
+ */
+public interface MatrixCacheKey {
+ /**
+ * @return matrix id.
+ */
+ public IgniteUuid matrixId();
+
+ /**
+ * @return affinity key.
+ */
+ public IgniteUuid affinityKey();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/RowColMatrixKey.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/RowColMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/RowColMatrixKey.java
new file mode 100644
index 0000000..168f49f
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/RowColMatrixKey.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.distributed.keys;
+
+import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
+
+/**
+ * Cache key for {@link SparseDistributedMatrix}.
+ */
+public interface RowColMatrixKey extends MatrixCacheKey {
+ /**
+ * Return index value(blockId, Row/Col index, etc.)
+ */
+ public int index();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/BlockMatrixKey.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/BlockMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/BlockMatrixKey.java
new file mode 100644
index 0000000..5fd1a16
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/BlockMatrixKey.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.distributed.keys.impl;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.internal.binary.BinaryUtils;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.ml.math.impls.matrix.BlockEntry;
+import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Key implementation for {@link BlockEntry} using for {@link SparseBlockDistributedMatrix}.
+ */
+public class BlockMatrixKey implements org.apache.ignite.ml.math.distributed.keys.BlockMatrixKey, Externalizable, Binarylizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+ /** Block ID */
+ private long blockId;
+ /** Matrix ID */
+ private IgniteUuid matrixUuid;
+ /** Block affinity key. */
+ private IgniteUuid affinityKey;
+
+ /**
+ * Empty constructor required for {@link Externalizable}.
+ */
+ public BlockMatrixKey() {
+ // No-op.
+ }
+
+ /**
+ * Construct matrix block key.
+ *
+ * @param blockId Block id.
+ * @param matrixUuid Matrix uuid.
+ * @param affinityKey Affinity key.
+ */
+ public BlockMatrixKey(long blockId, IgniteUuid matrixUuid, @Nullable IgniteUuid affinityKey) {
+ assert blockId >= 0;
+ assert matrixUuid != null;
+
+ this.blockId = blockId;
+ this.matrixUuid = matrixUuid;
+ this.affinityKey = affinityKey;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long blockId() {
+ return blockId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid matrixId() {
+ return matrixUuid;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid affinityKey() {
+ return affinityKey;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ U.writeGridUuid(out, matrixUuid);
+ U.writeGridUuid(out, affinityKey);
+ out.writeLong(blockId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ matrixUuid = U.readGridUuid(in);
+ affinityKey = U.readGridUuid(in);
+ blockId = in.readLong();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+ BinaryRawWriter out = writer.rawWriter();
+
+ BinaryUtils.writeIgniteUuid(out, matrixUuid);
+ BinaryUtils.writeIgniteUuid(out, affinityKey);
+ out.writeLong(blockId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+ BinaryRawReader in = reader.rawReader();
+
+ matrixUuid = BinaryUtils.readIgniteUuid(in);
+ affinityKey = BinaryUtils.readIgniteUuid(in);
+ blockId = in.readLong();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return matrixUuid.hashCode() + (int)(blockId ^ (blockId >>> 32));
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+
+ if (obj == null || obj.getClass() != getClass())
+ return false;
+
+ BlockMatrixKey that = (BlockMatrixKey)obj;
+
+ return blockId == that.blockId && matrixUuid.equals(that.matrixUuid) && F.eq(affinityKey, that.affinityKey);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(BlockMatrixKey.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/SparseMatrixKey.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/SparseMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/SparseMatrixKey.java
new file mode 100644
index 0000000..0c34c8b
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/SparseMatrixKey.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.distributed.keys.impl;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryRawReader;
+import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.internal.binary.BinaryUtils;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey;
+import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
+
+/**
+ * Key implementation for {@link SparseDistributedMatrix}.
+ */
+public class SparseMatrixKey implements RowColMatrixKey, Externalizable, Binarylizable {
+ /** */
+ private int idx;
+ /** */
+ private IgniteUuid matrixId;
+ /** */
+ private IgniteUuid affinityKey;
+
+ /**
+ * Default constructor (required by Externalizable).
+ */
+ public SparseMatrixKey(){
+
+ }
+
+ /**
+ * Build Key.
+ */
+ public SparseMatrixKey(int idx, IgniteUuid matrixId, IgniteUuid affinityKey) {
+ assert idx >= 0 : "Index must be positive.";
+ assert matrixId != null : "Matrix id can`t be null.";
+
+ this.idx = idx;
+ this.matrixId = matrixId;
+ this.affinityKey = affinityKey;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int index() {
+ return idx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid matrixId() {
+ return matrixId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid affinityKey() {
+ return affinityKey;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ U.writeGridUuid(out, matrixId);
+ U.writeGridUuid(out, affinityKey);
+ out.writeInt(idx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ matrixId = U.readGridUuid(in);
+ affinityKey = U.readGridUuid(in);
+ idx = in.readInt();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+ BinaryRawWriter out = writer.rawWriter();
+
+ BinaryUtils.writeIgniteUuid(out, matrixId);
+ BinaryUtils.writeIgniteUuid(out, affinityKey);
+ out.writeInt(idx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+ BinaryRawReader in = reader.rawReader();
+
+ matrixId = BinaryUtils.readIgniteUuid(in);
+ affinityKey = BinaryUtils.readIgniteUuid(in);
+ idx = in.readInt();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = 1;
+
+ res += res * 37 + matrixId.hashCode();
+ res += res * 37 + idx;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+
+ if (obj == null || obj.getClass() != getClass())
+ return false;
+
+ SparseMatrixKey that = (SparseMatrixKey)obj;
+
+ return idx == that.idx && matrixId.equals(that.matrixId) && F.eq(affinityKey, that.affinityKey);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(SparseMatrixKey.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/package-info.java
new file mode 100644
index 0000000..3a68ee2
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * * <!-- Package description. -->
+ * Contains matrix cache key implementations.
+ */
+package org.apache.ignite.ml.math.distributed.keys.impl;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/package-info.java
new file mode 100644
index 0000000..8954c6e
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * * <!-- Package description. -->
+ * Contains matrix cache keys.
+ */
+package org.apache.ignite.ml.math.distributed.keys;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/46ec148c/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/package-info.java
new file mode 100644
index 0000000..ad7399b
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains classes for distribution support.
+ */
+package org.apache.ignite.ml.math.distributed;
\ No newline at end of file