You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2018/04/13 09:33:13 UTC
[01/54] [abbrv] ignite git commit: IGNITE-7222 Added ZooKeeper
discovery SPI
Repository: ignite
Updated Branches:
refs/heads/ignite-6083 6e92fffca -> ecefdd335
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/test/java/org/apache/zookeeper/ZkTestClientCnxnSocketNIO.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/zookeeper/ZkTestClientCnxnSocketNIO.java b/modules/zookeeper/src/test/java/org/apache/zookeeper/ZkTestClientCnxnSocketNIO.java
new file mode 100644
index 0000000..7892b5e
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/zookeeper/ZkTestClientCnxnSocketNIO.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.channels.SelectionKey;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.logger.java.JavaLogger;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class ZkTestClientCnxnSocketNIO extends ClientCnxnSocketNIO {
+ /** */
+ public static final IgniteLogger log = new JavaLogger().getLogger(ZkTestClientCnxnSocketNIO.class);
+
+ /** */
+ public static volatile boolean DEBUG = false;
+
+ /** */
+ public volatile CountDownLatch blockConnectLatch;
+
+ /** */
+ public static ConcurrentHashMap<String, ZkTestClientCnxnSocketNIO> clients = new ConcurrentHashMap<>();
+
+ /** */
+ private final String nodeName;
+
+ /**
+ *
+ */
+ public static void reset() {
+ clients.clear();
+ }
+
+ /**
+ * @param node Node.
+ * @return ZK client.
+ */
+ public static ZkTestClientCnxnSocketNIO forNode(Ignite node) {
+ return clients.get(node.name());
+ }
+
+ /**
+ * @param instanceName Ignite instance name.
+ * @return ZK client.
+ */
+ public static ZkTestClientCnxnSocketNIO forNode(String instanceName) {
+ return clients.get(instanceName);
+ }
+
+ /**
+ * @throws IOException If failed.
+ */
+ public ZkTestClientCnxnSocketNIO() throws IOException {
+ super();
+
+ String threadName = Thread.currentThread().getName();
+
+ nodeName = threadName.substring(threadName.indexOf('-') + 1);
+
+ if (DEBUG)
+ log.info("ZkTestClientCnxnSocketNIO created for node: " + nodeName);
+ }
+
+ /** {@inheritDoc} */
+ @Override void connect(InetSocketAddress addr) throws IOException {
+ CountDownLatch blockConnect = this.blockConnectLatch;
+
+ if (DEBUG)
+ log.info("ZkTestClientCnxnSocketNIO connect [node=" + nodeName + ", addr=" + addr + ']');
+
+ if (blockConnect != null && blockConnect.getCount() > 0) {
+ try {
+ log.info("ZkTestClientCnxnSocketNIO block connect");
+
+ blockConnect.await(60, TimeUnit.SECONDS);
+
+ log.info("ZkTestClientCnxnSocketNIO finish block connect");
+ }
+ catch (Exception e) {
+ log.error("Error in ZkTestClientCnxnSocketNIO: " + e, e);
+ }
+ }
+
+ super.connect(addr);
+
+ clients.put(nodeName, this);
+ }
+
+ /**
+ *
+ */
+ public void allowConnect() {
+ assert blockConnectLatch != null && blockConnectLatch.getCount() == 1 : blockConnectLatch;
+
+ log.info("ZkTestClientCnxnSocketNIO allowConnect [node=" + nodeName + ']');
+
+ blockConnectLatch.countDown();
+ }
+
+ /**
+ * @param blockConnect {@code True} to block client reconnect.
+ * @throws Exception If failed.
+ */
+ public void closeSocket(boolean blockConnect) throws Exception {
+ if (blockConnect)
+ blockConnectLatch = new CountDownLatch(1);
+
+ log.info("ZkTestClientCnxnSocketNIO closeSocket [node=" + nodeName + ", block=" + blockConnect + ']');
+
+ SelectionKey k = GridTestUtils.getFieldValue(this, ClientCnxnSocketNIO.class, "sockKey");
+
+ k.channel().close();
+ }
+}
[11/54] [abbrv] ignite git commit: IGNITE-7222 Added ZooKeeper
discovery SPI
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
index 9402a32..38450df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
@@ -155,22 +155,30 @@ final class BinaryMetadataTransport {
* @param metadata Metadata proposed for update.
* @return Future to wait for update result on.
*/
- GridFutureAdapter<MetadataUpdateResult> requestMetadataUpdate(BinaryMetadata metadata) throws IgniteCheckedException {
+ GridFutureAdapter<MetadataUpdateResult> requestMetadataUpdate(BinaryMetadata metadata) {
MetadataUpdateResultFuture resFut = new MetadataUpdateResultFuture();
if (log.isDebugEnabled())
log.debug("Requesting metadata update for " + metadata.typeId() + "; caller thread is blocked on future "
+ resFut);
- synchronized (this) {
- unlabeledFutures.add(resFut);
+ try {
+ synchronized (this) {
+ unlabeledFutures.add(resFut);
- if (!stopping)
- discoMgr.sendCustomEvent(new MetadataUpdateProposedMessage(metadata, ctx.localNodeId()));
- else
- resFut.onDone(MetadataUpdateResult.createUpdateDisabledResult());
+ if (!stopping)
+ discoMgr.sendCustomEvent(new MetadataUpdateProposedMessage(metadata, ctx.localNodeId()));
+ else
+ resFut.onDone(MetadataUpdateResult.createUpdateDisabledResult());
+ }
+ }
+ catch (Exception e) {
+ resFut.onDone(MetadataUpdateResult.createUpdateDisabledResult(), e);
}
+ if (ctx.clientDisconnected())
+ onDisconnected();
+
return resFut;
}
@@ -237,6 +245,8 @@ final class BinaryMetadataTransport {
for (MetadataUpdateResultFuture fut : unlabeledFutures)
fut.onDone(res);
+ unlabeledFutures.clear();
+
for (MetadataUpdateResultFuture fut : syncMap.values())
fut.onDone(res);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateAcceptedMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateAcceptedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateAcceptedMessage.java
index 0416746..df64613 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateAcceptedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateAcceptedMessage.java
@@ -71,6 +71,11 @@ public class MetadataUpdateAcceptedMessage implements DiscoveryCustomMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
AffinityTopologyVersion topVer, DiscoCache discoCache) {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateProposedMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateProposedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateProposedMessage.java
index f9bd660..84e32e1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateProposedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateProposedMessage.java
@@ -134,6 +134,11 @@ public final class MetadataUpdateProposedMessage implements DiscoveryCustomMessa
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
AffinityTopologyVersion topVer, DiscoCache discoCache) {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index 9b3c1ec..5bbbb31 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -598,6 +598,45 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
+ @Override public List<List<ClusterNode>> allOwners() {
+ lock.readLock().lock();
+
+ try {
+ int parts = partitions();
+
+ List<List<ClusterNode>> res = new ArrayList<>(parts);
+
+ for (int i = 0; i < parts; i++)
+ res.add(new ArrayList<>());
+
+ List<ClusterNode> allNodes = discoCache.cacheGroupAffinityNodes(grpId);
+
+ for (int i = 0; i < allNodes.size(); i++) {
+ ClusterNode node = allNodes.get(i);
+
+ GridDhtPartitionMap nodeParts = node2part.get(node.id());
+
+ if (nodeParts != null) {
+ for (Map.Entry<Integer, GridDhtPartitionState> e : nodeParts.map().entrySet()) {
+ if (e.getValue() == OWNING) {
+ int part = e.getKey();
+
+ List<ClusterNode> owners = res.get(part);
+
+ owners.add(node);
+ }
+ }
+ }
+ }
+
+ return res;
+ }
+ finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public List<ClusterNode> moving(int p) {
return nodes(p, AffinityTopologyVersion.NONE, MOVING, null);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index ba55543..ea99f5d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -970,6 +970,10 @@ public abstract class GridDhtCacheAdapter<K, V> extends GridDistributedCacheAdap
try {
ctx.io().send(nodeId, res, ctx.ioPolicy());
}
+ catch (ClusterTopologyCheckedException e) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to send get response to node, node failed: " + nodeId);
+ }
catch (IgniteCheckedException e) {
U.error(log, "Failed to send get response to node (is node still alive?) [nodeId=" + nodeId +
",req=" + req + ", res=" + res + ']', e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index 13564c2..7f900cb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -237,6 +237,12 @@ public interface GridDhtPartitionTopology {
public List<ClusterNode> owners(int p);
/**
+ * @return List indexed by partition number, each list element is collection of all nodes who
+ * owns corresponding partition.
+ */
+ public List<List<ClusterNode>> allOwners();
+
+ /**
* @param p Partition ID.
* @param topVer Topology version.
* @return Collection of all nodes who {@code own} this partition.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 528f0a6..538c57e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -1217,6 +1217,45 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
+ @Override public List<List<ClusterNode>> allOwners() {
+ lock.readLock().lock();
+
+ try {
+ int parts = partitions();
+
+ List<List<ClusterNode>> res = new ArrayList<>(parts);
+
+ for (int i = 0; i < parts; i++)
+ res.add(new ArrayList<>());
+
+ List<ClusterNode> allNodes = discoCache.cacheGroupAffinityNodes(grp.groupId());
+
+ for (int i = 0; i < allNodes.size(); i++) {
+ ClusterNode node = allNodes.get(i);
+
+ GridDhtPartitionMap nodeParts = node2part.get(node.id());
+
+ if (nodeParts != null) {
+ for (Map.Entry<Integer, GridDhtPartitionState> e : nodeParts.map().entrySet()) {
+ if (e.getValue() == OWNING) {
+ int part = e.getKey();
+
+ List<ClusterNode> owners = res.get(part);
+
+ owners.add(node);
+ }
+ }
+ }
+ }
+
+ return res;
+ }
+ finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public List<ClusterNode> moving(int p) {
if (!grp.rebalanceEnabled())
return ownersAndMoving(p, AffinityTopologyVersion.NONE);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 8da91a8..cbb4985 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1506,12 +1506,16 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
catch (ClusterTopologyCheckedException ignore) {
if (log.isDebugEnabled())
- log.debug("Oldest node left during partition exchange [nodeId=" + oldestNode.id() +
+ log.debug("Coordinator left during partition exchange [nodeId=" + oldestNode.id() +
", exchId=" + exchId + ']');
}
catch (IgniteCheckedException e) {
- U.error(log, "Failed to send local partitions to oldest node (will retry after timeout) [oldestNodeId=" +
- oldestNode.id() + ", exchId=" + exchId + ']', e);
+ if (reconnectOnError(e))
+ onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+ else {
+ U.error(log, "Failed to send local partitions to coordinator [crd=" + oldestNode.id() +
+ ", exchId=" + exchId + ']', e);
+ }
}
}
@@ -3369,9 +3373,13 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
if (allReceived) {
- awaitSingleMapUpdates();
+ cctx.kernalContext().getSystemExecutorService().submit(new Runnable() {
+ @Override public void run() {
+ awaitSingleMapUpdates();
- onAllReceived(null);
+ onAllReceived(null);
+ }
+ });
}
}
else {
@@ -3399,7 +3407,13 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
", newCrd=" + crd0.id() + ']');
}
- sendPartitions(crd0);
+ final ClusterNode newCrd = crd0;
+
+ cctx.kernalContext().getSystemExecutorService().submit(new Runnable() {
+ @Override public void run() {
+ sendPartitions(newCrd);
+ }
+ });
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
index d7dfa16..bbbd999 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
@@ -94,6 +94,11 @@ public class ChangeGlobalStateFinishMessage implements DiscoveryCustomMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
AffinityTopologyVersion topVer, DiscoCache discoCache) {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
index 50fc022..81855fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
@@ -131,6 +131,11 @@ public class ChangeGlobalStateMessage implements DiscoveryCustomMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr, AffinityTopologyVersion topVer,
DiscoCache discoCache) {
return mgr.createDiscoCacheOnCacheChange(topVer, discoCache);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
new file mode 100644
index 0000000..5c3044b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cluster;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.GridDirectMap;
+import org.apache.ignite.internal.managers.communication.GridIoMessageFactory;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class ClusterMetricsUpdateMessage implements Message {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private byte[] nodeMetrics;
+
+ /** */
+ @GridDirectMap(keyType = UUID.class, valueType = byte[].class)
+ private Map<UUID, byte[]> allNodesMetrics;
+
+ /**
+ * Required by {@link GridIoMessageFactory}.
+ */
+ public ClusterMetricsUpdateMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param nodeMetrics Node metrics.
+ */
+ ClusterMetricsUpdateMessage(byte[] nodeMetrics) {
+ this.nodeMetrics = nodeMetrics;
+ }
+
+ /**
+ * @param allNodesMetrics All nodes metrcis.
+ */
+ ClusterMetricsUpdateMessage(Map<UUID, byte[]> allNodesMetrics) {
+ this.allNodesMetrics = allNodesMetrics;
+ }
+
+ /**
+ * @return Node metrics.
+ */
+ @Nullable byte[] nodeMetrics() {
+ return nodeMetrics;
+ }
+
+ /**
+ * @return All nodes metrics.
+ */
+ @Nullable Map<UUID, byte[]> allNodesMetrics() {
+ return allNodesMetrics;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeMap("allNodesMetrics", allNodesMetrics, MessageCollectionItemType.UUID, MessageCollectionItemType.BYTE_ARR))
+ return false;
+
+ writer.incrementState();
+
+ case 1:
+ if (!writer.writeByteArray("nodeMetrics", nodeMetrics))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ allNodesMetrics = reader.readMap("allNodesMetrics", MessageCollectionItemType.UUID, MessageCollectionItemType.BYTE_ARR, false);
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 1:
+ nodeMetrics = reader.readByteArray("nodeMetrics");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(ClusterMetricsUpdateMessage.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return 133;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ClusterMetricsUpdateMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterNodeMetrics.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterNodeMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterNodeMetrics.java
new file mode 100644
index 0000000..22a385f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterNodeMetrics.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cluster;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.internal.ClusterMetricsSnapshot;
+
+/**
+ *
+ */
+class ClusterNodeMetrics implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final byte[] metrics;
+
+ /** */
+ private final Map<Integer, CacheMetrics> cacheMetrics;
+
+ /**
+ * @param metrics Metrics.
+ * @param cacheMetrics Cache metrics.
+ */
+ ClusterNodeMetrics(ClusterMetrics metrics, Map<Integer, CacheMetrics> cacheMetrics) {
+ this.metrics = ClusterMetricsSnapshot.serialize(metrics);
+ this.cacheMetrics = cacheMetrics;
+ }
+
+ /**
+ * @return Metrics.
+ */
+ byte[] metrics() {
+ return metrics;
+ }
+
+ /**
+ * @return Cache metrics.
+ */
+ Map<Integer, CacheMetrics> cacheMetrics() {
+ return cacheMetrics != null ? cacheMetrics : Collections.<Integer, CacheMetrics>emptyMap();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
index 5f2c66c..8796302 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
@@ -33,6 +33,8 @@ import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.ClusterMetricsSnapshot;
+import org.apache.ignite.internal.GridDirectMap;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteDiagnosticInfo;
import org.apache.ignite.internal.IgniteDiagnosticMessage;
@@ -42,21 +44,29 @@ import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.cluster.IgniteClusterImpl;
import org.apache.ignite.internal.managers.communication.GridIoPolicy;
import org.apache.ignite.internal.managers.communication.GridMessageListener;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.GridProcessorAdapter;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
import org.apache.ignite.internal.util.GridTimerTask;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.CI1;
+import org.apache.ignite.internal.util.typedef.internal.LT;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteClosure;
import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.marshaller.jdk.JdkMarshaller;
import org.apache.ignite.spi.discovery.DiscoveryDataBag;
import org.apache.ignite.spi.discovery.DiscoveryDataBag.GridDiscoveryData;
+import org.apache.ignite.spi.discovery.DiscoveryMetricsProvider;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_DIAGNOSTIC_ENABLED;
@@ -66,6 +76,7 @@ import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.internal.GridComponent.DiscoveryDataExchangeType.CLUSTER_PROC;
import static org.apache.ignite.internal.GridTopic.TOPIC_INTERNAL_DIAGNOSTIC;
+import static org.apache.ignite.internal.GridTopic.TOPIC_METRICS;
import static org.apache.ignite.internal.IgniteVersionUtils.VER_STR;
/**
@@ -102,6 +113,18 @@ public class ClusterProcessor extends GridProcessorAdapter {
/** */
private final AtomicLong diagFutId = new AtomicLong();
+ /** */
+ private final Map<UUID, byte[]> allNodesMetrics = new ConcurrentHashMap<>();
+
+ /** */
+ private final JdkMarshaller marsh = new JdkMarshaller();
+
+ /** */
+ private DiscoveryMetricsProvider metricsProvider;
+
+ /** */
+ private boolean sndMetrics;
+
/**
* @param ctx Kernal context.
*/
@@ -111,6 +134,8 @@ public class ClusterProcessor extends GridProcessorAdapter {
notifyEnabled.set(IgniteSystemProperties.getBoolean(IGNITE_UPDATE_NOTIFIER, true));
cluster = new IgniteClusterImpl(ctx);
+
+ sndMetrics = !(ctx.config().getDiscoverySpi() instanceof TcpDiscoverySpi);
}
/**
@@ -120,33 +145,31 @@ public class ClusterProcessor extends GridProcessorAdapter {
return getBoolean(IGNITE_DIAGNOSTIC_ENABLED, true);
}
- /** */
- private final JdkMarshaller marsh = new JdkMarshaller();
-
/**
* @throws IgniteCheckedException If failed.
*/
public void initDiagnosticListeners() throws IgniteCheckedException {
ctx.event().addLocalEventListener(new GridLocalEventListener() {
- @Override public void onEvent(Event evt) {
- assert evt instanceof DiscoveryEvent;
- assert evt.type() == EVT_NODE_FAILED || evt.type() == EVT_NODE_LEFT;
+ @Override public void onEvent(Event evt) {
+ assert evt instanceof DiscoveryEvent;
+ assert evt.type() == EVT_NODE_FAILED || evt.type() == EVT_NODE_LEFT;
- DiscoveryEvent discoEvt = (DiscoveryEvent)evt;
+ DiscoveryEvent discoEvt = (DiscoveryEvent)evt;
- UUID nodeId = discoEvt.eventNode().id();
+ UUID nodeId = discoEvt.eventNode().id();
- ConcurrentHashMap<Long, InternalDiagnosticFuture> futs = diagnosticFutMap.get();
+ ConcurrentHashMap<Long, InternalDiagnosticFuture> futs = diagnosticFutMap.get();
- if (futs != null) {
- for (InternalDiagnosticFuture fut : futs.values()) {
- if (fut.nodeId.equals(nodeId))
- fut.onDone(new IgniteDiagnosticInfo("Target node failed: " + nodeId));
- }
+ if (futs != null) {
+ for (InternalDiagnosticFuture fut : futs.values()) {
+ if (fut.nodeId.equals(nodeId))
+ fut.onDone(new IgniteDiagnosticInfo("Target node failed: " + nodeId));
}
}
- },
- EVT_NODE_FAILED, EVT_NODE_LEFT);
+
+ allNodesMetrics.remove(nodeId);
+ }
+ }, EVT_NODE_FAILED, EVT_NODE_LEFT);
ctx.io().addMessageListener(TOPIC_INTERNAL_DIAGNOSTIC, new GridMessageListener() {
@Override public void onMessage(UUID nodeId, Object msg, byte plc) {
@@ -233,6 +256,17 @@ public class ClusterProcessor extends GridProcessorAdapter {
U.warn(diagnosticLog, "Received unexpected message: " + msg);
}
});
+
+ if (sndMetrics) {
+ ctx.io().addMessageListener(TOPIC_METRICS, new GridMessageListener() {
+ @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
+ if (msg instanceof ClusterMetricsUpdateMessage)
+ processMetricsUpdateMessage(nodeId, (ClusterMetricsUpdateMessage)msg);
+ else
+ U.warn(log, "Received unexpected message for TOPIC_METRICS: " + msg);
+ }
+ });
+ }
}
/**
@@ -296,7 +330,6 @@ public class ClusterProcessor extends GridProcessorAdapter {
}
}
-
/**
* @param vals collection to seek through.
*/
@@ -334,6 +367,14 @@ public class ClusterProcessor extends GridProcessorAdapter {
log.debug("Failed to create GridUpdateNotifier: " + e);
}
}
+
+ if (sndMetrics) {
+ metricsProvider = ctx.discovery().createMetricsProvider();
+
+ long updateFreq = ctx.config().getMetricsUpdateFrequency();
+
+ ctx.timeout().addTimeoutObject(new MetricsUpdateTimeoutObject(updateFreq));
+ }
}
/** {@inheritDoc} */
@@ -352,6 +393,133 @@ public class ClusterProcessor extends GridProcessorAdapter {
}
/**
+ * @param sndNodeId Sender node ID.
+ * @param msg Message.
+ */
+ private void processMetricsUpdateMessage(UUID sndNodeId, ClusterMetricsUpdateMessage msg) {
+ byte[] nodeMetrics = msg.nodeMetrics();
+
+ if (nodeMetrics != null) {
+ assert msg.allNodesMetrics() == null;
+
+ allNodesMetrics.put(sndNodeId, nodeMetrics);
+
+ updateNodeMetrics(ctx.discovery().discoCache(), sndNodeId, nodeMetrics);
+ }
+ else {
+ Map<UUID, byte[]> allNodesMetrics = msg.allNodesMetrics();
+
+ assert allNodesMetrics != null;
+
+ DiscoCache discoCache = ctx.discovery().discoCache();
+
+ for (Map.Entry<UUID, byte[]> e : allNodesMetrics.entrySet()) {
+ if (!ctx.localNodeId().equals(e.getKey()))
+ updateNodeMetrics(discoCache, e.getKey(), e.getValue());
+ }
+ }
+ }
+
+ /**
+ * @param discoCache Discovery data cache.
+ * @param nodeId Node ID.
+ * @param metricsBytes Marshalled metrics.
+ */
+ private void updateNodeMetrics(DiscoCache discoCache, UUID nodeId, byte[] metricsBytes) {
+ ClusterNode node = discoCache.node(nodeId);
+
+ if (node == null || !discoCache.alive(nodeId))
+ return;
+
+ try {
+ ClusterNodeMetrics metrics = U.unmarshalZip(ctx.config().getMarshaller(), metricsBytes, null);
+
+ assert node instanceof IgniteClusterNode : node;
+
+ IgniteClusterNode node0 = (IgniteClusterNode)node;
+
+ node0.setMetrics(ClusterMetricsSnapshot.deserialize(metrics.metrics(), 0));
+ node0.setCacheMetrics(metrics.cacheMetrics());
+
+ ctx.discovery().metricsUpdateEvent(discoCache, node0);
+ }
+ catch (IgniteCheckedException e) {
+ U.warn(log, "Failed to unmarshal node metrics: " + e);
+ }
+ }
+
+ /**
+ *
+ */
+ private void updateMetrics() {
+ if (ctx.isStopping() || ctx.clientDisconnected())
+ return;
+
+ ClusterNode oldest = ctx.discovery().oldestAliveServerNode(AffinityTopologyVersion.NONE);
+
+ if (oldest == null)
+ return;
+
+ if (ctx.localNodeId().equals(oldest.id())) {
+ IgniteClusterNode locNode = (IgniteClusterNode)ctx.discovery().localNode();
+
+ locNode.setMetrics(metricsProvider.metrics());
+ locNode.setCacheMetrics(metricsProvider.cacheMetrics());
+
+ ClusterNodeMetrics metrics = new ClusterNodeMetrics(locNode.metrics(), locNode.cacheMetrics());
+
+ try {
+ byte[] metricsBytes = U.zip(U.marshal(ctx.config().getMarshaller(), metrics));
+
+ allNodesMetrics.put(ctx.localNodeId(), metricsBytes);
+ }
+ catch (IgniteCheckedException e) {
+ U.warn(log, "Failed to marshal local node metrics: " + e, e);
+ }
+
+ ctx.discovery().metricsUpdateEvent(ctx.discovery().discoCache(), locNode);
+
+ Collection<ClusterNode> allNodes = ctx.discovery().allNodes();
+
+ ClusterMetricsUpdateMessage msg = new ClusterMetricsUpdateMessage(new HashMap<>(allNodesMetrics));
+
+ for (ClusterNode node : allNodes) {
+ if (ctx.localNodeId().equals(node.id()) || !ctx.discovery().alive(node.id()))
+ continue;
+
+ try {
+ ctx.io().sendToGridTopic(node, TOPIC_METRICS, msg, GridIoPolicy.SYSTEM_POOL);
+ }
+ catch (ClusterTopologyCheckedException e) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to send metrics update, node failed: " + e);
+ }
+ catch (IgniteCheckedException e) {
+ U.warn(log, "Failed to send metrics update: " + e, e);
+ }
+ }
+ }
+ else {
+ ClusterNodeMetrics metrics = new ClusterNodeMetrics(metricsProvider.metrics(), metricsProvider.cacheMetrics());
+
+ try {
+ byte[] metricsBytes = U.zip(U.marshal(ctx.config().getMarshaller(), metrics));
+
+ ClusterMetricsUpdateMessage msg = new ClusterMetricsUpdateMessage(metricsBytes);
+
+ ctx.io().sendToGridTopic(oldest, TOPIC_METRICS, msg, GridIoPolicy.SYSTEM_POOL);
+ }
+ catch (ClusterTopologyCheckedException e) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to send metrics update to oldest, node failed: " + e);
+ }
+ catch (IgniteCheckedException e) {
+ LT.warn(log, e, "Failed to send metrics update to oldest: " + e, false, false);
+ }
+ }
+ }
+
+ /**
* Disables update notifier.
*/
public void disableUpdateNotifier() {
@@ -571,4 +739,51 @@ public class ClusterProcessor extends GridProcessorAdapter {
return S.toString(InternalDiagnosticFuture.class, this);
}
}
+
+ /**
+ *
+ */
+ private class MetricsUpdateTimeoutObject implements GridTimeoutObject, Runnable {
+ /** */
+ private final IgniteUuid id = IgniteUuid.randomUuid();
+
+ /** */
+ private long endTime;
+
+ /** */
+ private final long timeout;
+
+ /**
+ * @param timeout Timeout.
+ */
+ MetricsUpdateTimeoutObject(long timeout) {
+ this.timeout = timeout;
+
+ endTime = U.currentTimeMillis() + timeout;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid timeoutId() {
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long endTime() {
+ return endTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ updateMetrics();
+
+ endTime = U.currentTimeMillis() + timeout;
+
+ ctx.timeout().addTimeoutObject(this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onTimeout() {
+ ctx.getSystemExecutorService().execute(this);
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/AbstractContinuousMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/AbstractContinuousMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/AbstractContinuousMessage.java
index e9754d1..928c619 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/AbstractContinuousMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/AbstractContinuousMessage.java
@@ -63,6 +63,11 @@ public abstract class AbstractContinuousMessage implements DiscoveryCustomMessag
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
AffinityTopologyVersion topVer, DiscoCache discoCache) {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineInfo.java
new file mode 100644
index 0000000..fc0f181
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineInfo.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.io.Serializable;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+class ContinuousRoutineInfo implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ UUID srcNodeId;
+
+ /** */
+ final UUID routineId;
+
+ /** */
+ final byte[] hnd;
+
+ /** */
+ final byte[] nodeFilter;
+
+ /** */
+ final int bufSize;
+
+ /** */
+ final long interval;
+
+ /** */
+ final boolean autoUnsubscribe;
+
+ /** */
+ transient boolean disconnected;
+
+ /**
+ * @param srcNodeId Source node ID.
+ * @param routineId Routine ID.
+ * @param hnd Marshalled handler.
+ * @param nodeFilter Marshalled node filter.
+ * @param bufSize Handler buffer size.
+ * @param interval Time interval.
+ * @param autoUnsubscribe Auto unsubscribe flag.
+ */
+ ContinuousRoutineInfo(
+ UUID srcNodeId,
+ UUID routineId,
+ byte[] hnd,
+ byte[] nodeFilter,
+ int bufSize,
+ long interval,
+ boolean autoUnsubscribe)
+ {
+ this.srcNodeId = srcNodeId;
+ this.routineId = routineId;
+ this.hnd = hnd;
+ this.nodeFilter = nodeFilter;
+ this.bufSize = bufSize;
+ this.interval = interval;
+ this.autoUnsubscribe = autoUnsubscribe;
+ }
+
+ /**
+ * @param srcNodeId Source node ID.
+ */
+ void sourceNodeId(UUID srcNodeId) {
+ this.srcNodeId = srcNodeId;
+ }
+
+ /**
+ *
+ */
+ void onDisconnected() {
+ disconnected = true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ContinuousRoutineInfo.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
new file mode 100644
index 0000000..581ac60
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class ContinuousRoutineStartResultMessage implements Message {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private static final int ERROR_FLAG = 0x01;
+
+ /** */
+ private UUID routineId;
+
+ /** */
+ private byte[] errBytes;
+
+ /** */
+ private byte[] cntrsMapBytes;
+
+ /** */
+ private int flags;
+
+ /**
+ *
+ */
+ public ContinuousRoutineStartResultMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param routineId Routine ID.
+ * @param cntrsMapBytes Marshalled {@link CachePartitionPartialCountersMap}.
+ * @param errBytes Error bytes.
+ * @param err {@code True} if failed to start routine.
+ */
+ ContinuousRoutineStartResultMessage(UUID routineId, byte[] cntrsMapBytes, byte[] errBytes, boolean err) {
+ this.routineId = routineId;
+ this.cntrsMapBytes = cntrsMapBytes;
+ this.errBytes = errBytes;
+
+ if (err)
+ flags |= ERROR_FLAG;
+ }
+
+ /**
+ * @return Marshalled {@link CachePartitionPartialCountersMap}.
+ */
+ @Nullable byte[] countersMapBytes() {
+ return cntrsMapBytes;
+ }
+
+ /**
+ * @return {@code True} if failed to start routine.
+ */
+ boolean error() {
+ return (flags & ERROR_FLAG) != 0;
+ }
+
+ /**
+ * @return Routine ID.
+ */
+ UUID routineId() {
+ return routineId;
+ }
+
+ /**
+ * @return Error bytes.
+ */
+ @Nullable byte[] errorBytes() {
+ return errBytes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeByteArray("cntrsMapBytes", cntrsMapBytes))
+ return false;
+
+ writer.incrementState();
+
+ case 1:
+ if (!writer.writeByteArray("errBytes", errBytes))
+ return false;
+
+ writer.incrementState();
+
+ case 2:
+ if (!writer.writeInt("flags", flags))
+ return false;
+
+ writer.incrementState();
+
+ case 3:
+ if (!writer.writeUuid("routineId", routineId))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ cntrsMapBytes = reader.readByteArray("cntrsMapBytes");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 1:
+ errBytes = reader.readByteArray("errBytes");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 2:
+ flags = reader.readInt("flags");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 3:
+ routineId = reader.readUuid("routineId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(ContinuousRoutineStartResultMessage.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return 134;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 4;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ContinuousRoutineStartResultMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesCommonDiscoveryData.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesCommonDiscoveryData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesCommonDiscoveryData.java
new file mode 100644
index 0000000..d29de89
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesCommonDiscoveryData.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+public class ContinuousRoutinesCommonDiscoveryData implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final List<ContinuousRoutineInfo> startedRoutines;
+
+ /**
+ * @param startedRoutines Routines started in cluster.
+ */
+ ContinuousRoutinesCommonDiscoveryData(List<ContinuousRoutineInfo> startedRoutines) {
+ this.startedRoutines = startedRoutines;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ContinuousRoutinesCommonDiscoveryData.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesInfo.java
new file mode 100644
index 0000000..ad24ff1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesInfo.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.DiscoveryDataBag;
+
+import static org.apache.ignite.internal.GridComponent.DiscoveryDataExchangeType.CONTINUOUS_PROC;
+
+/**
+ *
+ */
+class ContinuousRoutinesInfo {
+ /** */
+ private final Map<UUID, ContinuousRoutineInfo> startedRoutines = new HashMap<>();
+
+ /**
+ * @param dataBag Discovery data bag.
+ */
+ void collectGridNodeData(DiscoveryDataBag dataBag) {
+ synchronized (startedRoutines) {
+ if (!dataBag.commonDataCollectedFor(CONTINUOUS_PROC.ordinal()))
+ dataBag.addGridCommonData(CONTINUOUS_PROC.ordinal(),
+ new ContinuousRoutinesCommonDiscoveryData(new ArrayList<>(startedRoutines.values())));
+ }
+ }
+
+ /**
+ * @param dataBag Discovery data bag.
+ */
+ void collectJoiningNodeData(DiscoveryDataBag dataBag) {
+ synchronized (startedRoutines) {
+ for (ContinuousRoutineInfo info : startedRoutines.values()) {
+ if (info.disconnected)
+ info.sourceNodeId(dataBag.joiningNodeId());
+ }
+
+ dataBag.addJoiningNodeData(CONTINUOUS_PROC.ordinal(),
+ new ContinuousRoutinesJoiningNodeDiscoveryData(new ArrayList<>(startedRoutines.values())));
+ }
+ }
+
+ /**
+ * @param info Routine info.
+ */
+ void addRoutineInfo(ContinuousRoutineInfo info) {
+ synchronized (startedRoutines) {
+ startedRoutines.put(info.routineId, info);
+ }
+ }
+
+ /**
+ * @param routineId Routine ID.
+ * @return {@code True} if routine exists.
+ */
+ boolean routineExists(UUID routineId) {
+ synchronized (startedRoutines) {
+ return startedRoutines.containsKey(routineId);
+ }
+ }
+
+ /**
+ * @param routineId Routine ID.
+ */
+ void removeRoutine(UUID routineId) {
+ synchronized (startedRoutines) {
+ startedRoutines.remove(routineId);
+ }
+ }
+
+ /**
+ * @param locRoutines Routines IDs which can survive reconnect.
+ */
+ void onClientDisconnected(Collection<UUID> locRoutines) {
+ synchronized (startedRoutines) {
+ for (Iterator<Map.Entry<UUID, ContinuousRoutineInfo>> it = startedRoutines.entrySet().iterator(); it.hasNext();) {
+ Map.Entry<UUID, ContinuousRoutineInfo> e = it.next();
+
+ ContinuousRoutineInfo info = e.getValue();
+
+ if (!locRoutines.contains(info.routineId))
+ it.remove();
+ else
+ info.onDisconnected();
+ }
+ }
+ }
+
+ /**
+ * Removes all routines with autoUnsubscribe=false started by given node.
+ *
+ * @param nodeId Node ID.
+ */
+ void onNodeFail(UUID nodeId) {
+ synchronized (startedRoutines) {
+ for (Iterator<Map.Entry<UUID, ContinuousRoutineInfo>> it = startedRoutines.entrySet().iterator(); it.hasNext();) {
+ Map.Entry<UUID, ContinuousRoutineInfo> e = it.next();
+
+ ContinuousRoutineInfo info = e.getValue();
+
+ if (info.autoUnsubscribe && info.srcNodeId.equals(nodeId))
+ it.remove();
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ContinuousRoutinesInfo.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesJoiningNodeDiscoveryData.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesJoiningNodeDiscoveryData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesJoiningNodeDiscoveryData.java
new file mode 100644
index 0000000..9be6ef8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesJoiningNodeDiscoveryData.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+public class ContinuousRoutinesJoiningNodeDiscoveryData implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final List<ContinuousRoutineInfo> startedRoutines;
+
+ /**
+ * @param startedRoutines Routines registered on nodes, to be started in cluster.
+ */
+ ContinuousRoutinesJoiningNodeDiscoveryData(List<ContinuousRoutineInfo> startedRoutines) {
+ this.startedRoutines = startedRoutines;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ContinuousRoutinesJoiningNodeDiscoveryData.class, this);
+ }
+}
[25/54] [abbrv] ignite git commit: IGNITE-8153 Nodes fail to connect
each other when SSL is enabled - Fixes #3773.
Posted by ag...@apache.org.
IGNITE-8153 Nodes fail to connect each other when SSL is enabled - Fixes #3773.
Signed-off-by: Valentin Kulichenko <va...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f4de6df7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f4de6df7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f4de6df7
Branch: refs/heads/ignite-6083
Commit: f4de6df71b256506ce36c1c4e16533bb063782a0
Parents: a96ac04
Author: mcherkasov <mc...@gridgain.com>
Authored: Tue Apr 10 17:23:29 2018 -0700
Committer: Valentin Kulichenko <va...@gmail.com>
Committed: Tue Apr 10 17:23:29 2018 -0700
----------------------------------------------------------------------
.../ignite/internal/util/nio/ssl/BlockingSslHandler.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4de6df7/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
index 638106f..0099c46 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
@@ -373,9 +373,10 @@ public class BlockingSslHandler {
* @throws GridNioException If failed to pass event to the next filter.
*/
private Status unwrapHandshake() throws SSLException, IgniteCheckedException {
- // Flip input buffer so we can read the collected data.
- readFromNet();
+ if(!inNetBuf.hasRemaining())
+ readFromNet();
+ // Flip input buffer so we can read the collected data.
inNetBuf.flip();
SSLEngineResult res = unwrap0();
@@ -399,7 +400,10 @@ public class BlockingSslHandler {
else if (res.getStatus() == BUFFER_UNDERFLOW) {
inNetBuf.compact();
- inNetBuf = expandBuffer(inNetBuf, inNetBuf.capacity() * 2);
+ if(inNetBuf.capacity() == inNetBuf.limit())
+ inNetBuf = expandBuffer(inNetBuf, inNetBuf.capacity() * 2);
+
+ readFromNet();
}
else
// prepare to be written again
[23/54] [abbrv] ignite git commit: IGNITE-6430 Complete failing test
early
Posted by ag...@apache.org.
IGNITE-6430 Complete failing test early
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3a717658
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3a717658
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3a717658
Branch: refs/heads/ignite-6083
Commit: 3a7176582d622fc8a52a730f2a3f1e4b4319e4a3
Parents: 9bb4ce8
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Tue Apr 10 20:33:47 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Apr 10 20:33:47 2018 +0300
----------------------------------------------------------------------
.../cache/CacheGroupsMetricsRebalanceTest.java | 31 ++++++++++++--------
1 file changed, 18 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3a717658/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
index 89c8236..ceb9852 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
@@ -227,29 +227,34 @@ public class CacheGroupsMetricsRebalanceTest extends GridCommonAbstractTest {
System.out.println("Wait until keys left will be less " + keysLine);
- while (finishRebalanceLatch.getCount() != 0) {
- CacheMetrics m = ig2.cache(CACHE1).localMetrics();
+ try {
+ while (finishRebalanceLatch.getCount() != 0) {
+ CacheMetrics m = ig2.cache(CACHE1).localMetrics();
- long keyLeft = m.getKeysToRebalanceLeft();
+ long keyLeft = m.getKeysToRebalanceLeft();
- if (keyLeft > 0 && keyLeft < keysLine)
- latch.countDown();
+ if (keyLeft > 0 && keyLeft < keysLine)
+ latch.countDown();
- System.out.println("Keys left: " + m.getKeysToRebalanceLeft());
+ System.out.println("Keys left: " + m.getKeysToRebalanceLeft());
- try {
- Thread.sleep(1_000);
- }
- catch (InterruptedException e) {
- System.out.println("Interrupt thread: " + e.getMessage());
+ try {
+ Thread.sleep(1_000);
+ }
+ catch (InterruptedException e) {
+ System.out.println("Interrupt thread: " + e.getMessage());
- Thread.currentThread().interrupt();
+ Thread.currentThread().interrupt();
+ }
}
}
+ finally {
+ latch.countDown();
+ }
}
});
- latch.await();
+ assertTrue(latch.await(getTestTimeout(), TimeUnit.MILLISECONDS));
long finishTime = ig2.cache(CACHE1).localMetrics().getEstimatedRebalancingFinishTime();
[31/54] [abbrv] ignite git commit: IGNITE-8111 Add extra validation
for WAL segment size - Fixes #3768.
Posted by ag...@apache.org.
IGNITE-8111 Add extra validation for WAL segment size - Fixes #3768.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/97524668
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/97524668
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/97524668
Branch: refs/heads/ignite-6083
Commit: 975246687c9d143830501340e597a35d1a4c492a
Parents: a4653b7
Author: denis.garus <d....@isimplelab.com>
Authored: Wed Apr 11 13:01:22 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Wed Apr 11 13:01:22 2018 +0300
----------------------------------------------------------------------
.../configuration/DataStorageConfiguration.java | 6 ++--
.../DataStorageConfigurationValidationTest.java | 33 ++++++++++++++++++--
.../db/wal/IgniteWalFlushFailoverTest.java | 4 +--
...lFlushMultiNodeFailoverAbstractSelfTest.java | 4 +--
4 files changed, 39 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/97524668/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
index a433760..747efd8 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
@@ -533,12 +533,14 @@ public class DataStorageConfiguration implements Serializable {
/**
* Sets size of a WAL segment.
+ * If value is not set (or zero), {@link #DFLT_WAL_SEGMENT_SIZE} will be used.
*
- * @param walSegmentSize WAL segment size. 64 MB is used by default. Maximum value is 2Gb.
+ * @param walSegmentSize WAL segment size. Value must be between 512Kb and 2Gb.
* @return {@code This} for chaining.
*/
public DataStorageConfiguration setWalSegmentSize(int walSegmentSize) {
- A.ensure(walSegmentSize >= 0, "WAL segment size must be non-negative and less than 2 Gb.");
+ if (walSegmentSize != 0)
+ A.ensure(walSegmentSize >= 512 * 1024, "WAL segment size must be between 512Kb and 2Gb.");
this.walSegmentSize = walSegmentSize;
http://git-wip-us.apache.org/repos/asf/ignite/blob/97524668/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java
index 7f667ee..9471a82 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java
@@ -31,11 +31,10 @@ public class DataStorageConfigurationValidationTest extends TestCase {
*
* @throws Exception If failed.
*/
- public void testWalSegmentSizeOveflow() throws Exception {
+ public void testWalSegmentSizeOverflow() throws Exception {
final DataStorageConfiguration cfg = new DataStorageConfiguration();
GridTestUtils.assertThrows(null, new Callable<Void>() {
- /** {@inheritDoc} */
@Override public Void call() {
cfg.setWalSegmentSize(1 << 31);
@@ -43,4 +42,34 @@ public class DataStorageConfigurationValidationTest extends TestCase {
}
}, IllegalArgumentException.class, null);
}
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSetWalSegmentSizeShouldThrowExceptionWhenSizeLessThen512Kb() throws Exception {
+ final DataStorageConfiguration cfg = new DataStorageConfiguration();
+
+ GridTestUtils.assertThrows(null, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ cfg.setWalSegmentSize(512 * 1024 - 1);
+
+ return null;
+ }
+ }, IllegalArgumentException.class, null);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSetWalSegmentSizeShouldBeOkWhenSizeBetween512KbAnd2Gb() throws Exception {
+ final DataStorageConfiguration cfg = new DataStorageConfiguration();
+
+ cfg.setWalSegmentSize(512 * 1024);
+
+ assertEquals(512 * 1024, cfg.getWalSegmentSize());
+
+ cfg.setWalSegmentSize(Integer.MAX_VALUE);
+
+ assertEquals(Integer.MAX_VALUE, cfg.getWalSegmentSize());
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/97524668/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
index 042a447..351a42c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
@@ -92,8 +92,8 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
.setDefaultDataRegionConfiguration(
new DataRegionConfiguration().setMaxSize(2048L * 1024 * 1024).setPersistenceEnabled(true))
.setWalMode(WALMode.BACKGROUND)
- .setWalBufferSize(128 * 1024)// Setting WAL Segment size to high values forces flushing by timeout.
- .setWalSegmentSize(flushByTimeout ? 500_000 : 50_000);
+ .setWalBufferSize(1024 * 1024)// Setting WAL Segment size to high values forces flushing by timeout.
+ .setWalSegmentSize(flushByTimeout ? 2 * 1024 * 1024 : 512 * 1024);
cfg.setDataStorageConfiguration(memCfg);
http://git-wip-us.apache.org/repos/asf/ignite/blob/97524668/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index fe16328..cc0986a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -106,8 +106,8 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
.setDefaultDataRegionConfiguration(
new DataRegionConfiguration().setMaxSize(2048L * 1024 * 1024).setPersistenceEnabled(true))
.setWalMode(this.walMode())
- .setWalSegmentSize(50_000)
- .setWalBufferSize(50_000);
+ .setWalSegmentSize(512 * 1024)
+ .setWalBufferSize(512 * 1024);
cfg.setDataStorageConfiguration(memCfg);
[22/54] [abbrv] ignite git commit: IGNITE-8069
IgniteOutOfMemoryException should be handled accordingly to provided failure
handler
Posted by ag...@apache.org.
IGNITE-8069 IgniteOutOfMemoryException should be handled accordingly to provided failure handler
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9bb4ce8a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9bb4ce8a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9bb4ce8a
Branch: refs/heads/ignite-6083
Commit: 9bb4ce8ab9770967d39f0acac5cdc1dc4230abb4
Parents: c807ae9
Author: Aleksey Plekhanov <pl...@gmail.com>
Authored: Tue Apr 10 18:54:03 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Apr 10 18:54:03 2018 +0300
----------------------------------------------------------------------
.../pagemem/impl/PageMemoryNoStoreImpl.java | 17 ++-
.../persistence/pagemem/PageMemoryImpl.java | 17 ++-
.../failure/AbstractFailureHandlerTest.java | 74 ++++++++++
.../ignite/failure/IoomFailureHandlerTest.java | 144 +++++++++++++++++++
.../persistence/pagemem/PageMemoryImplTest.java | 9 ++
.../ignite/testsuites/IgniteBasicTestSuite.java | 2 +
6 files changed, 259 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
index 7424af6..d4b22a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
@@ -28,6 +28,8 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
import org.apache.ignite.internal.mem.DirectMemoryRegion;
import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
@@ -158,6 +160,9 @@ public class PageMemoryNoStoreImpl implements PageMemory {
/** */
private final boolean trackAcquiredPages;
+ /** Shared context. */
+ private final GridCacheSharedContext<?, ?> ctx;
+
/**
* @param log Logger.
* @param directMemoryProvider Memory allocator to use.
@@ -184,6 +189,7 @@ public class PageMemoryNoStoreImpl implements PageMemory {
this.trackAcquiredPages = trackAcquiredPages;
this.memMetrics = memMetrics;
this.dataRegionCfg = dataRegionCfg;
+ this.ctx = sharedCtx;
sysPageSize = pageSize + PAGE_OVERHEAD;
@@ -288,8 +294,8 @@ public class PageMemoryNoStoreImpl implements PageMemory {
}
}
- if (relPtr == INVALID_REL_PTR)
- throw new IgniteOutOfMemoryException("Out of memory in data region [" +
+ if (relPtr == INVALID_REL_PTR) {
+ IgniteOutOfMemoryException oom = new IgniteOutOfMemoryException("Out of memory in data region [" +
"name=" + dataRegionCfg.getName() +
", initSize=" + U.readableSize(dataRegionCfg.getInitialSize(), false) +
", maxSize=" + U.readableSize(dataRegionCfg.getMaxSize(), false) +
@@ -299,6 +305,13 @@ public class PageMemoryNoStoreImpl implements PageMemory {
" ^-- Enable eviction or expiration policies"
);
+ if (ctx != null)
+ ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom));
+
+ throw oom;
+ }
+
+
assert (relPtr & ~PageIdUtils.PAGE_IDX_MASK) == 0 : U.hexLong(relPtr & ~PageIdUtils.PAGE_IDX_MASK);
// Assign page ID according to flags and partition ID.
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 46fb7dd..4463224 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -40,6 +40,8 @@ import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
import org.apache.ignite.internal.mem.DirectMemoryRegion;
@@ -543,7 +545,7 @@ public class PageMemoryImpl implements PageMemoryEx {
catch (IgniteOutOfMemoryException oom) {
DataRegionConfiguration dataRegionCfg = getDataRegionConfiguration();
- throw (IgniteOutOfMemoryException) new IgniteOutOfMemoryException("Out of memory in data region [" +
+ IgniteOutOfMemoryException e = new IgniteOutOfMemoryException("Out of memory in data region [" +
"name=" + dataRegionCfg.getName() +
", initSize=" + U.readableSize(dataRegionCfg.getInitialSize(), false) +
", maxSize=" + U.readableSize(dataRegionCfg.getMaxSize(), false) +
@@ -551,7 +553,13 @@ public class PageMemoryImpl implements PageMemoryEx {
" ^-- Increase maximum off-heap memory size (DataRegionConfiguration.maxSize)" + U.nl() +
" ^-- Enable Ignite persistence (DataRegionConfiguration.persistenceEnabled)" + U.nl() +
" ^-- Enable eviction or expiration policies"
- ).initCause(oom);
+ );
+
+ e.initCause(oom);
+
+ ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+
+ throw e;
}
finally {
seg.writeLock().unlock();
@@ -746,6 +754,11 @@ public class PageMemoryImpl implements PageMemoryEx {
return absPtr;
}
+ catch (IgniteOutOfMemoryException oom) {
+ ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom));
+
+ throw oom;
+ }
finally {
seg.writeLock().unlock();
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
new file mode 100644
index 0000000..dc5f1f5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Abstract failure handler test.
+ */
+public class AbstractFailureHandlerTest extends GridCommonAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected FailureHandler getFailureHandler(String igniteInstanceName) {
+ return new DummyFailureHandler();
+ }
+
+ /**
+ * Gets dummy failure handler for ignite instance.
+ *
+ * @param ignite Ignite.
+ */
+ protected static DummyFailureHandler dummyFailureHandler(Ignite ignite) {
+ return (DummyFailureHandler)ignite.configuration().getFailureHandler();
+ }
+
+ /**
+ *
+ */
+ protected static class DummyFailureHandler implements FailureHandler {
+ /** Failure. */
+ private volatile boolean failure;
+
+ /** Failure context. */
+ private volatile FailureContext ctx;
+
+ /** {@inheritDoc} */
+ @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+ failure = true;
+
+ ctx = failureCtx;
+
+ return true;
+ }
+
+ /**
+ * @return Failure.
+ */
+ public boolean failure() {
+ return failure;
+ }
+
+ /**
+ * @return Failure context.
+ */
+ public FailureContext failureContext() {
+ return ctx;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java
new file mode 100644
index 0000000..a777f81
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.transactions.Transaction;
+
+/**
+ * IgniteOutOfMemoryError failure handler test.
+ */
+public class IoomFailureHandlerTest extends AbstractFailureHandlerTest {
+ /** Offheap size for memory policy. */
+ private static final int SIZE = 10 * 1024 * 1024;
+
+ /** Page size. */
+ static final int PAGE_SIZE = 2048;
+
+ /** Number of entries. */
+ static final int ENTRIES = 10_000;
+
+ /** PDS enabled. */
+ private boolean pds;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+
+ DataRegionConfiguration dfltPlcCfg = new DataRegionConfiguration();
+
+ dfltPlcCfg.setName("dfltPlc");
+ dfltPlcCfg.setInitialSize(SIZE);
+ dfltPlcCfg.setMaxSize(SIZE);
+
+ if (pds)
+ dfltPlcCfg.setPersistenceEnabled(true);
+
+ dsCfg.setDefaultDataRegionConfiguration(dfltPlcCfg);
+ dsCfg.setPageSize(PAGE_SIZE);
+
+ cfg.setDataStorageConfiguration(dsCfg);
+
+ CacheConfiguration<?, ?> ccfg = new CacheConfiguration<>()
+ .setName(DEFAULT_CACHE_NAME)
+ .setCacheMode(CacheMode.PARTITIONED)
+ .setBackups(0)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ cleanPersistenceDir();
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * Test IgniteOutOfMemoryException handling with no store.
+ */
+ public void testIoomErrorNoStoreHandling() throws Exception {
+ testIoomErrorHandling(false);
+ }
+
+ /**
+ * Test IgniteOutOfMemoryException handling with PDS.
+ */
+ public void testIoomErrorPdsHandling() throws Exception {
+ testIoomErrorHandling(true);
+ }
+
+ /**
+ * Test IOOME handling.
+ */
+ public void testIoomErrorHandling(boolean pds) throws Exception {
+ this.pds = pds;
+
+ IgniteEx ignite0 = startGrid(0);
+ IgniteEx ignite1 = startGrid(1);
+
+ try {
+ if (pds)
+ ignite0.cluster().active(true);
+
+ IgniteCache<Integer, Object> cache0 = ignite0.getOrCreateCache(DEFAULT_CACHE_NAME);
+ IgniteCache<Integer, Object> cache1 = ignite1.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+ awaitPartitionMapExchange();
+
+ try (Transaction tx = ignite0.transactions().txStart()) {
+ for (Integer i : primaryKeys(cache1, ENTRIES))
+ cache0.put(i, new byte[PAGE_SIZE / 3 * 2]);
+
+ tx.commit();
+ }
+ catch (Throwable ignore) {
+ // Expected.
+ }
+
+ assertFalse(dummyFailureHandler(ignite0).failure());
+ assertTrue(dummyFailureHandler(ignite1).failure());
+ assertTrue(X.hasCause(dummyFailureHandler(ignite1).failureContext().error(), IgniteOutOfMemoryException.class));
+ }
+ finally {
+ stopGrid(1);
+ stopGrid(0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
index 31af118..3697c4c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
@@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.NoOpFailureHandler;
import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
@@ -41,6 +42,7 @@ import org.apache.ignite.internal.processors.cache.persistence.CheckpointWritePr
import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor;
import org.apache.ignite.internal.util.lang.GridInClosure3X;
import org.apache.ignite.plugin.PluginProvider;
@@ -268,10 +270,17 @@ public class PageMemoryImplTest extends GridCommonAbstractTest {
IgniteConfiguration igniteCfg = new IgniteConfiguration();
igniteCfg.setDataStorageConfiguration(new DataStorageConfiguration());
+ igniteCfg.setFailureHandler(new NoOpFailureHandler());
GridTestKernalContext kernalCtx = new GridTestKernalContext(new GridTestLog4jLogger(), igniteCfg);
kernalCtx.add(new IgnitePluginProcessor(kernalCtx, igniteCfg, Collections.<PluginProvider>emptyList()));
+ FailureProcessor failureProc = new FailureProcessor(kernalCtx);
+
+ failureProc.start();
+
+ kernalCtx.add(failureProc);
+
GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
kernalCtx,
null,
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index dd9cdfd..c4b7d92 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -21,6 +21,7 @@ import java.util.Set;
import junit.framework.TestSuite;
import org.apache.ignite.GridSuppressedExceptionSelfTest;
import org.apache.ignite.failure.FailureHandlerTriggeredTest;
+import org.apache.ignite.failure.IoomFailureHandlerTest;
import org.apache.ignite.failure.StopNodeFailureHandlerTest;
import org.apache.ignite.failure.StopNodeOrHaltFailureHandlerTest;
import org.apache.ignite.internal.ClassSetTest;
@@ -197,6 +198,7 @@ public class IgniteBasicTestSuite extends TestSuite {
suite.addTestSuite(FailureHandlerTriggeredTest.class);
suite.addTestSuite(StopNodeFailureHandlerTest.class);
suite.addTestSuite(StopNodeOrHaltFailureHandlerTest.class);
+ suite.addTestSuite(IoomFailureHandlerTest.class);
return suite;
}
[08/54] [abbrv] ignite git commit: IGNITE-7222 Added ZooKeeper
discovery SPI
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectFailoverAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectFailoverAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectFailoverAbstractTest.java
index 3e98051..37292ff 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectFailoverAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectFailoverAbstractTest.java
@@ -29,6 +29,7 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteClientDisconnectedException;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -87,9 +88,9 @@ public abstract class IgniteClientReconnectFailoverAbstractTest extends IgniteCl
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
- TestTcpDiscoverySpi srvSpi = spi(srv);
+ IgniteDiscoverySpi srvSpi = spi0(srv);
final AtomicBoolean stop = new AtomicBoolean(false);
@@ -209,14 +210,17 @@ public abstract class IgniteClientReconnectFailoverAbstractTest extends IgniteCl
}
if (err != null) {
- log.error(err);
+ log.error("Test error: " + err);
U.dumpThreads(log);
CyclicBarrier barrier0 = barrier;
- if (barrier0 != null)
+ if (barrier0 != null) {
+ barrier = null;
+
barrier0.reset();
+ }
stop.set(true);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectServicesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectServicesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectServicesTest.java
index 3e961e5..1e6dd64 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectServicesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectServicesTest.java
@@ -65,7 +65,7 @@ public class IgniteClientReconnectServicesTest extends IgniteClientReconnectAbst
assertEquals((Object)topVer, srvc.test());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
reconnectClientNode(client, srv, null);
@@ -88,7 +88,7 @@ public class IgniteClientReconnectServicesTest extends IgniteClientReconnectAbst
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
IgniteServices clnServices = client.services();
@@ -132,7 +132,7 @@ public class IgniteClientReconnectServicesTest extends IgniteClientReconnectAbst
final IgniteServices services = client.services();
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
BlockTcpCommunicationSpi commSpi = commSpi(srv);
@@ -179,7 +179,7 @@ public class IgniteClientReconnectServicesTest extends IgniteClientReconnectAbst
final IgniteServices services = client.services();
- final Ignite srv = clientRouter(client);
+ final Ignite srv = ignite(0);
services.deployClusterSingleton("testReconnectInProgress", new TestServiceImpl());
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStopTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStopTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStopTest.java
index e863cdf..b5c3ee8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStopTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStopTest.java
@@ -23,8 +23,10 @@ import org.apache.ignite.IgniteClientDisconnectedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED;
@@ -50,15 +52,19 @@ public class IgniteClientReconnectStopTest extends IgniteClientReconnectAbstract
Ignite srv = clientRouter(client);
- TestTcpDiscoverySpi srvSpi = spi(srv);
+ DiscoverySpi srvSpi = spi0(srv);
final CountDownLatch disconnectLatch = new CountDownLatch(1);
final CountDownLatch reconnectLatch = new CountDownLatch(1);
- final TestTcpDiscoverySpi clientSpi = spi(client);
+ final IgniteDiscoverySpi clientSpi = spi0(client);
+
+ DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+ clientSpi.setInternalListener(lsnr);
log.info("Block reconnect.");
- clientSpi.writeLatch = new CountDownLatch(1);
+ lsnr.startBlockJoin();
client.events().localListen(new IgnitePredicate<Event>() {
@Override public boolean apply(Event evt) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStreamerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStreamerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStreamerTest.java
index 3959feb..36b9890 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStreamerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStreamerTest.java
@@ -71,7 +71,7 @@ public class IgniteClientReconnectStreamerTest extends IgniteClientReconnectAbst
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final IgniteCache<Object, Object> srvCache = srv.cache(CACHE_NAME);
@@ -135,7 +135,7 @@ public class IgniteClientReconnectStreamerTest extends IgniteClientReconnectAbst
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final IgniteCache<Object, Object> srvCache = srv.cache(CACHE_NAME);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
index a5d42e9..8edbb52 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
@@ -256,6 +256,9 @@ public class IgniteClientRejoinTest extends GridCommonAbstractTest {
Ignite srv1 = startGrid("server1");
+ if (!tcpDiscovery())
+ return;
+
crd = ((IgniteKernal)srv1).localNode();
Ignite srv2 = startGrid("server2");
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
index a8afa8b..8fad640 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
@@ -178,16 +178,20 @@ public class GridDiscoveryManagerAliveCacheSelfTest extends GridCommonAbstractTe
* Waits while topology on all nodes became equals to the expected size.
*
* @param nodesCnt Expected nodes count.
- * @throws InterruptedException If interrupted.
+ * @throws Exception If interrupted.
*/
@SuppressWarnings("BusyWait")
- private void awaitDiscovery(long nodesCnt) throws InterruptedException {
- for (Ignite g : alive) {
- ((TcpDiscoverySpi)g.configuration().getDiscoverySpi()).waitForClientMessagePrecessed();
+ private void awaitDiscovery(int nodesCnt) throws Exception {
+ if (tcpDiscovery()) {
+ for (Ignite g : alive) {
+ ((TcpDiscoverySpi)g.configuration().getDiscoverySpi()).waitForClientMessagePrecessed();
- while (g.cluster().nodes().size() != nodesCnt)
- Thread.sleep(10);
+ while (g.cluster().nodes().size() != nodesCnt)
+ Thread.sleep(10);
+ }
}
+ else
+ waitForTopology(nodesCnt);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessorAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessorAbstractSelfTest.java
index 1d70246..aa2abae 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessorAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessorAbstractSelfTest.java
@@ -91,12 +91,12 @@ public abstract class GridAffinityProcessorAbstractSelfTest extends GridCommonAb
@Override protected void beforeTestsStarted() throws Exception {
assert NODES_CNT >= 1;
- withCache = false;
+ withCache = true;
for (int i = 0; i < NODES_CNT; i++)
startGrid(i);
- withCache = true;
+ withCache = false;
for (int i = NODES_CNT; i < 2 * NODES_CNT; i++)
startGrid(i);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java
index aefbc23..b8f9d70 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java
@@ -27,9 +27,9 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.Event;
import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
import org.apache.ignite.lang.IgniteClosure;
import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
@@ -103,7 +103,7 @@ public class CacheMetricsForClusterGroupSelfTest extends GridCommonAbstractTest
Collection<ClusterNode> nodes = grid(0).cluster().forRemotes().nodes();
for (ClusterNode node : nodes) {
- Map<Integer, CacheMetrics> metrics = ((TcpDiscoveryNode)node).cacheMetrics();
+ Map<Integer, CacheMetrics> metrics = ((IgniteClusterNode)node).cacheMetrics();
assertNotNull(metrics);
assertFalse(metrics.isEmpty());
}
@@ -118,6 +118,8 @@ public class CacheMetricsForClusterGroupSelfTest extends GridCommonAbstractTest
/**
* Test cluster group metrics in case of statistics disabled.
+ *
+ * @throws Exception If failed.
*/
public void testMetricsStatisticsDisabled() throws Exception {
createCaches(false);
@@ -134,7 +136,7 @@ public class CacheMetricsForClusterGroupSelfTest extends GridCommonAbstractTest
Collection<ClusterNode> nodes = grid(0).cluster().forRemotes().nodes();
for (ClusterNode node : nodes) {
- Map<Integer, CacheMetrics> metrics = ((TcpDiscoveryNode) node).cacheMetrics();
+ Map<Integer, CacheMetrics> metrics = ((IgniteClusterNode)node).cacheMetrics();
assertNotNull(metrics);
assertTrue(metrics.isEmpty());
}
@@ -172,7 +174,9 @@ public class CacheMetricsForClusterGroupSelfTest extends GridCommonAbstractTest
}
/**
- * Wait for {@link EventType#EVT_NODE_METRICS_UPDATED} event will be receieved.
+ * Wait for {@link EventType#EVT_NODE_METRICS_UPDATED} event will be received.
+ *
+ * @throws InterruptedException If interrupted.
*/
private void awaitMetricsUpdate() throws InterruptedException {
final CountDownLatch latch = new CountDownLatch((GRID_CNT + 1) * 2);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
index df93ae4..299dbf4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
@@ -115,6 +115,8 @@ public abstract class GridCacheAbstractSelfTest extends GridCommonAbstractTest {
protected void initStoreStrategy() throws IgniteCheckedException {
if (storeStgy == null)
storeStgy = isMultiJvm() ? new H2CacheStoreStrategy() : new MapCacheStoreStrategy();
+ else if (isMultiJvm() && !(storeStgy instanceof H2CacheStoreStrategy))
+ storeStgy = new H2CacheStoreStrategy();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java
index 0069110..c135f2d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java
@@ -45,7 +45,9 @@ public class IgniteCacheNearLockValueSelfTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
- startGridsMultiThreaded(2);
+ startGrid(1);
+
+ startGrid(0);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingErrorTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingErrorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingErrorTest.java
index f321453..55ff31a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingErrorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingErrorTest.java
@@ -67,6 +67,17 @@ public class IgniteCacheP2pUnmarshallingErrorTest extends IgniteCacheAbstractTes
}
/** {@inheritDoc} */
+ @Override protected void startGrids() throws Exception {
+ int cnt = gridCount();
+
+ assert cnt >= 1 : "At least one grid must be started";
+
+ startGridsMultiThreaded(1, cnt - 1);
+
+ startGrid(0);
+ }
+
+ /** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 2337329..838e56d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -38,6 +38,7 @@ import org.apache.ignite.internal.IgniteClientReconnectAbstractTest;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
@@ -1109,6 +1110,70 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
}
/**
+ * @throws Exception If failed.
+ */
+ public void testActivateFailover3() throws Exception {
+ stateChangeFailover3(true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeactivateFailover3() throws Exception {
+ stateChangeFailover3(false);
+ }
+
+ /**
+ * @param activate If {@code true} tests activation, otherwise deactivation.
+ * @throws Exception If failed.
+ */
+ private void stateChangeFailover3(boolean activate) throws Exception {
+ testReconnectSpi = true;
+
+ startNodesAndBlockStatusChange(4, 0, 0, !activate);
+
+ client = false;
+
+ IgniteInternalFuture startFut1 = GridTestUtils.runAsync(new Callable() {
+ @Override public Object call() throws Exception {
+ startGrid(4);
+
+ return null;
+ }
+ }, "start-node1");
+
+ IgniteInternalFuture startFut2 = GridTestUtils.runAsync(new Callable() {
+ @Override public Object call() throws Exception {
+ startGrid(5);
+
+ return null;
+ }
+ }, "start-node2");
+
+ U.sleep(1000);
+
+ // Stop all nodes participating in state change and not allow last node to finish exchange.
+ for (int i = 0; i < 4; i++)
+ ((IgniteDiscoverySpi)ignite(i).configuration().getDiscoverySpi()).simulateNodeFailure();
+
+ for (int i = 0; i < 4; i++)
+ stopGrid(getTestIgniteInstanceName(i), true, false);
+
+ startFut1.get();
+ startFut2.get();
+
+ assertFalse(ignite(4).active());
+ assertFalse(ignite(5).active());
+
+ ignite(4).active(true);
+
+ for (int i = 0; i < 4; i++)
+ startGrid(i);
+
+ checkCaches1(6);
+ }
+
+ /**
* @param exp If {@code true} there should be recorded messages.
*/
private void checkRecordedMessages(boolean exp) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java
index 566860d..2f9bd53 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java
@@ -79,7 +79,7 @@ public class IgniteDaemonNodeMarshallerCacheTest extends GridCommonAbstractTest
* @param startFirst If {@code true} daemon node is started first.
* @throws Exception If failed.
*/
- public void marshalOnDaemonNode(boolean startFirst) throws Exception {
+ private void marshalOnDaemonNode(boolean startFirst) throws Exception {
int nodeIdx = 0;
if (!startFirst) {
@@ -92,6 +92,7 @@ public class IgniteDaemonNodeMarshallerCacheTest extends GridCommonAbstractTest
Ignite daemonNode = startGrid(nodeIdx++);
+ assertTrue(daemonNode.cluster().localNode().isDaemon());
assertEquals("true", daemonNode.cluster().localNode().attribute(ATTR_DAEMON));
daemon = false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataUpdatesFlowTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataUpdatesFlowTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataUpdatesFlowTest.java
index 3ee51c8..7e8c086 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataUpdatesFlowTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataUpdatesFlowTest.java
@@ -184,6 +184,7 @@ public class BinaryMetadataUpdatesFlowTest extends GridCommonAbstractTest {
* Starts new ignite node and submits computation job to it.
* @param idx Index.
* @param stopFlag Stop flag.
+ * @throws Exception If failed.
*/
private void startComputation(int idx, AtomicBoolean stopFlag) throws Exception {
clientMode = false;
@@ -199,6 +200,7 @@ public class BinaryMetadataUpdatesFlowTest extends GridCommonAbstractTest {
* @param idx Index.
* @param deafClient Deaf client.
* @param observedIds Observed ids.
+ * @throws Exception If failed.
*/
private void startListening(int idx, boolean deafClient, Set<Integer> observedIds) throws Exception {
clientMode = true;
@@ -269,7 +271,7 @@ public class BinaryMetadataUpdatesFlowTest extends GridCommonAbstractTest {
}
/**
- *
+ * @throws Exception If failed.
*/
public void testFlowNoConflicts() throws Exception {
startComputation(0, stopFlag0);
@@ -311,11 +313,14 @@ public class BinaryMetadataUpdatesFlowTest extends GridCommonAbstractTest {
}
/**
- *
+ * @throws Exception If failed.
*/
public void testFlowNoConflictsWithClients() throws Exception {
startComputation(0, stopFlag0);
+ if (!tcpDiscovery())
+ return;
+
startComputation(1, stopFlag1);
startComputation(2, stopFlag2);
@@ -617,6 +622,9 @@ public class BinaryMetadataUpdatesFlowTest extends GridCommonAbstractTest {
while (!updatesQueue.isEmpty()) {
BinaryUpdateDescription desc = updatesQueue.poll();
+ if (desc == null)
+ break;
+
BinaryObjectBuilder builder = ignite.binary().builder(BINARY_TYPE_NAME);
BinaryObject bo = newBinaryObject(builder, desc);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheClientNodeBinaryObjectMetadataMultinodeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheClientNodeBinaryObjectMetadataMultinodeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheClientNodeBinaryObjectMetadataMultinodeTest.java
index 313aaf9..81614cb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheClientNodeBinaryObjectMetadataMultinodeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheClientNodeBinaryObjectMetadataMultinodeTest.java
@@ -242,7 +242,7 @@ public class GridCacheClientNodeBinaryObjectMetadataMultinodeTest extends GridCo
@Override public boolean apply() {
Collection<BinaryType> metaCol = p0.types();
- return metaCol.size() == 1000;
+ return metaCol.size() >= 1000;
}
}, getTestTimeout());
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java
index ed54377..dac3ff6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java
@@ -32,6 +32,9 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+/**
+ *
+ */
public class GridCacheQueueClientDisconnectTest extends GridCommonAbstractTest {
/** */
private static final String IGNITE_QUEUE_NAME = "ignite-queue-client-reconnect-test";
@@ -66,6 +69,10 @@ public class GridCacheQueueClientDisconnectTest extends GridCommonAbstractTest {
return cfg;
}
+ /**
+ * @param cacheAtomicityMode Atomicity mode.
+ * @return Configuration.
+ */
private static CollectionConfiguration collectionConfiguration(CacheAtomicityMode cacheAtomicityMode) {
CollectionConfiguration colCfg = new CollectionConfiguration();
@@ -74,6 +81,9 @@ public class GridCacheQueueClientDisconnectTest extends GridCommonAbstractTest {
return colCfg;
}
+ /**
+ * @throws Exception If failed.
+ */
public void testClientDisconnect() throws Exception {
try {
Ignite server = startGrid(0);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteClientDataStructuresAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteClientDataStructuresAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteClientDataStructuresAbstractTest.java
index 51764b5..d85201a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteClientDataStructuresAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteClientDataStructuresAbstractTest.java
@@ -494,7 +494,8 @@ public abstract class IgniteClientDataStructuresAbstractTest extends GridCommonA
assertTrue(ignite.configuration().isClientMode());
- assertEquals(clientDiscovery(), ignite.configuration().getDiscoverySpi().isClientMode());
+ if (tcpDiscovery())
+ assertEquals(clientDiscovery(), ignite.configuration().getDiscoverySpi().isClientMode());
return ignite;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 0704dbd..e456047 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -52,6 +52,7 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.internal.DiscoverySpiTestListener;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.cluster.NodeOrderComparator;
import org.apache.ignite.internal.cluster.NodeOrderLegacyComparator;
@@ -60,7 +61,7 @@ import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.IgniteNodeAttributes;
import org.apache.ignite.internal.TestRecordingCommunicationSpi;
import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
-import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.affinity.GridAffinityFunctionContextImpl;
import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
@@ -88,7 +89,6 @@ import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.services.Service;
import org.apache.ignite.services.ServiceContext;
-import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -158,7 +158,7 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
cfg.setCommunicationSpi(commSpi);
- TestTcpDiscoverySpi discoSpi = new TestTcpDiscoverySpi();
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
discoSpi.setForceServerMode(forceSrvMode);
discoSpi.setIpFinder(ipFinder);
@@ -674,9 +674,11 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
checkAffinity(4, topVer(4, 0), true);
- TestTcpDiscoverySpi discoSpi = (TestTcpDiscoverySpi)ignite0.configuration().getDiscoverySpi();
+ DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
- discoSpi.blockCustomEvent();
+ ((IgniteDiscoverySpi)ignite0.configuration().getDiscoverySpi()).setInternalListener(lsnr);
+
+ lsnr.blockCustomEvent(CacheAffinityChangeMessage.class);
stopGrid(1);
@@ -687,7 +689,7 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
for (IgniteInternalFuture<?> fut : futs)
assertFalse(fut.isDone());
- discoSpi.stopBlock();
+ lsnr.stopBlockCustomEvents();
checkAffinity(3, topVer(5, 0), false);
@@ -1409,8 +1411,10 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
public void testDelayAssignmentAffinityChanged() throws Exception {
Ignite ignite0 = startServer(0, 1);
- TestTcpDiscoverySpi discoSpi0 =
- (TestTcpDiscoverySpi)ignite0.configuration().getDiscoverySpi();
+ DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+ ((IgniteDiscoverySpi)ignite0.configuration().getDiscoverySpi()).setInternalListener(lsnr);
+
TestRecordingCommunicationSpi commSpi0 =
(TestRecordingCommunicationSpi)ignite0.configuration().getCommunicationSpi();
@@ -1418,19 +1422,19 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
checkAffinity(2, topVer(2, 0), true);
- discoSpi0.blockCustomEvent();
+ lsnr.blockCustomEvent(CacheAffinityChangeMessage.class);
startServer(2, 3);
checkAffinity(3, topVer(3, 0), false);
- discoSpi0.waitCustomEvent();
+ lsnr.waitCustomEvent();
blockSupplySend(commSpi0, CACHE_NAME1);
startServer(3, 4);
- discoSpi0.stopBlock();
+ lsnr.stopBlockCustomEvents();
checkAffinity(4, topVer(4, 0), false);
@@ -1452,8 +1456,10 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
try {
Ignite ignite0 = startServer(0, 1);
- TestTcpDiscoverySpi discoSpi0 =
- (TestTcpDiscoverySpi)ignite0.configuration().getDiscoverySpi();
+ DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+ ((IgniteDiscoverySpi)ignite0.configuration().getDiscoverySpi()).setInternalListener(lsnr);
+
TestRecordingCommunicationSpi commSpi0 =
(TestRecordingCommunicationSpi)ignite0.configuration().getCommunicationSpi();
@@ -1465,11 +1471,11 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
checkAffinity(3, topVer(3, 1), false);
- discoSpi0.blockCustomEvent();
+ lsnr.blockCustomEvent(CacheAffinityChangeMessage.class);
stopNode(2, 4);
- discoSpi0.waitCustomEvent();
+ lsnr.waitCustomEvent();
blockSupplySend(commSpi0, CACHE_NAME1);
@@ -1483,7 +1489,7 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
Thread.sleep(2_000);
- discoSpi0.stopBlock();
+ lsnr.stopBlockCustomEvents();
boolean started = GridTestUtils.waitForCondition(new GridAbsPredicate() {
@Override public boolean apply() {
@@ -1534,14 +1540,16 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
ignite0.createCache(ccfg);
- TestTcpDiscoverySpi discoSpi0 =
- (TestTcpDiscoverySpi)ignite0.configuration().getDiscoverySpi();
+ DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+ ((IgniteDiscoverySpi)ignite0.configuration().getDiscoverySpi()).setInternalListener(lsnr);
+
TestRecordingCommunicationSpi spi =
(TestRecordingCommunicationSpi)ignite0.configuration().getCommunicationSpi();
blockSupplySend(spi, CACHE_NAME2);
- discoSpi0.blockCustomEvent();
+ lsnr.blockCustomEvent(CacheAffinityChangeMessage.class);
startServer(1, 2);
@@ -1551,7 +1559,7 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
spi.stopBlock();
- discoSpi0.waitCustomEvent();
+ lsnr.waitCustomEvent();
ignite0.destroyCache(CACHE_NAME2);
@@ -1561,7 +1569,7 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
ignite0.createCache(ccfg);
- discoSpi0.stopBlock();
+ lsnr.stopBlockCustomEvents();
checkAffinity(3, topVer(3, 1), false);
checkAffinity(3, topVer(3, 2), false);
@@ -2967,83 +2975,6 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
/**
*
*/
- static class TestTcpDiscoverySpi extends TcpDiscoverySpi {
- /** */
- private boolean blockCustomEvt;
-
- /** */
- private final Object mux = new Object();
-
- /** */
- private List<DiscoverySpiCustomMessage> blockedMsgs = new ArrayList<>();
-
- /** {@inheritDoc} */
- @Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) throws IgniteException {
- synchronized (mux) {
- if (blockCustomEvt) {
- DiscoveryCustomMessage msg0 = GridTestUtils.getFieldValue(msg, "delegate");
-
- if (msg0 instanceof CacheAffinityChangeMessage) {
- log.info("Block custom message: " + msg0);
-
- blockedMsgs.add(msg);
-
- mux.notifyAll();
-
- return;
- }
- }
- }
-
- super.sendCustomEvent(msg);
- }
-
- /**
- *
- */
- public void blockCustomEvent() {
- synchronized (mux) {
- assert blockedMsgs.isEmpty() : blockedMsgs;
-
- blockCustomEvt = true;
- }
- }
-
- /**
- * @throws InterruptedException If interrupted.
- */
- public void waitCustomEvent() throws InterruptedException {
- synchronized (mux) {
- while (blockedMsgs.isEmpty())
- mux.wait();
- }
- }
-
- /**
- *
- */
- public void stopBlock() {
- List<DiscoverySpiCustomMessage> msgs;
-
- synchronized (this) {
- msgs = new ArrayList<>(blockedMsgs);
-
- blockCustomEvt = false;
-
- blockedMsgs.clear();
- }
-
- for (DiscoverySpiCustomMessage msg : msgs) {
- log.info("Resend blocked message: " + msg);
-
- super.sendCustomEvent(msg);
- }
- }
- }
-
- /**
- *
- */
static class TestEntryProcessor implements EntryProcessor<Object, Object, Object> {
/** */
private Object val;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java
index 3834df9..5dea5d9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java
@@ -45,6 +45,7 @@ import org.apache.ignite.transactions.TransactionIsolation;
import static org.apache.ignite.IgniteState.STOPPED;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_TX_SALVAGE_TIMEOUT;
import static org.apache.ignite.IgniteSystemProperties.getInteger;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
@@ -188,7 +189,7 @@ public abstract class GridCacheNodeFailureAbstractTest extends GridCommonAbstrac
return true;
}
- }, EVT_NODE_LEFT);
+ }, EVT_NODE_LEFT, EVT_NODE_FAILED);
stopGrid(idx);
@@ -268,7 +269,7 @@ public abstract class GridCacheNodeFailureAbstractTest extends GridCommonAbstrac
return true;
}
- }, EVT_NODE_LEFT);
+ }, EVT_NODE_LEFT, EVT_NODE_FAILED);
stopGrid(idx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java
index e71d3ee..b7ae844 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java
@@ -168,6 +168,8 @@ public class IgniteCache150ClientsTest extends GridCommonAbstractTest {
log.info("Started all clients.");
+ waitForTopology(CLIENTS + 1);
+
checkNodes(CLIENTS + 1);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheManyClientsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheManyClientsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheManyClientsTest.java
index a0be40e..7785a3c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheManyClientsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheManyClientsTest.java
@@ -178,7 +178,7 @@ public class IgniteCacheManyClientsTest extends GridCommonAbstractTest {
log.info("All clients started.");
try {
- checkNodes(SRVS + CLIENTS);
+ checkNodes0(SRVS + CLIENTS);
}
finally {
for (Ignite client : clients)
@@ -188,6 +188,30 @@ public class IgniteCacheManyClientsTest extends GridCommonAbstractTest {
/**
* @param expCnt Expected number of nodes.
+ * @throws Exception If failed.
+ */
+ private void checkNodes0(final int expCnt) throws Exception {
+ boolean wait = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ try {
+ checkNodes(expCnt);
+
+ return true;
+ }
+ catch (AssertionFailedError e) {
+ log.info("Check failed, will retry: " + e);
+ }
+
+ return false;
+ }
+ }, 10_000);
+
+ if (!wait)
+ checkNodes(expCnt);
+ }
+
+ /**
+ * @param expCnt Expected number of nodes.
*/
private void checkNodes(int expCnt) {
assertEquals(expCnt, G.allGrids().size());
@@ -297,23 +321,7 @@ public class IgniteCacheManyClientsTest extends GridCommonAbstractTest {
if (err0 != null)
throw err0;
- boolean wait = GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- try {
- checkNodes(SRVS + THREADS);
-
- return true;
- }
- catch (AssertionFailedError e) {
- log.info("Check failed, will retry: " + e);
- }
-
- return false;
- }
- }, 10_000);
-
- if (!wait)
- checkNodes(SRVS + THREADS);
+ checkNodes0(SRVS + THREADS);
log.info("Stop clients.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
index a55f21d..486fd60 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
@@ -713,6 +713,8 @@ public class IgniteOptimisticTxSuspendResumeTest extends GridCommonAbstractTest
", backups=" + ccfg.getBackups() +
", near=" + (ccfg.getNearConfiguration() != null) + "]");
+ awaitPartitionMapExchange();
+
int srvNum = serversNumber();
if (serversNumber() > 1) {
ignite(serversNumber() + 1).createNearCache(ccfg.getName(), new NearCacheConfiguration<>());
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java
index e3fa116..60f2f0a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java
@@ -116,6 +116,8 @@ public class GridCacheDhtPreloadMultiThreadedSelfTest extends GridCommonAbstract
@Nullable @Override public Object call() throws Exception {
IgniteConfiguration cfg = loadConfiguration("modules/core/src/test/config/spring-multicache.xml");
+ cfg.setGridLogger(getTestResources().getLogger());
+
startGrid(Thread.currentThread().getName(), cfg);
return null;
@@ -161,6 +163,8 @@ public class GridCacheDhtPreloadMultiThreadedSelfTest extends GridCommonAbstract
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = loadConfiguration("modules/core/src/test/config/spring-multicache.xml");
+ cfg.setGridLogger(getTestResources().getLogger());
+
cfg.setIgniteInstanceName(igniteInstanceName);
for (CacheConfiguration cCfg : cfg.getCacheConfiguration()) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
index 05a9759..83eff89 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
@@ -308,6 +308,8 @@ public class GridCacheDhtPreloadSelfTest extends GridCommonAbstractTest {
// Check all left nodes.
checkActiveState(ignites);
+
+ awaitPartitionMapExchange(); // Need wait, otherwise test logic is broken if EVT_NODE_FAILED exchanges are merged.
}
info("Waiting for preload futures: " + F.view(futs, new IgnitePredicate<IgniteFuture<?>>() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
index 7b350c8..060af21 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
@@ -34,11 +34,11 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.spi.IgniteSpiException;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
import org.apache.ignite.transactions.TransactionConcurrency;
@@ -65,7 +65,6 @@ public class TxRecoveryStoreEnabledTest extends GridCommonAbstractTest {
IgniteConfiguration cfg = super.getConfiguration(gridName);
cfg.setCommunicationSpi(new TestCommunicationSpi());
- cfg.setDiscoverySpi(new TestDiscoverySpi());
CacheConfiguration ccfg = defaultCacheConfiguration();
@@ -126,7 +125,7 @@ public class TxRecoveryStoreEnabledTest extends GridCommonAbstractTest {
IgniteConfiguration cfg = node0.configuration();
((TestCommunicationSpi)cfg.getCommunicationSpi()).block();
- ((TestDiscoverySpi)cfg.getDiscoverySpi()).simulateNodeFailure();
+ ((IgniteDiscoverySpi)cfg.getDiscoverySpi()).simulateNodeFailure();
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
@@ -201,16 +200,6 @@ public class TxRecoveryStoreEnabledTest extends GridCommonAbstractTest {
/**
*
*/
- private static class TestDiscoverySpi extends TcpDiscoverySpi {
- /** {@inheritDoc} */
- @Override protected void simulateNodeFailure() {
- super.simulateNodeFailure();
- }
- }
-
- /**
- *
- */
private static class TestCommunicationSpi extends TcpCommunicationSpi {
/** Block. */
private volatile boolean block;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java
index 3c57957..96fb8f6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java
@@ -37,6 +37,7 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
/**
@@ -125,7 +126,7 @@ public class GridCachePartitionedExplicitLockNodeFailureSelfTest extends GridCom
return true;
}
- }, EVT_NODE_LEFT));
+ }, EVT_NODE_LEFT, EVT_NODE_FAILED));
}
stopGrid(idx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ClientReconnectContinuousQueryTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ClientReconnectContinuousQueryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ClientReconnectContinuousQueryTest.java
index c8b3bb6..9b531c6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ClientReconnectContinuousQueryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ClientReconnectContinuousQueryTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache.query.continuous;
+import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import javax.cache.event.CacheEntryListenerException;
@@ -28,6 +29,7 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.Event;
import org.apache.ignite.events.EventType;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.managers.communication.GridIoManager;
import org.apache.ignite.internal.util.nio.GridNioServer;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -90,7 +92,7 @@ public class ClientReconnectContinuousQueryTest extends GridCommonAbstractTest {
try {
startGrids(2);
- IgniteEx client = grid(CLIENT_IDX);
+ final IgniteEx client = grid(CLIENT_IDX);
client.events().localListen(new DisconnectListener(), EventType.EVT_CLIENT_NODE_DISCONNECTED);
@@ -112,11 +114,19 @@ public class ClientReconnectContinuousQueryTest extends GridCommonAbstractTest {
skipRead(client, true);
- putSomeKeys(1_000);
+ IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ assertTrue(disconLatch.await(10_000, TimeUnit.MILLISECONDS));
+
+ skipRead(client, false);
- assertTrue(disconLatch.await(10_000, TimeUnit.MILLISECONDS));
+ return null;
+ }
+ });
- skipRead(client, false);
+ putSomeKeys(1_000);
+
+ fut.get();
assertTrue(reconLatch.await(10_000, TimeUnit.MILLISECONDS));
@@ -129,7 +139,6 @@ public class ClientReconnectContinuousQueryTest extends GridCommonAbstractTest {
finally {
stopAllGrids();
}
-
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryRemoteFilterMissingInClassPathSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryRemoteFilterMissingInClassPathSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryRemoteFilterMissingInClassPathSelfTest.java
index 92c1760..226302f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryRemoteFilterMissingInClassPathSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryRemoteFilterMissingInClassPathSelfTest.java
@@ -104,7 +104,7 @@ public class ContinuousQueryRemoteFilterMissingInClassPathSelfTest extends GridC
setExternalLoader = true;
final Ignite ignite0 = startGrid(1);
- executeContiniouseQuery(ignite0.cache("simple"));
+ executeContinuousQuery(ignite0.cache("simple"));
log = new GridStringLogger();
clientMode = true;
@@ -112,8 +112,10 @@ public class ContinuousQueryRemoteFilterMissingInClassPathSelfTest extends GridC
startGrid(2);
- assertTrue(log.toString().contains("Failed to unmarshal continuous query remote filter on client node. " +
- "Can be ignored."));
+ String logStr = log.toString();
+
+ assertTrue(logStr.contains("Failed to unmarshal continuous query remote filter on client node. " +
+ "Can be ignored.") || logStr.contains("Failed to unmarshal continuous routine handler"));
}
/**
@@ -127,7 +129,7 @@ public class ContinuousQueryRemoteFilterMissingInClassPathSelfTest extends GridC
clientMode = false;
final Ignite ignite0 = startGrid(1);
- executeContiniouseQuery(ignite0.cache("simple"));
+ executeContinuousQuery(ignite0.cache("simple"));
log = new GridStringLogger();
clientMode = true;
@@ -149,15 +151,18 @@ public class ContinuousQueryRemoteFilterMissingInClassPathSelfTest extends GridC
setExternalLoader = true;
final Ignite ignite0 = startGrid(1);
- executeContiniouseQuery(ignite0.cache("simple"));
+ executeContinuousQuery(ignite0.cache("simple"));
log = new GridStringLogger();
setExternalLoader = false;
startGrid(2);
- assertTrue(log.toString().contains("class org.apache.ignite.IgniteCheckedException: " +
- "Failed to find class with given class loader for unmarshalling"));
+ String logStr = log.toString();
+
+ assertTrue(logStr.contains("class org.apache.ignite.IgniteCheckedException: " +
+ "Failed to find class with given class loader for unmarshalling")
+ || logStr.contains("Failed to unmarshal continuous routine handler"));
}
/**
@@ -171,7 +176,7 @@ public class ContinuousQueryRemoteFilterMissingInClassPathSelfTest extends GridC
setExternalLoader = true;
final Ignite ignite0 = startGrid(1);
- executeContiniouseQuery(ignite0.cache("simple"));
+ executeContinuousQuery(ignite0.cache("simple"));
log = new GridStringLogger();
@@ -185,7 +190,7 @@ public class ContinuousQueryRemoteFilterMissingInClassPathSelfTest extends GridC
* @param cache Ignite cache.
* @throws Exception If fail.
*/
- private void executeContiniouseQuery(IgniteCache cache) throws Exception {
+ private void executeContinuousQuery(IgniteCache cache) throws Exception {
ContinuousQuery<Integer, String> qry = new ContinuousQuery<>();
qry.setLocalListener(
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java
index 9ad6d4e..906cc7d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java
@@ -119,6 +119,9 @@ public class IgniteCacheContinuousQueryClientReconnectTest extends IgniteClientR
* @throws Exception If failed.
*/
public void testReconnectClientAndLeftRouter() throws Exception {
+ if (!tcpDiscovery())
+ return;
+
Ignite client = grid(serverCount());
final Ignite srv = clientRouter(client);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryAbstractTest.java
index 61ceef7..16ea848 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryAbstractTest.java
@@ -18,11 +18,12 @@
package org.apache.ignite.internal.processors.cache.version;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
import javax.cache.Cache;
import javax.cache.processor.EntryProcessor;
import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
import javax.cache.processor.MutableEntry;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheEntry;
@@ -56,23 +57,15 @@ public abstract class CacheVersionedEntryAbstractTest extends GridCacheAbstractS
public void testInvoke() throws Exception {
Cache<Integer, String> cache = grid(0).cache(DEFAULT_CACHE_NAME);
- final AtomicInteger invoked = new AtomicInteger();
-
- cache.invoke(100, new EntryProcessor<Integer, String, Object>() {
- @Override public Object process(MutableEntry<Integer, String> entry, Object... arguments)
- throws EntryProcessorException {
-
- invoked.incrementAndGet();
-
+ assertNotNull(cache.invoke(100, new EntryProcessor<Integer, String, Object>() {
+ @Override public Object process(MutableEntry<Integer, String> entry, Object... args) {
CacheEntry<Integer, String> verEntry = entry.unwrap(CacheEntry.class);
checkVersionedEntry(verEntry);
- return entry;
+ return verEntry.version();
}
- });
-
- assert invoked.get() > 0;
+ }));
}
/**
@@ -86,23 +79,17 @@ public abstract class CacheVersionedEntryAbstractTest extends GridCacheAbstractS
for (int i = 0; i < ENTRIES_NUM; i++)
keys.add(i);
- final AtomicInteger invoked = new AtomicInteger();
-
- cache.invokeAll(keys, new EntryProcessor<Integer, String, Object>() {
- @Override public Object process(MutableEntry<Integer, String> entry, Object... arguments)
- throws EntryProcessorException {
-
- invoked.incrementAndGet();
-
+ Map<Integer, EntryProcessorResult<Object>> res = cache.invokeAll(keys, new EntryProcessor<Integer, String, Object>() {
+ @Override public Object process(MutableEntry<Integer, String> entry, Object... args) {
CacheEntry<Integer, String> verEntry = entry.unwrap(CacheEntry.class);
checkVersionedEntry(verEntry);
- return null;
+ return verEntry.version();
}
});
- assert invoked.get() > 0;
+ assertEquals(ENTRIES_NUM, res.size());
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/continuous/GridEventConsumeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/continuous/GridEventConsumeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/continuous/GridEventConsumeSelfTest.java
index f07b1a3..1a7abd4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/continuous/GridEventConsumeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/continuous/GridEventConsumeSelfTest.java
@@ -447,7 +447,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().broadcast(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(GRID_CNT, nodeIds.size());
assertEquals(GRID_CNT, cnt.get());
@@ -488,7 +488,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().broadcast(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(GRID_CNT, nodeIds.size());
assertEquals(GRID_CNT, cnt.get());
@@ -532,7 +532,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().broadcast(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(GRID_CNT, nodeIds.size());
assertEquals(GRID_CNT, cnt.get());
@@ -578,7 +578,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().broadcast(F.noop());
grid(0).compute().withName("exclude").run(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(GRID_CNT, nodeIds.size());
assertEquals(GRID_CNT, cnt.get());
@@ -619,7 +619,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().broadcast(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(GRID_CNT - 1, nodeIds.size());
assertEquals(GRID_CNT - 1, cnt.get());
@@ -660,7 +660,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().broadcast(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(GRID_CNT - 1, nodeIds.size());
assertEquals(GRID_CNT - 1, cnt.get());
@@ -701,7 +701,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().broadcast(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(1, nodeIds.size());
assertEquals(1, cnt.get());
@@ -744,7 +744,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().broadcast(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(1, nodeIds.size());
assertEquals(1, cnt.get());
@@ -785,7 +785,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().run(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(1, nodeIds.size());
assertEquals(1, cnt.get());
@@ -828,7 +828,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
compute(grid(0).cluster().forLocal()).run(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(1, cnt.get());
@@ -878,7 +878,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().broadcast(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(GRID_CNT + 1, nodeIds.size());
assertEquals(GRID_CNT + 1, cnt.get());
@@ -929,7 +929,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().broadcast(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(GRID_CNT, nodeIds.size());
assertEquals(GRID_CNT, cnt.get());
@@ -980,7 +980,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().broadcast(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(GRID_CNT + 1, nodeIds.size());
assertEquals(GRID_CNT + 1, cnt.get());
@@ -1036,7 +1036,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
grid(0).compute().broadcast(F.noop());
- assert latch.await(2, SECONDS);
+ assert latch.await(10, SECONDS) : latch;
assertEquals(GRID_CNT, nodeIds.size());
assertEquals(GRID_CNT, cnt.get());
@@ -1145,7 +1145,7 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
/**
* @throws Exception If failed.
*/
- public void testMultithreadedWithNodeRestart() throws Exception {
+ public void _testMultithreadedWithNodeRestart() throws Exception {
final AtomicBoolean stop = new AtomicBoolean();
final BlockingQueue<IgniteBiTuple<Integer, UUID>> queue = new LinkedBlockingQueue<>();
final Collection<UUID> started = new GridConcurrentHashSet<>();
@@ -1153,9 +1153,11 @@ public class GridEventConsumeSelfTest extends GridCommonAbstractTest {
final Random rnd = new Random();
+ final int consumeCnt = tcpDiscovery() ? CONSUME_CNT : CONSUME_CNT / 2;
+
IgniteInternalFuture<?> starterFut = multithreadedAsync(new Callable<Object>() {
@Override public Object call() throws Exception {
- for (int i = 0; i < CONSUME_CNT; i++) {
+ for (int i = 0; i < consumeCnt; i++) {
int idx = rnd.nextInt(GRID_CNT);
try {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ClosureServiceClientsNodesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ClosureServiceClientsNodesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ClosureServiceClientsNodesTest.java
index c1af323..8f03c4c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ClosureServiceClientsNodesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ClosureServiceClientsNodesTest.java
@@ -49,6 +49,9 @@ public class ClosureServiceClientsNodesTest extends GridCommonAbstractTest {
/** Number of grids started for tests. */
private static final int NODES_CNT = 4;
+ /** */
+ private static final int CLIENT_IDX = 1;
+
/** Test singleton service name. */
private static final String SINGLETON_NAME = "testSingleton";
@@ -61,11 +64,11 @@ public class ClosureServiceClientsNodesTest extends GridCommonAbstractTest {
cfg.setMarshaller(new BinaryMarshaller());
- cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(ipFinder).setForceServerMode(true));
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
cfg.setCacheConfiguration();
- if (igniteInstanceName.equals(getTestIgniteInstanceName(0)))
+ if (igniteInstanceName.equals(getTestIgniteInstanceName(CLIENT_IDX)))
cfg.setClientMode(true);
return cfg;
@@ -88,8 +91,10 @@ public class ClosureServiceClientsNodesTest extends GridCommonAbstractTest {
public void testDefaultClosure() throws Exception {
Set<String> srvNames = new HashSet<>(NODES_CNT - 1);
- for (int i = 1; i < NODES_CNT; ++i)
- srvNames.add(getTestIgniteInstanceName(i));
+ for (int i = 0; i < NODES_CNT; ++i) {
+ if (i != CLIENT_IDX)
+ srvNames.add(getTestIgniteInstanceName(i));
+ }
for (int i = 0 ; i < NODES_CNT; i++) {
log.info("Iteration: " + i);
@@ -137,7 +142,7 @@ public class ClosureServiceClientsNodesTest extends GridCommonAbstractTest {
assertEquals(1, res.size());
- assertEquals(getTestIgniteInstanceName(0), F.first(res));
+ assertEquals(getTestIgniteInstanceName(CLIENT_IDX), F.first(res));
}
}
@@ -168,7 +173,7 @@ public class ClosureServiceClientsNodesTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
public void testDefaultService() throws Exception {
- UUID clientNodeId = grid(0).cluster().localNode().id();
+ UUID clientNodeId = grid(CLIENT_IDX).cluster().localNode().id();
for (int i = 0 ; i < NODES_CNT; i++) {
log.info("Iteration: " + i);
@@ -209,7 +214,7 @@ public class ClosureServiceClientsNodesTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
public void testClientService() throws Exception {
- UUID clientNodeId = grid(0).cluster().localNode().id();
+ UUID clientNodeId = grid(CLIENT_IDX).cluster().localNode().id();
for (int i = 0 ; i < NODES_CNT; i++) {
log.info("Iteration: " + i);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java b/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java
index 5da9042..2d26b72 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java
@@ -32,6 +32,15 @@ public class GridTestClockTimer implements Runnable {
}
}
+ /**
+ * @return {@code True} if need start test time.
+ */
+ public static boolean startTestTimer() {
+ synchronized (IgniteUtils.mux) {
+ return IgniteUtils.gridCnt == 0;
+ }
+ }
+
/** {@inheritDoc} */
@Override public void run() {
while (true) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
index cd8e757..78f3c03 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
@@ -30,6 +30,7 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
/**
@@ -157,6 +158,9 @@ public class GridMarshallerMappingConsistencyTest extends GridCommonAbstractTest
Ignite g2 = startGrid(2);
startGrid(1);
+ assertTrue("Failed to wait for automatic grid activation",
+ GridTestUtils.waitForCondition(() -> g2.cluster().active(), getTestTimeout()));
+
IgniteCache<Integer, DummyObject> c2 = g2.cache(CACHE_NAME);
assertEquals(k, c2.get(k).val);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/messaging/GridMessagingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/messaging/GridMessagingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/messaging/GridMessagingSelfTest.java
index 7541cec..a7c4521 100644
--- a/modules/core/src/test/java/org/apache/ignite/messaging/GridMessagingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/messaging/GridMessagingSelfTest.java
@@ -24,7 +24,6 @@ import java.io.ObjectOutput;
import java.io.Serializable;
import java.net.URL;
import java.net.URLClassLoader;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -37,22 +36,20 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteMessaging;
import org.apache.ignite.cluster.ClusterGroup;
import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.DiscoverySpiTestListener;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.processors.continuous.StartRoutineDiscoveryMessage;
+import org.apache.ignite.internal.processors.continuous.StartRoutineDiscoveryMessageV2;
import org.apache.ignite.internal.processors.continuous.StopRoutineDiscoveryMessage;
-import org.apache.ignite.internal.processors.marshaller.MappingAcceptedMessage;
-import org.apache.ignite.internal.processors.marshaller.MappingProposedMessage;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.typedef.P2;
import org.apache.ignite.internal.util.typedef.PA;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -206,11 +203,7 @@ public class GridMessagingSelfTest extends GridCommonAbstractTest implements Ser
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
- TestTcpDiscoverySpi discoSpi = new TestTcpDiscoverySpi();
-
- discoSpi.setIpFinder(ipFinder);
-
- cfg.setDiscoverySpi(discoSpi);
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
return cfg;
}
@@ -1036,7 +1029,11 @@ public class GridMessagingSelfTest extends GridCommonAbstractTest implements Ser
public void testAsyncOld() throws Exception {
final AtomicInteger msgCnt = new AtomicInteger();
- TestTcpDiscoverySpi discoSpi = (TestTcpDiscoverySpi)ignite2.configuration().getDiscoverySpi();
+ IgniteDiscoverySpi discoSpi = (IgniteDiscoverySpi)ignite2.configuration().getDiscoverySpi();
+
+ DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+ discoSpi.setInternalListener(lsnr);
assertFalse(ignite2.message().isAsync());
@@ -1054,7 +1051,7 @@ public class GridMessagingSelfTest extends GridCommonAbstractTest implements Ser
}
}, IllegalStateException.class, null);
- discoSpi.blockCustomEvent();
+ lsnr.blockCustomEvent(StartRoutineDiscoveryMessage.class, StartRoutineDiscoveryMessageV2.class);
final String topic = "topic";
@@ -1079,7 +1076,7 @@ public class GridMessagingSelfTest extends GridCommonAbstractTest implements Ser
Assert.assertFalse(starFut.isDone());
- discoSpi.stopBlock();
+ lsnr.stopBlockCustomEvents();
GridTestUtils.assertThrows(log, new Callable<Void>() {
@Override public Void call() throws Exception {
@@ -1095,7 +1092,7 @@ public class GridMessagingSelfTest extends GridCommonAbstractTest implements Ser
Assert.assertTrue(starFut.isDone());
- discoSpi.blockCustomEvent();
+ lsnr.blockCustomEvent(StopRoutineDiscoveryMessage.class);
message(ignite1.cluster().forRemotes()).send(topic, "msg1");
@@ -1125,7 +1122,7 @@ public class GridMessagingSelfTest extends GridCommonAbstractTest implements Ser
Assert.assertFalse(stopFut.isDone());
- discoSpi.stopBlock();
+ lsnr.stopBlockCustomEvents();
stopFut.get();
@@ -1144,9 +1141,13 @@ public class GridMessagingSelfTest extends GridCommonAbstractTest implements Ser
public void testAsync() throws Exception {
final AtomicInteger msgCnt = new AtomicInteger();
- TestTcpDiscoverySpi discoSpi = (TestTcpDiscoverySpi)ignite2.configuration().getDiscoverySpi();
+ IgniteDiscoverySpi discoSpi = (IgniteDiscoverySpi)ignite2.configuration().getDiscoverySpi();
- discoSpi.blockCustomEvent();
+ DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+ discoSpi.setInternalListener(lsnr);
+
+ lsnr.blockCustomEvent(StartRoutineDiscoveryMessage.class, StartRoutineDiscoveryMessageV2.class);
final String topic = "topic";
@@ -1167,7 +1168,7 @@ public class GridMessagingSelfTest extends GridCommonAbstractTest implements Ser
Assert.assertFalse(starFut.isDone());
- discoSpi.stopBlock();
+ lsnr.stopBlockCustomEvents();
UUID id = starFut.get();
@@ -1175,7 +1176,7 @@ public class GridMessagingSelfTest extends GridCommonAbstractTest implements Ser
Assert.assertTrue(starFut.isDone());
- discoSpi.blockCustomEvent();
+ lsnr.blockCustomEvent(StopRoutineDiscoveryMessage.class);
message(ignite1.cluster().forRemotes()).send(topic, "msg1");
@@ -1195,7 +1196,7 @@ public class GridMessagingSelfTest extends GridCommonAbstractTest implements Ser
Assert.assertFalse(stopFut.isDone());
- discoSpi.stopBlock();
+ lsnr.stopBlockCustomEvents();
stopFut.get();
@@ -1209,89 +1210,6 @@ public class GridMessagingSelfTest extends GridCommonAbstractTest implements Ser
}
/**
- *
- */
- static class TestTcpDiscoverySpi extends TcpDiscoverySpi {
- /** */
- private boolean blockCustomEvt;
-
- /** */
- private final Object mux = new Object();
-
- /** */
- private List<DiscoverySpiCustomMessage> blockedMsgs = new ArrayList<>();
-
- /** {@inheritDoc} */
- @Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) throws IgniteException {
- synchronized (mux) {
- if (blockCustomEvt) {
- DiscoveryCustomMessage msg0 = GridTestUtils.getFieldValue(msg, "delegate");
-
- if (msg0 instanceof MappingProposedMessage || msg0 instanceof MappingAcceptedMessage){
- super.sendCustomEvent(msg);
-
- return;
- }
-
- if (msg0 instanceof StopRoutineDiscoveryMessage || msg0 instanceof StartRoutineDiscoveryMessage) {
- log.info("Block custom message: " + msg0);
-
- blockedMsgs.add(msg);
-
- mux.notifyAll();
-
- return;
- }
- }
- }
-
- super.sendCustomEvent(msg);
- }
-
- /**
- *
- */
- public void blockCustomEvent() {
- synchronized (mux) {
- assert blockedMsgs.isEmpty() : blockedMsgs;
-
- blockCustomEvt = true;
- }
- }
-
- /**
- * @throws InterruptedException If interrupted.
- */
- public void waitCustomEvent() throws InterruptedException {
- synchronized (mux) {
- while (blockedMsgs.isEmpty())
- mux.wait();
- }
- }
-
- /**
- *
- */
- public void stopBlock() {
- List<DiscoverySpiCustomMessage> msgs;
-
- synchronized (this) {
- msgs = new ArrayList<>(blockedMsgs);
-
- blockCustomEvt = false;
-
- blockedMsgs.clear();
- }
-
- for (DiscoverySpiCustomMessage msg : msgs) {
- log.info("Resend blocked message: " + msg);
-
- super.sendCustomEvent(msg);
- }
- }
- }
-
- /**
* Tests that message listener registers only for one oldest node.
*
* @throws Exception If an error occurred.
[04/54] [abbrv] ignite git commit: IGNITE-7222 Added ZooKeeper
discovery SPI
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
new file mode 100644
index 0000000..7708358
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -0,0 +1,4464 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.ByteArrayInputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.zip.DataFormatException;
+import java.util.zip.Deflater;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteInterruptedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CommunicationFailureResolver;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.ClusterMetricsSnapshot;
+import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.events.DiscoveryCustomEvent;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.internal.util.GridLongList;
+import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.io.GridByteArrayOutputStream;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.LT;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.marshaller.MarshallerUtils;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.apache.ignite.plugin.security.SecurityCredentials;
+import org.apache.ignite.spi.IgniteNodeValidationResult;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.apache.ignite.spi.discovery.DiscoveryDataBag;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange;
+import org.apache.ignite.spi.discovery.DiscoverySpiListener;
+import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
+import org.apache.ignite.thread.IgniteThreadPoolExecutor;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.data.Stat;
+import org.jboss.netty.util.internal.ConcurrentHashMap;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
+import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
+import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2;
+import static org.apache.zookeeper.CreateMode.EPHEMERAL_SEQUENTIAL;
+import static org.apache.zookeeper.CreateMode.PERSISTENT;
+
+/**
+ * Zookeeper Discovery Impl.
+ */
+public class ZookeeperDiscoveryImpl {
+ /** */
+ static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD";
+
+ /** */
+ static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_TIMEOUT = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_TIMEOUT";
+
+ /** */
+ static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS";
+
+ /** */
+ private static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_EVTS_THROTTLE = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_EVTS_THROTTLE";
+
+ /** */
+ final ZookeeperDiscoverySpi spi;
+
+ /** */
+ private final String igniteInstanceName;
+
+ /** */
+ private final String connectString;
+
+ /** */
+ private final int sesTimeout;
+
+ /** */
+ private final JdkMarshaller marsh = new JdkMarshaller();
+
+ /** */
+ private final ZkIgnitePaths zkPaths;
+
+ /** */
+ private final IgniteLogger log;
+
+ /** */
+ final GridSpinBusyLock busyLock = new GridSpinBusyLock();
+
+ /** */
+ private final ZookeeperClusterNode locNode;
+
+ /** */
+ private final DiscoverySpiListener lsnr;
+
+ /** */
+ private final DiscoverySpiDataExchange exchange;
+
+ /** */
+ private final boolean clientReconnectEnabled;
+
+ /** */
+ private final GridFutureAdapter<Void> joinFut = new GridFutureAdapter<>();
+
+ /** */
+ private final int evtsAckThreshold;
+
+ /** */
+ private IgniteThreadPoolExecutor utilityPool;
+
+ /** */
+ private ZkRuntimeState rtState;
+
+ /** */
+ private volatile ConnectionState connState = ConnectionState.STARTED;
+
+ /** */
+ private final AtomicBoolean stop = new AtomicBoolean();
+
+ /** */
+ private final Object stateMux = new Object();
+
+ /** */
+ public volatile IgniteDiscoverySpiInternalListener internalLsnr;
+
+ /** */
+ private final ConcurrentHashMap<Long, PingFuture> pingFuts = new ConcurrentHashMap<>();
+
+ /** */
+ private final AtomicReference<ZkCommunicationErrorProcessFuture> commErrProcFut = new AtomicReference<>();
+
+ /** */
+ private long prevSavedEvtsTopVer;
+
+ /**
+ * @param spi Discovery SPI.
+ * @param igniteInstanceName Instance name.
+ * @param log Logger.
+ * @param zkRootPath Zookeeper base path node all nodes.
+ * @param locNode Local node instance.
+ * @param lsnr Discovery events listener.
+ * @param exchange Discovery data exchange.
+ * @param internalLsnr Internal listener (used for testing only).
+ */
+ public ZookeeperDiscoveryImpl(
+ ZookeeperDiscoverySpi spi,
+ String igniteInstanceName,
+ IgniteLogger log,
+ String zkRootPath,
+ ZookeeperClusterNode locNode,
+ DiscoverySpiListener lsnr,
+ DiscoverySpiDataExchange exchange,
+ IgniteDiscoverySpiInternalListener internalLsnr) {
+ assert locNode.id() != null && locNode.isLocal() : locNode;
+
+ MarshallerUtils.setNodeName(marsh, igniteInstanceName);
+
+ zkPaths = new ZkIgnitePaths(zkRootPath);
+
+ this.spi = spi;
+ this.igniteInstanceName = igniteInstanceName;
+ this.connectString = spi.getZkConnectionString();
+ this.sesTimeout = (int)spi.getSessionTimeout();
+ this.log = log.getLogger(getClass());
+ this.locNode = locNode;
+ this.lsnr = lsnr;
+ this.exchange = exchange;
+ this.clientReconnectEnabled = locNode.isClient() && !spi.isClientReconnectDisabled();
+
+ int evtsAckThreshold = IgniteSystemProperties.getInteger(IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD, 5);
+
+ if (evtsAckThreshold <= 0)
+ evtsAckThreshold = 1;
+
+ this.evtsAckThreshold = evtsAckThreshold;
+
+ if (internalLsnr != null)
+ this.internalLsnr = internalLsnr;
+ }
+
+ /**
+ * @return Exception.
+ */
+ private static IgniteClientDisconnectedCheckedException disconnectError() {
+ return new IgniteClientDisconnectedCheckedException(null, "Client node disconnected.");
+ }
+
+ /**
+ * @return Logger.
+ */
+ IgniteLogger log() {
+ return log;
+ }
+
+ /**
+ * @return Local node instance.
+ */
+ public ClusterNode localNode() {
+ return locNode;
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @return Node instance.
+ */
+ @Nullable public ZookeeperClusterNode node(UUID nodeId) {
+ assert nodeId != null;
+
+ return rtState.top.nodesById.get(nodeId);
+ }
+
+ /**
+ * @param nodeOrder Node order.
+ * @return Node instance.
+ */
+ @Nullable public ZookeeperClusterNode node(long nodeOrder) {
+ assert nodeOrder > 0 : nodeOrder;
+
+ return rtState.top.nodesByOrder.get(nodeOrder);
+ }
+
+ /**
+ * @param fut Future to remove.
+ */
+ void clearCommunicationErrorProcessFuture(ZkCommunicationErrorProcessFuture fut) {
+ assert fut.isDone() : fut;
+
+ commErrProcFut.compareAndSet(fut, null);
+ }
+
+ /**
+ * @param node0 Problem node ID
+ * @param err Connect error.
+ */
+ public void resolveCommunicationError(ClusterNode node0, Exception err) {
+ ZookeeperClusterNode node = node(node0.id());
+
+ if (node == null)
+ throw new IgniteSpiException(new ClusterTopologyCheckedException("Node failed: " + node0.id()));
+
+ IgniteInternalFuture<Boolean> nodeStatusFut;
+
+ for (;;) {
+ checkState();
+
+ ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
+
+ if (fut == null || fut.isDone()) {
+ ZkCommunicationErrorProcessFuture newFut = ZkCommunicationErrorProcessFuture.createOnCommunicationError(
+ this,
+ node.sessionTimeout() + 1000);
+
+ if (commErrProcFut.compareAndSet(fut, newFut)) {
+ fut = newFut;
+
+ if (log.isInfoEnabled()) {
+ log.info("Created new communication error process future [errNode=" + node0.id() +
+ ", err=" + err + ']');
+ }
+
+ try {
+ checkState();
+ }
+ catch (Exception e) {
+ fut.onError(e);
+
+ throw e;
+ }
+
+ fut.scheduleCheckOnTimeout();
+ }
+ else {
+ fut = commErrProcFut.get();
+
+ if (fut == null)
+ continue;
+ }
+ }
+
+ nodeStatusFut = fut.nodeStatusFuture(node);
+
+ if (nodeStatusFut != null)
+ break;
+ else {
+ try {
+ fut.get();
+ }
+ catch (IgniteCheckedException e) {
+ U.warn(log, "Previous communication error process future failed: " + e);
+ }
+ }
+ }
+
+ try {
+ if (!nodeStatusFut.get())
+ throw new IgniteSpiException(new ClusterTopologyCheckedException("Node failed: " + node0.id()));
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteSpiException(e);
+ }
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @return Ping result.
+ */
+ public boolean pingNode(UUID nodeId) {
+ checkState();
+
+ ZkRuntimeState rtState = this.rtState;
+
+ ZookeeperClusterNode node = rtState.top.nodesById.get(nodeId);
+
+ if (node == null)
+ return false;
+
+ if (node.isLocal())
+ return true;
+
+ PingFuture fut = pingFuts.get(node.order());
+
+ if (fut == null) {
+ fut = new PingFuture(rtState, node);
+
+ PingFuture old = pingFuts.putIfAbsent(node.order(), fut);
+
+ if (old == null) {
+ if (fut.checkNodeAndState())
+ spi.getSpiContext().addTimeoutObject(fut);
+ else
+ assert fut.isDone();
+ }
+ else
+ fut = old;
+ }
+
+ try {
+ return fut.get();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteSpiException(e);
+ }
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @param warning Warning.
+ */
+ public void failNode(UUID nodeId, @Nullable String warning) {
+ ZookeeperClusterNode node = rtState.top.nodesById.get(nodeId);
+
+ if (node == null) {
+ if (log.isDebugEnabled())
+ log.debug("Ignore forcible node fail request, node does not exist: " + nodeId);
+
+ return;
+ }
+
+ if (!node.isClient()) {
+ U.warn(log, "Ignore forcible node fail request for non-client node: " + node);
+
+ return;
+ }
+
+ sendCustomMessage(new ZkForceNodeFailMessage(node.internalId(), warning));
+ }
+
+ /**
+ *
+ */
+ public void reconnect() {
+ assert clientReconnectEnabled;
+
+ synchronized (stateMux) {
+ if (connState == ConnectionState.STARTED) {
+ connState = ConnectionState.DISCONNECTED;
+
+ rtState.onCloseStart(disconnectError());
+ }
+ else
+ return;
+ }
+
+ busyLock.block();
+
+ busyLock.unblock();
+
+ rtState.zkClient.close();
+
+ UUID newId = UUID.randomUUID();
+
+ U.quietAndWarn(log, "Local node will try to reconnect to cluster with new id due to network problems [" +
+ "newId=" + newId +
+ ", prevId=" + locNode.id() +
+ ", locNode=" + locNode + ']');
+
+ runInWorkerThread(new ReconnectClosure(newId));
+ }
+
+ /**
+ * @param newId New ID.
+ */
+ private void doReconnect(UUID newId) {
+ if (rtState.joined) {
+ assert rtState.evtsData != null;
+
+ lsnr.onDiscovery(EVT_CLIENT_NODE_DISCONNECTED,
+ rtState.evtsData.topVer,
+ locNode,
+ rtState.top.topologySnapshot(),
+ Collections.<Long, Collection<ClusterNode>>emptyMap(),
+ null);
+ }
+
+ try {
+ locNode.onClientDisconnected(newId);
+
+ joinTopology(rtState);
+ }
+ catch (Exception e) {
+ if (stopping()) {
+ if (log.isDebugEnabled())
+ log.debug("Reconnect failed, node is stopping [err=" + e + ']');
+
+ return;
+ }
+
+ U.error(log, "Failed to reconnect: " + e, e);
+
+ onSegmented(e);
+ }
+ }
+
+ /**
+ * @return {@code True} if started to stop.
+ */
+ private boolean stopping() {
+ if (stop.get())
+ return true;
+
+ synchronized (stateMux) {
+ if (connState == ConnectionState.STOPPED)
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @param e Error.
+ */
+ private void onSegmented(Exception e) {
+ rtState.errForClose = e;
+
+ if (rtState.joined || joinFut.isDone()) {
+ synchronized (stateMux) {
+ connState = ConnectionState.STOPPED;
+ }
+
+ notifySegmented();
+ }
+ else
+ joinFut.onDone(e);
+ }
+
+ /**
+ *
+ */
+ private void notifySegmented() {
+ List<ClusterNode> nodes = rtState.top.topologySnapshot();
+
+ if (nodes.isEmpty())
+ nodes = Collections.singletonList((ClusterNode)locNode);
+
+ lsnr.onDiscovery(EVT_NODE_SEGMENTED,
+ rtState.evtsData != null ? rtState.evtsData.topVer : 1L,
+ locNode,
+ nodes,
+ Collections.<Long, Collection<ClusterNode>>emptyMap(),
+ null);
+ }
+
+ /**
+ * @return Remote nodes.
+ */
+ public Collection<ClusterNode> remoteNodes() {
+ checkState();
+
+ return rtState.top.remoteNodes();
+ }
+
+ /**
+ *
+ */
+ private void checkState() {
+ switch (connState) {
+ case STARTED:
+ break;
+
+ case STOPPED:
+ throw new IgniteSpiException("Node stopped.");
+
+ case DISCONNECTED:
+ throw new IgniteClientDisconnectedException(null, "Client is disconnected.");
+ }
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @return {@code True} if node joined or joining topology.
+ */
+ public boolean knownNode(UUID nodeId) {
+ while (!busyLock.enterBusy())
+ checkState();
+
+ try {
+ List<String> children = rtState.zkClient.getChildren(zkPaths.aliveNodesDir);
+
+ for (int i = 0; i < children.size(); i++) {
+ UUID id = ZkIgnitePaths.aliveNodeId(children.get(i));
+
+ if (nodeId.equals(id))
+ return true;
+ }
+
+ return false;
+ }
+ catch (ZookeeperClientFailedException e) {
+ if (clientReconnectEnabled)
+ throw new IgniteClientDisconnectedException(null, "Client is disconnected.");
+
+ throw new IgniteException(e);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteInterruptedException(e);
+ }
+ finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * @param msg Message.
+ */
+ public void sendCustomMessage(DiscoverySpiCustomMessage msg) {
+ assert msg != null;
+
+ byte[] msgBytes;
+
+ try {
+ msgBytes = marshalZip(msg);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteSpiException("Failed to marshal custom message: " + msg, e);
+ }
+
+ while (!busyLock.enterBusy())
+ checkState();
+
+ try {
+ ZookeeperClient zkClient = rtState.zkClient;
+
+ saveCustomMessage(zkClient, msgBytes);
+ }
+ catch (ZookeeperClientFailedException e) {
+ if (clientReconnectEnabled)
+ throw new IgniteClientDisconnectedException(null, "Client is disconnected.");
+
+ throw new IgniteException(e);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteInterruptedException(e);
+ }
+ finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * @param zkClient Client.
+ * @param msgBytes Marshalled message.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ private void saveCustomMessage(ZookeeperClient zkClient, byte[] msgBytes)
+ throws ZookeeperClientFailedException, InterruptedException
+ {
+ String prefix = UUID.randomUUID().toString();
+
+ int partCnt = 1;
+
+ int overhead = 10;
+
+ UUID locId = locNode.id();
+
+ String path = zkPaths.createCustomEventPath(prefix, locId, partCnt);
+
+ if (zkClient.needSplitNodeData(path, msgBytes, overhead)) {
+ List<byte[]> parts = zkClient.splitNodeData(path, msgBytes, overhead);
+
+ String partsBasePath = zkPaths.customEventPartsBasePath(prefix, locId);
+
+ saveMultipleParts(zkClient, partsBasePath, parts);
+
+ msgBytes = null;
+
+ partCnt = parts.size();
+ }
+
+ zkClient.createSequential(prefix,
+ zkPaths.customEvtsDir,
+ zkPaths.createCustomEventPath(prefix, locId, partCnt),
+ msgBytes,
+ CreateMode.PERSISTENT_SEQUENTIAL);
+ }
+
+ /**
+ * @return Cluster start time.
+ */
+ public long gridStartTime() {
+ return rtState.gridStartTime;
+ }
+
+ /**
+ * Starts join procedure and waits for {@link EventType#EVT_NODE_JOINED} event for local node.
+ *
+ * @throws InterruptedException If interrupted.
+ */
+ public void startJoinAndWait() throws InterruptedException {
+ joinTopology(null);
+
+ for (;;) {
+ try {
+ joinFut.get(10_000);
+
+ break;
+ }
+ catch (IgniteFutureTimeoutCheckedException e) {
+ U.warn(log, "Waiting for local join event [nodeId=" + locNode.id() + ", name=" + igniteInstanceName + ']');
+ }
+ catch (Exception e) {
+ IgniteSpiException spiErr = X.cause(e, IgniteSpiException.class);
+
+ if (spiErr != null)
+ throw spiErr;
+
+ throw new IgniteSpiException("Failed to join cluster", e);
+ }
+ }
+ }
+
+ /**
+ * @param prevState Previous state in case of connect retry.
+ * @throws InterruptedException If interrupted.
+ */
+ private void joinTopology(@Nullable ZkRuntimeState prevState) throws InterruptedException {
+ if (!busyLock.enterBusy())
+ return;
+
+ try {
+ boolean reconnect = prevState != null;
+
+ // Need fire EVT_CLIENT_NODE_RECONNECTED event if reconnect after already joined.
+ boolean prevJoined = prevState != null && prevState.joined;
+
+ IgniteDiscoverySpiInternalListener internalLsnr = this.internalLsnr;
+
+ if (internalLsnr != null)
+ internalLsnr.beforeJoin(locNode, log);
+
+ if (locNode.isClient() && reconnect)
+ locNode.setAttributes(spi.getSpiContext().nodeAttributes());
+
+ marshalCredentialsOnJoin(locNode);
+
+ synchronized (stateMux) {
+ if (connState == ConnectionState.STOPPED)
+ return;
+
+ connState = ConnectionState.STARTED;
+ }
+
+ ZkRuntimeState rtState = this.rtState = new ZkRuntimeState(prevJoined);
+
+ DiscoveryDataBag discoDataBag = new DiscoveryDataBag(locNode.id(), locNode.isClient());
+
+ exchange.collect(discoDataBag);
+
+ ZkJoiningNodeData joinData = new ZkJoiningNodeData(locNode, discoDataBag.joiningNodeData());
+
+ byte[] joinDataBytes;
+
+ try {
+ joinDataBytes = marshalZip(joinData);
+ }
+ catch (Exception e) {
+ throw new IgniteSpiException("Failed to marshal joining node data", e);
+ }
+
+ try {
+ rtState.zkClient = new ZookeeperClient(
+ igniteInstanceName,
+ log,
+ connectString,
+ sesTimeout,
+ new ConnectionLossListener());
+ }
+ catch (Exception e) {
+ throw new IgniteSpiException("Failed to create Zookeeper client", e);
+ }
+
+ startJoin(rtState, prevState, joinDataBytes);
+ }
+ finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * @throws InterruptedException If interrupted.
+ */
+ private void initZkNodes() throws InterruptedException {
+ try {
+ ZookeeperClient client = rtState.zkClient;
+
+ if (!client.exists(zkPaths.clusterDir)) {
+ createRootPathParents(zkPaths.clusterDir, client);
+
+ client.createIfNeeded(zkPaths.clusterDir, null, PERSISTENT);
+ }
+
+ List<String> createdDirs = client.getChildren(zkPaths.clusterDir);
+
+ String[] requiredDirs = {
+ zkPaths.evtsPath,
+ zkPaths.joinDataDir,
+ zkPaths.customEvtsDir,
+ zkPaths.customEvtsPartsDir,
+ zkPaths.customEvtsAcksDir,
+ zkPaths.aliveNodesDir};
+
+ List<String> dirs = new ArrayList<>();
+
+ for (String dir : requiredDirs) {
+ String dir0 = dir.substring(zkPaths.clusterDir.length() + 1);
+
+ if (!createdDirs.contains(dir0))
+ dirs.add(dir);
+ }
+
+ try {
+ if (!dirs.isEmpty())
+ client.createAll(dirs, PERSISTENT);
+ }
+ catch (KeeperException.NodeExistsException e) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to create nodes using bulk operation: " + e);
+
+ for (String dir : dirs)
+ client.createIfNeeded(dir, null, PERSISTENT);
+ }
+ }
+ catch (ZookeeperClientFailedException e) {
+ throw new IgniteSpiException("Failed to initialize Zookeeper nodes", e);
+ }
+ }
+
+ /**
+ * @param rootDir Root directory.
+ * @param client Client.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ private void createRootPathParents(String rootDir, ZookeeperClient client)
+ throws ZookeeperClientFailedException, InterruptedException {
+ int startIdx = 0;
+
+ for (;;) {
+ int separatorIdx = rootDir.indexOf(ZkIgnitePaths.PATH_SEPARATOR, startIdx);
+
+ if (separatorIdx == -1)
+ break;
+
+ if (separatorIdx > 0) {
+ String path = rootDir.substring(0, separatorIdx);
+
+ client.createIfNeeded(path, null, CreateMode.PERSISTENT);
+ }
+
+ startIdx = separatorIdx + 1;
+ }
+ }
+
+ /**
+ * @param zkClient Client.
+ * @param basePath Base path.
+ * @param partCnt Parts count.
+ */
+ private void deleteMultiplePartsAsync(ZookeeperClient zkClient, String basePath, int partCnt) {
+ for (int i = 0; i < partCnt; i++) {
+ String path = multipartPathName(basePath, i);
+
+ zkClient.deleteIfExistsAsync(path);
+ }
+ }
+
+ /**
+ * @param zkClient Client.
+ * @param basePath Base path.
+ * @param partCnt Parts count.
+ * @return Read parts.
+ * @throws Exception If failed.
+ */
+ private byte[] readMultipleParts(ZookeeperClient zkClient, String basePath, int partCnt)
+ throws Exception {
+ assert partCnt >= 1;
+
+ if (partCnt > 1) {
+ List<byte[]> parts = new ArrayList<>(partCnt);
+
+ int totSize = 0;
+
+ for (int i = 0; i < partCnt; i++) {
+ byte[] part = zkClient.getData(multipartPathName(basePath, i));
+
+ parts.add(part);
+
+ totSize += part.length;
+ }
+
+ byte[] res = new byte[totSize];
+
+ int pos = 0;
+
+ for (int i = 0; i < partCnt; i++) {
+ byte[] part = parts.get(i);
+
+ System.arraycopy(part, 0, res, pos, part.length);
+
+ pos += part.length;
+ }
+
+ return res;
+ }
+ else
+ return zkClient.getData(multipartPathName(basePath, 0));
+ }
+
+ /**
+ * @param zkClient Client.
+ * @param basePath Base path.
+ * @param parts Data parts.
+ * @return Number of parts.
+ * @throws ZookeeperClientFailedException If client failed.
+ * @throws InterruptedException If interrupted.
+ */
+ private int saveMultipleParts(ZookeeperClient zkClient, String basePath, List<byte[]> parts)
+ throws ZookeeperClientFailedException, InterruptedException
+ {
+ assert parts.size() > 1;
+
+ for (int i = 0; i < parts.size(); i++) {
+ byte[] part = parts.get(i);
+
+ String path = multipartPathName(basePath, i);
+
+ zkClient.createIfNeeded(path, part, PERSISTENT);
+ }
+
+ return parts.size();
+ }
+
+ /**
+ * @param basePath Base path.
+ * @param part Part number.
+ * @return Path.
+ */
+ private static String multipartPathName(String basePath, int part) {
+ return basePath + String.format("%04d", part);
+ }
+
+ /**
+ * @param rtState Runtime state.
+ * @param joinDataBytes Joining node data.
+ * @param prevState Previous state in case of connect retry.
+ * @throws InterruptedException If interrupted.
+ */
+ private void startJoin(ZkRuntimeState rtState, @Nullable ZkRuntimeState prevState, final byte[] joinDataBytes)
+ throws InterruptedException
+ {
+ try {
+ long startTime = System.currentTimeMillis();
+
+ initZkNodes();
+
+ String prefix = UUID.randomUUID().toString();
+
+ rtState.init(new ZkWatcher(rtState), new AliveNodeDataWatcher(rtState));
+
+ ZookeeperClient zkClient = rtState.zkClient;
+
+ final int OVERHEAD = 5;
+
+ // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8193
+ String joinDataPath = zkPaths.joinDataDir + "/" + prefix + ":" + locNode.id();
+
+ if (zkClient.needSplitNodeData(joinDataPath, joinDataBytes, OVERHEAD)) {
+ List<byte[]> parts = zkClient.splitNodeData(joinDataPath, joinDataBytes, OVERHEAD);
+
+ rtState.joinDataPartCnt = parts.size();
+
+ saveMultipleParts(zkClient, joinDataPath + ":", parts);
+
+ joinDataPath = zkClient.createIfNeeded(
+ joinDataPath,
+ marshalZip(new ZkJoiningNodeData(parts.size())),
+ PERSISTENT);
+ }
+ else {
+ joinDataPath = zkClient.createIfNeeded(
+ joinDataPath,
+ joinDataBytes,
+ PERSISTENT);
+ }
+
+ rtState.locNodeZkPath = zkClient.createSequential(
+ prefix,
+ zkPaths.aliveNodesDir,
+ zkPaths.aliveNodePathForCreate(prefix, locNode),
+ null,
+ EPHEMERAL_SEQUENTIAL);
+
+ rtState.internalOrder = ZkIgnitePaths.aliveInternalId(rtState.locNodeZkPath);
+
+ if (log.isInfoEnabled()) {
+ log.info("Node started join [nodeId=" + locNode.id() +
+ ", instanceName=" + locNode.attribute(ATTR_IGNITE_INSTANCE_NAME) +
+ ", zkSessionId=0x" + Long.toHexString(rtState.zkClient.zk().getSessionId()) +
+ ", joinDataSize=" + joinDataBytes.length +
+ (rtState.joinDataPartCnt > 1 ? (", joinDataPartCnt=" + rtState.joinDataPartCnt) : "") +
+ ", consistentId=" + locNode.consistentId() +
+ ", initTime=" + (System.currentTimeMillis() - startTime) +
+ ", nodePath=" + rtState.locNodeZkPath + ']');
+ }
+
+ /*
+ If node can not join due to validation error this error is reported in join data,
+ As a minor optimization do not start watch join data immediately, but only if do not receive
+ join event after some timeout.
+ */
+ CheckJoinErrorWatcher joinErrorWatcher = new CheckJoinErrorWatcher(5000, joinDataPath, rtState);
+
+ rtState.joinErrTo = joinErrorWatcher.timeoutObj;
+
+ if (locNode.isClient() && spi.getJoinTimeout() > 0) {
+ ZkTimeoutObject joinTimeoutObj = prevState != null ? prevState.joinTo : null;
+
+ if (joinTimeoutObj == null) {
+ joinTimeoutObj = new JoinTimeoutObject(spi.getJoinTimeout());
+
+ spi.getSpiContext().addTimeoutObject(joinTimeoutObj);
+ }
+
+ rtState.joinTo = joinTimeoutObj;
+ }
+
+ if (!locNode.isClient())
+ zkClient.getChildrenAsync(zkPaths.aliveNodesDir, null, new CheckCoordinatorCallback(rtState));
+
+ zkClient.getDataAsync(zkPaths.evtsPath, rtState.watcher, rtState.watcher);
+
+ spi.getSpiContext().addTimeoutObject(rtState.joinErrTo);
+ }
+ catch (IgniteCheckedException | ZookeeperClientFailedException e) {
+ throw new IgniteSpiException("Failed to initialize Zookeeper nodes", e);
+ }
+ }
+
+ /**
+ * Authenticate local node.
+ *
+ * @param nodeAuth Authenticator.
+ * @param locCred Local security credentials for authentication.
+ * @throws IgniteSpiException If any error occurs.
+ */
+ private void localAuthentication(DiscoverySpiNodeAuthenticator nodeAuth, SecurityCredentials locCred){
+ assert nodeAuth != null;
+ assert locCred != null;
+
+ try {
+ SecurityContext subj = nodeAuth.authenticateNode(locNode, locCred);
+
+ // Note: exception message is checked in tests.
+ if (subj == null)
+ throw new IgniteSpiException("Authentication failed for local node.");
+
+ if (!(subj instanceof Serializable))
+ throw new IgniteSpiException("Authentication subject is not Serializable.");
+
+ Map<String, Object> attrs = new HashMap<>(locNode.attributes());
+
+ attrs.put(ATTR_SECURITY_SUBJECT_V2, U.marshal(marsh, subj));
+
+ locNode.setAttributes(attrs);
+ }
+ catch (Exception e) {
+ throw new IgniteSpiException("Failed to authenticate local node (will shutdown local node).", e);
+ }
+ }
+
+ /**
+ * @param node Node.
+ * @param zipBytes Zip-compressed marshalled security subject.
+ * @throws Exception If failed.
+ */
+ private void setNodeSecuritySubject(ZookeeperClusterNode node, byte[] zipBytes) throws Exception {
+ assert zipBytes != null;
+
+ Map<String, Object> attrs = new HashMap<>(node.getAttributes());
+
+ attrs.put(ATTR_SECURITY_SUBJECT_V2, unzip(zipBytes));
+
+ node.setAttributes(attrs);
+ }
+
+ /**
+ * @param node Node.
+ * @return Credentials.
+ * @throws IgniteCheckedException If failed to unmarshal.
+ */
+ private SecurityCredentials unmarshalCredentials(ZookeeperClusterNode node) throws Exception {
+ byte[] credBytes = (byte[])node.getAttributes().get(ATTR_SECURITY_CREDENTIALS);
+
+ if (credBytes == null)
+ return null;
+
+ return unmarshalZip(credBytes);
+ }
+
+ /**
+ * Marshalls credentials with discovery SPI marshaller (will replace attribute value).
+ *
+ * @param node Node to marshall credentials for.
+ * @throws IgniteSpiException If marshalling failed.
+ */
+ private void marshalCredentialsOnJoin(ZookeeperClusterNode node) throws IgniteSpiException {
+ try {
+ // Use security-unsafe getter.
+ Map<String, Object> attrs0 = node.getAttributes();
+
+ Object creds = attrs0.get(ATTR_SECURITY_CREDENTIALS);
+
+ if (creds != null) {
+ Map<String, Object> attrs = new HashMap<>(attrs0);
+
+ assert !(creds instanceof byte[]);
+
+ attrs.put(ATTR_SECURITY_CREDENTIALS, marshalZip(creds));
+
+ node.setAttributes(attrs);
+ }
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteSpiException("Failed to marshal node security credentials: " + node.id(), e);
+ }
+ }
+
+ /**
+ *
+ */
+ private class UpdateProcessedEventsTimeoutObject extends ZkTimeoutObject {
+ /** */
+ private final ZkRuntimeState rtState;
+
+ /**
+ * @param rtState Runtime state.
+ * @param timeout Timeout.
+ */
+ UpdateProcessedEventsTimeoutObject(ZkRuntimeState rtState, long timeout) {
+ super(timeout);
+
+ this.rtState = rtState;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onTimeout() {
+ runInWorkerThread(new ZkRunnable(rtState, ZookeeperDiscoveryImpl.this) {
+ @Override protected void run0() throws Exception {
+ updateProcessedEventsOnTimeout(rtState, UpdateProcessedEventsTimeoutObject.this);
+ }
+ });
+ }
+ }
+
+ /**
+ *
+ */
+ private class JoinTimeoutObject extends ZkTimeoutObject {
+ /**
+ * @param timeout Timeout.
+ */
+ JoinTimeoutObject(long timeout) {
+ super(timeout);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onTimeout() {
+ if (cancelled || rtState.joined)
+ return;
+
+ runInWorkerThread(new Runnable() {
+ @Override public void run() {
+ synchronized (stateMux) {
+ if (cancelled || rtState.joined)
+ return;
+
+ if (connState == ConnectionState.STOPPED)
+ return;
+
+ connState = ConnectionState.STOPPED;
+ }
+
+ U.warn(log, "Failed to connect to cluster, either connection to ZooKeeper can not be established or there " +
+ "are no alive server nodes (consider increasing 'joinTimeout' configuration property) [" +
+ "joinTimeout=" + spi.getJoinTimeout() + ']');
+
+ // Note: exception message is checked in tests.
+ onSegmented(new IgniteSpiException("Failed to connect to cluster within configured timeout"));
+ }
+ });
+ }
+ }
+
+ /**
+ *
+ */
+ private class CheckJoinErrorWatcher extends ZkAbstractWatcher implements AsyncCallback.DataCallback {
+ /** */
+ private final String joinDataPath;
+
+ /** */
+ private ZkTimeoutObject timeoutObj;
+
+ /**
+ * @param timeout Timeout.
+ * @param joinDataPath0 Node joined data path.
+ * @param rtState0 State.
+ */
+ CheckJoinErrorWatcher(long timeout, String joinDataPath0, ZkRuntimeState rtState0) {
+ super(rtState0, ZookeeperDiscoveryImpl.this);
+
+ this.joinDataPath = joinDataPath0;
+
+ timeoutObj = new ZkTimeoutObject(timeout) {
+ @Override public void onTimeout() {
+ if (rtState.errForClose != null || rtState.joined)
+ return;
+
+ synchronized (stateMux) {
+ if (connState != ConnectionState.STARTED)
+ return;
+ }
+
+ rtState.zkClient.getDataAsync(joinDataPath,
+ CheckJoinErrorWatcher.this,
+ CheckJoinErrorWatcher.this);
+ }
+ };
+ }
+
+ /** {@inheritDoc} */
+ @Override public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) {
+ if (rc != 0)
+ return;
+
+ if (!onProcessStart())
+ return;
+
+ try {
+ Object obj = unmarshalZip(data);
+
+ if (obj instanceof ZkInternalJoinErrorMessage) {
+ ZkInternalJoinErrorMessage joinErr = (ZkInternalJoinErrorMessage)obj;
+
+ onSegmented(new IgniteSpiException(joinErr.err));
+ }
+
+ onProcessEnd();
+ }
+ catch (Throwable e) {
+ onProcessError(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void process0(WatchedEvent evt) {
+ if (rtState.errForClose != null || rtState.joined)
+ return;
+
+ if (evt.getType() == Event.EventType.NodeDataChanged)
+ rtState.zkClient.getDataAsync(evt.getPath(), this, this);
+ }
+ }
+
+ /**
+ * @param aliveNodes Alive nodes.
+ * @throws Exception If failed.
+ */
+ private void checkIsCoordinator(final List<String> aliveNodes) throws Exception {
+ assert !locNode.isClient();
+
+ TreeMap<Long, String> aliveSrvs = new TreeMap<>();
+
+ long locInternalOrder = rtState.internalOrder;
+
+ for (String aliveNodePath : aliveNodes) {
+ if (ZkIgnitePaths.aliveNodeClientFlag(aliveNodePath))
+ continue;
+
+ Long internalId = ZkIgnitePaths.aliveInternalId(aliveNodePath);
+
+ aliveSrvs.put(internalId, aliveNodePath);
+ }
+
+ assert !aliveSrvs.isEmpty();
+
+ Map.Entry<Long, String> crdE = aliveSrvs.firstEntry();
+
+ if (locInternalOrder == crdE.getKey())
+ onBecomeCoordinator(aliveNodes);
+ else {
+ assert aliveSrvs.size() > 1 : aliveSrvs;
+
+ Map.Entry<Long, String> prevE = aliveSrvs.floorEntry(locInternalOrder - 1);
+
+ assert prevE != null;
+
+ if (log.isInfoEnabled()) {
+ log.info("Discovery coordinator already exists, watch for previous server node [" +
+ "locId=" + locNode.id() +
+ ", watchPath=" + prevE.getValue() + ']');
+ }
+
+ PreviousNodeWatcher watcher = new ServerPreviousNodeWatcher(rtState);
+
+ rtState.zkClient.existsAsync(zkPaths.aliveNodesDir + "/" + prevE.getValue(), watcher, watcher);
+ }
+ }
+
+ /**
+ * @param aliveNodes Alive nodes.
+ * @throws Exception If failed.
+ */
+ private void checkClientsStatus(final List<String> aliveNodes) throws Exception {
+ assert locNode.isClient() : locNode;
+ assert rtState.joined;
+ assert rtState.evtsData != null;
+
+ TreeMap<Long, String> aliveClients = new TreeMap<>();
+
+ String srvPath = null;
+ Long srvInternalOrder = null;
+
+ long locInternalOrder = rtState.internalOrder;
+
+ for (String aliveNodePath : aliveNodes) {
+ Long internalId = ZkIgnitePaths.aliveInternalId(aliveNodePath);
+
+ if (ZkIgnitePaths.aliveNodeClientFlag(aliveNodePath))
+ aliveClients.put(internalId, aliveNodePath);
+ else {
+ if (srvInternalOrder == null || internalId < srvInternalOrder) {
+ srvPath = aliveNodePath;
+ srvInternalOrder = internalId;
+ }
+ }
+ }
+
+ assert !aliveClients.isEmpty();
+
+ Map.Entry<Long, String> oldest = aliveClients.firstEntry();
+
+ boolean oldestClient = locInternalOrder == oldest.getKey();
+
+ if (srvPath == null) {
+ if (oldestClient) {
+ Stat stat = new Stat();
+
+ ZkDiscoveryEventsData prevEvts = rtState.evtsData;
+
+ byte[] evtsBytes = rtState.zkClient.getData(zkPaths.evtsPath, stat);
+
+ assert evtsBytes.length > 0;
+
+ ZkDiscoveryEventsData newEvts = unmarshalZip(evtsBytes);
+
+ if (prevEvts.clusterId.equals(newEvts.clusterId)) {
+ U.warn(log, "All server nodes failed, notify all clients [locId=" + locNode.id() + ']');
+
+ generateNoServersEvent(newEvts, stat);
+ }
+ else
+ U.warn(log, "All server nodes failed (received events from new cluster).");
+ }
+ }
+ else {
+ String watchPath;
+
+ if (oldestClient) {
+ watchPath = srvPath;
+
+ if (log.isInfoEnabled()) {
+ log.info("Servers exists, watch for server node [locId=" + locNode.id() +
+ ", watchPath=" + watchPath + ']');
+ }
+ }
+ else {
+ assert aliveClients.size() > 1 : aliveClients;
+
+ Map.Entry<Long, String> prevE = aliveClients.floorEntry(locInternalOrder - 1);
+
+ assert prevE != null;
+
+ watchPath = prevE.getValue();
+
+ if (log.isInfoEnabled()) {
+ log.info("Servers exists, watch for previous node [locId=" + locNode.id() +
+ ", watchPath=" + watchPath + ']');
+ }
+ }
+
+ PreviousNodeWatcher watcher = new ClientPreviousNodeWatcher(rtState);
+
+ rtState.zkClient.existsAsync(zkPaths.aliveNodesDir + "/" + watchPath, watcher, watcher);
+ }
+ }
+
+ /**
+ * @param evtsData Events data.
+ * @param evtsStat Events zookeeper state.
+ * @throws Exception If failed.
+ */
+ private void generateNoServersEvent(ZkDiscoveryEventsData evtsData, Stat evtsStat) throws Exception {
+ evtsData.evtIdGen++;
+
+ ZkDiscoveryCustomEventData evtData = new ZkDiscoveryCustomEventData(
+ evtsData.evtIdGen,
+ 0L,
+ evtsData.topVer,
+ locNode.id(),
+ new ZkNoServersMessage(),
+ null);
+
+ Collection<ZookeeperClusterNode> nodesToAck = Collections.emptyList();
+
+ evtsData.addEvent(nodesToAck, evtData);
+
+ byte[] newEvtsBytes = marshalZip(evtsData);
+
+ try {
+ rtState.zkClient.setData(zkPaths.evtsPath, newEvtsBytes, evtsStat.getVersion());
+ }
+ catch (KeeperException.BadVersionException e) {
+ // Version can change if new cluster started and saved new events.
+ if (log.isDebugEnabled())
+ log.debug("Failed to save no servers message");
+ }
+ }
+
+ /**
+ * @param lastEvts Last events from previous coordinator.
+ * @throws Exception If failed.
+ */
+ private void previousCoordinatorCleanup(ZkDiscoveryEventsData lastEvts) throws Exception {
+ for (ZkDiscoveryEventData evtData : lastEvts.evts.values()) {
+ if (evtData instanceof ZkDiscoveryCustomEventData) {
+ ZkDiscoveryCustomEventData evtData0 = (ZkDiscoveryCustomEventData)evtData;
+
+ // It is possible previous coordinator failed before finished cleanup.
+ if (evtData0.msg instanceof ZkCommunicationErrorResolveFinishMessage) {
+ try {
+ ZkCommunicationErrorResolveFinishMessage msg =
+ (ZkCommunicationErrorResolveFinishMessage)evtData0.msg;
+
+ ZkCommunicationErrorResolveResult res = unmarshalZip(
+ ZkDistributedCollectDataFuture.readResult(rtState.zkClient, zkPaths, msg.futId));
+
+ deleteAliveNodes(res.killedNodes);
+ }
+ catch (KeeperException.NoNodeException ignore) {
+ // No-op.
+ }
+ }
+ else if (evtData0.resolvedMsg instanceof ZkForceNodeFailMessage)
+ deleteAliveNode(((ZkForceNodeFailMessage)evtData0.resolvedMsg).nodeInternalId);
+ }
+ }
+ }
+
+ /**
+ * @param aliveNodes Alive nodes paths.
+ * @throws Exception If failed.
+ */
+ private void onBecomeCoordinator(List<String> aliveNodes) throws Exception {
+ ZkDiscoveryEventsData prevEvts = processNewEvents(rtState.zkClient.getData(zkPaths.evtsPath));
+
+ rtState.crd = true;
+
+ if (rtState.joined) {
+ if (log.isInfoEnabled())
+ log.info("Node is new discovery coordinator [locId=" + locNode.id() + ']');
+
+ assert locNode.order() > 0 : locNode;
+ assert rtState.evtsData != null;
+
+ previousCoordinatorCleanup(rtState.evtsData);
+
+ UUID futId = rtState.evtsData.communicationErrorResolveFutureId();
+
+ if (futId != null) {
+ if (log.isInfoEnabled()) {
+ log.info("New discovery coordinator will handle already started cluster-wide communication " +
+ "error resolve [reqId=" + futId + ']');
+ }
+
+ ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
+
+ ZkDistributedCollectDataFuture collectResFut = collectCommunicationStatusFuture(futId);
+
+ if (fut != null)
+ fut.nodeResultCollectFuture(collectResFut);
+ }
+
+ for (ZkDiscoveryEventData evtData : rtState.evtsData.evts.values())
+ evtData.initRemainingAcks(rtState.top.nodesByOrder.values());
+
+ handleProcessedEvents("crd");
+ }
+ else {
+ String locAlivePath = rtState.locNodeZkPath.substring(rtState.locNodeZkPath.lastIndexOf('/') + 1);
+
+ deleteJoiningNodeData(locNode.id(),
+ ZkIgnitePaths.aliveNodePrefixId(locAlivePath),
+ rtState.joinDataPartCnt);
+
+ DiscoverySpiNodeAuthenticator nodeAuth = spi.getAuthenticator();
+
+ if (nodeAuth != null) {
+ try {
+ if (log.isInfoEnabled()) {
+ log.info("Node is first server node in cluster, try authenticate local node " +
+ "[locId=" + locNode.id() + ']');
+ }
+
+ localAuthentication(nodeAuth, unmarshalCredentials(locNode));
+ }
+ catch (Exception e) {
+ U.warn(log, "Local node authentication failed: " + e, e);
+
+ onSegmented(e);
+
+ // Stop any further processing.
+ throw new ZookeeperClientFailedException("Local node authentication failed: " + e);
+ }
+ }
+
+ newClusterStarted(prevEvts);
+ }
+
+ rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
+ rtState.zkClient.getChildrenAsync(zkPaths.customEvtsDir, rtState.watcher, rtState.watcher);
+
+ for (String alivePath : aliveNodes)
+ watchAliveNodeData(alivePath);
+ }
+
+ /**
+ * @param alivePath Node path.
+ */
+ private void watchAliveNodeData(String alivePath) {
+ assert rtState.locNodeZkPath != null;
+
+ String path = zkPaths.aliveNodesDir + "/" + alivePath;
+
+ if (!path.equals(rtState.locNodeZkPath))
+ rtState.zkClient.getDataAsync(path, rtState.aliveNodeDataWatcher, rtState.aliveNodeDataWatcher);
+ }
+
+ /**
+ * @param aliveNodes ZK nodes representing alive cluster nodes.
+ * @throws Exception If failed.
+ */
+ private void generateTopologyEvents(List<String> aliveNodes) throws Exception {
+ assert rtState.crd;
+
+ if (log.isInfoEnabled())
+ log.info("Process alive nodes change [alives=" + aliveNodes.size() + "]");
+
+ if (rtState.updateAlives) {
+ aliveNodes = rtState.zkClient.getChildren(zkPaths.aliveNodesDir);
+
+ rtState.updateAlives = false;
+ }
+
+ TreeMap<Long, String> alives = new TreeMap<>();
+
+ for (String child : aliveNodes) {
+ Long internalId = ZkIgnitePaths.aliveInternalId(child);
+
+ Object old = alives.put(internalId, child);
+
+ assert old == null;
+ }
+
+ TreeMap<Long, ZookeeperClusterNode> curTop = new TreeMap<>(rtState.top.nodesByOrder);
+
+ int newEvts = 0;
+
+ final int MAX_NEW_EVTS = IgniteSystemProperties.getInteger(IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS, 100);
+
+ List<ZookeeperClusterNode> failedNodes = null;
+
+ for (Map.Entry<Long, ZookeeperClusterNode> e : rtState.top.nodesByInternalId.entrySet()) {
+ if (!alives.containsKey(e.getKey())) {
+ ZookeeperClusterNode failedNode = e.getValue();
+
+ if (failedNodes == null)
+ failedNodes = new ArrayList<>();
+
+ failedNodes.add(failedNode);
+
+ generateNodeFail(curTop, failedNode);
+
+ newEvts++;
+
+ if (newEvts == MAX_NEW_EVTS) {
+ saveAndProcessNewEvents();
+
+ if (log.isInfoEnabled()) {
+ log.info("Delay alive nodes change process, max event threshold reached [newEvts=" + newEvts +
+ ", totalEvts=" + rtState.evtsData.evts.size() + ']');
+ }
+
+ handleProcessedEventsOnNodesFail(failedNodes);
+
+ throttleNewEventsGeneration();
+
+ rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
+
+ return;
+ }
+ }
+ }
+
+ // Process failures before processing join, otherwise conflicts are possible in case of fast node stop/re-start.
+ if (newEvts > 0) {
+ saveAndProcessNewEvents();
+
+ handleProcessedEventsOnNodesFail(failedNodes);
+
+ rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
+
+ return;
+ }
+
+ generateJoinEvents(curTop, alives, MAX_NEW_EVTS);
+
+ if (failedNodes != null)
+ handleProcessedEventsOnNodesFail(failedNodes);
+ }
+
+ /**
+ * @param curTop Current topology.
+ * @param alives Alive znodes.
+ * @param MAX_NEW_EVTS Max event to process.
+ * @throws Exception If failed.
+ */
+ private void generateJoinEvents(TreeMap<Long, ZookeeperClusterNode> curTop,
+ TreeMap<Long, String> alives,
+ final int MAX_NEW_EVTS) throws Exception
+ {
+ ZkBulkJoinContext joinCtx = new ZkBulkJoinContext();
+
+ for (Map.Entry<Long, String> e : alives.entrySet()) {
+ Long internalId = e.getKey();
+
+ if (!rtState.top.nodesByInternalId.containsKey(internalId)) {
+ UUID rslvFutId = rtState.evtsData.communicationErrorResolveFutureId();
+
+ if (rslvFutId != null) {
+ if (log.isInfoEnabled()) {
+ log.info("Delay alive nodes change process while communication error resolve " +
+ "is in progress [reqId=" + rslvFutId + ']');
+ }
+
+ break;
+ }
+
+ processJoinOnCoordinator(joinCtx, curTop, internalId, e.getValue());
+
+ if (joinCtx.nodes() == MAX_NEW_EVTS) {
+ generateBulkJoinEvent(curTop, joinCtx);
+
+ if (log.isInfoEnabled()) {
+ log.info("Delay alive nodes change process, max event threshold reached [" +
+ "newEvts=" + joinCtx.nodes() +
+ ", totalEvts=" + rtState.evtsData.evts.size() + ']');
+ }
+
+ throttleNewEventsGeneration();
+
+ rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
+
+ return;
+ }
+ }
+ }
+
+ if (joinCtx.nodes() > 0)
+ generateBulkJoinEvent(curTop, joinCtx);
+ }
+
+ /**
+ * @param curTop Current topology.
+ * @param joinCtx Joined nodes context.
+ * @throws Exception If failed.
+ */
+ private void generateBulkJoinEvent(TreeMap<Long, ZookeeperClusterNode> curTop, ZkBulkJoinContext joinCtx)
+ throws Exception
+ {
+ rtState.evtsData.evtIdGen++;
+
+ long evtId = rtState.evtsData.evtIdGen;
+
+ List<T2<ZkJoinedNodeEvtData, Map<Integer, Serializable>>> nodes = joinCtx.nodes;
+
+ assert nodes != null && nodes.size() > 0;
+
+ int nodeCnt = nodes.size();
+
+ List<ZkJoinedNodeEvtData> joinedNodes = new ArrayList<>(nodeCnt);
+
+ Map<Long, byte[]> discoDataMap = U.newHashMap(nodeCnt);
+ Map<Long, Long> dupDiscoData = null;
+
+ for (int i = 0; i < nodeCnt; i++) {
+ T2<ZkJoinedNodeEvtData, Map<Integer, Serializable>> nodeEvtData = nodes.get(i);
+
+ Map<Integer, Serializable> discoData = nodeEvtData.get2();
+
+ byte[] discoDataBytes = U.marshal(marsh, discoData);
+
+ Long dupDataNode = null;
+
+ for (Map.Entry<Long, byte[]> e : discoDataMap.entrySet()) {
+ if (Arrays.equals(discoDataBytes, e.getValue())) {
+ dupDataNode = e.getKey();
+
+ break;
+ }
+ }
+
+ long nodeTopVer = nodeEvtData.get1().topVer;
+
+ if (dupDataNode != null) {
+ if (dupDiscoData == null)
+ dupDiscoData = new HashMap<>();
+
+ Long old = dupDiscoData.put(nodeTopVer, dupDataNode);
+
+ assert old == null : old;
+ }
+ else
+ discoDataMap.put(nodeTopVer, discoDataBytes);
+
+ joinedNodes.add(nodeEvtData.get1());
+ }
+
+ int overhead = 5;
+
+ ZkJoinEventDataForJoined dataForJoined = new ZkJoinEventDataForJoined(
+ new ArrayList<>(curTop.values()),
+ discoDataMap,
+ dupDiscoData);
+
+ byte[] dataForJoinedBytes = marshalZip(dataForJoined);
+
+ long addDataStart = System.currentTimeMillis();
+
+ int dataForJoinedPartCnt = saveData(zkPaths.joinEventDataPathForJoined(evtId),
+ dataForJoinedBytes,
+ overhead);
+
+ long addDataTime = System.currentTimeMillis() - addDataStart;
+
+ ZkDiscoveryNodeJoinEventData evtData = new ZkDiscoveryNodeJoinEventData(
+ evtId,
+ rtState.evtsData.topVer,
+ joinedNodes,
+ dataForJoinedPartCnt);
+
+ rtState.evtsData.addEvent(curTop.values(), evtData);
+
+ if (log.isInfoEnabled()) {
+ if (nodeCnt > 1) {
+ log.info("Generated NODE_JOINED bulk event [" +
+ "nodeCnt=" + nodeCnt +
+ ", dataForJoinedSize=" + dataForJoinedBytes.length +
+ ", dataForJoinedPartCnt=" + dataForJoinedPartCnt +
+ ", addDataTime=" + addDataTime +
+ ", evt=" + evtData + ']');
+ }
+ else {
+ log.info("Generated NODE_JOINED event [" +
+ "dataForJoinedSize=" + dataForJoinedBytes.length +
+ ", dataForJoinedPartCnt=" + dataForJoinedPartCnt +
+ ", addDataTime=" + addDataTime +
+ ", evt=" + evtData + ']');
+ }
+ }
+
+ saveAndProcessNewEvents();
+ }
+
+ /**
+ *
+ */
+ private void throttleNewEventsGeneration() {
+ long delay = IgniteSystemProperties.getLong(IGNITE_ZOOKEEPER_DISCOVERY_SPI_EVTS_THROTTLE, 0);
+
+ if (delay > 0) {
+ if (log.isInfoEnabled())
+ log.info("Sleep delay before generate new events [delay=" + delay + ']');
+
+ try {
+ Thread.sleep(delay);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @param prefixId Path prefix.
+ * @return Join data.
+ * @throws Exception If failed.
+ */
+ private ZkJoiningNodeData unmarshalJoinData(UUID nodeId, UUID prefixId) throws Exception {
+ String joinDataPath = zkPaths.joiningNodeDataPath(nodeId, prefixId);
+
+ byte[] joinData = rtState.zkClient.getData(joinDataPath);
+
+ Object dataObj = unmarshalZip(joinData);
+
+ if (!(dataObj instanceof ZkJoiningNodeData))
+ throw new Exception("Invalid joined node data: " + dataObj);
+
+ ZkJoiningNodeData joiningNodeData = (ZkJoiningNodeData)dataObj;
+
+ if (joiningNodeData.partCount() > 1) {
+ joinData = readMultipleParts(rtState.zkClient, joinDataPath + ":", joiningNodeData.partCount());
+
+ joiningNodeData = unmarshalZip(joinData);
+ }
+
+ return joiningNodeData;
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @param prefixId Path prefix.
+ * @param aliveNodePath Node path.
+ * @return Join data.
+ * @throws Exception If failed.
+ */
+ private Object unmarshalJoinDataOnCoordinator(UUID nodeId, UUID prefixId, String aliveNodePath) throws Exception {
+ String joinDataPath = zkPaths.joiningNodeDataPath(nodeId, prefixId);
+
+ byte[] joinData = rtState.zkClient.getData(joinDataPath);
+
+ Object dataObj;
+
+ try {
+ dataObj = unmarshalZip(joinData);
+
+ if (dataObj instanceof ZkInternalJoinErrorMessage)
+ return dataObj;
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to unmarshal joining node data [nodePath=" + aliveNodePath + "']", e);
+
+ return new ZkInternalJoinErrorMessage(ZkIgnitePaths.aliveInternalId(aliveNodePath),
+ "Failed to unmarshal join data: " + e);
+ }
+
+ assert dataObj instanceof ZkJoiningNodeData : dataObj;
+
+ ZkJoiningNodeData joiningNodeData = (ZkJoiningNodeData)dataObj;
+
+ if (joiningNodeData.partCount() > 1) {
+ joinData = readMultipleParts(rtState.zkClient, joinDataPath + ":", joiningNodeData.partCount());
+
+ try {
+ joiningNodeData = unmarshalZip(joinData);
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to unmarshal joining node data [nodePath=" + aliveNodePath + "']", e);
+
+ return new ZkInternalJoinErrorMessage(ZkIgnitePaths.aliveInternalId(aliveNodePath),
+ "Failed to unmarshal join data: " + e);
+ }
+ }
+
+ assert joiningNodeData.node() != null : joiningNodeData;
+
+ return joiningNodeData;
+ }
+
+ /**
+ * @param joinCtx Joined nodes context.
+ * @param curTop Current nodes.
+ * @param internalId Joined node internal ID.
+ * @param aliveNodePath Joined node path.
+ * @throws Exception If failed.
+ */
+ private void processJoinOnCoordinator(
+ ZkBulkJoinContext joinCtx,
+ TreeMap<Long, ZookeeperClusterNode> curTop,
+ long internalId,
+ String aliveNodePath)
+ throws Exception
+ {
+ UUID nodeId = ZkIgnitePaths.aliveNodeId(aliveNodePath);
+ UUID prefixId = ZkIgnitePaths.aliveNodePrefixId(aliveNodePath);
+
+ Object data = unmarshalJoinDataOnCoordinator(nodeId, prefixId, aliveNodePath);
+
+ if (data instanceof ZkJoiningNodeData) {
+ ZkJoiningNodeData joiningNodeData = (ZkJoiningNodeData)data;
+
+ ZkNodeValidateResult validateRes = validateJoiningNode(joiningNodeData.node());
+
+ if (validateRes.err == null) {
+ ZookeeperClusterNode joinedNode = joiningNodeData.node();
+
+ assert nodeId.equals(joinedNode.id()) : joiningNodeData.node();
+
+ addJoinedNode(
+ joinCtx,
+ curTop,
+ joiningNodeData,
+ internalId,
+ prefixId,
+ validateRes.secSubjZipBytes);
+
+ watchAliveNodeData(aliveNodePath);
+ }
+ else {
+ ZkInternalJoinErrorMessage joinErr = new ZkInternalJoinErrorMessage(
+ ZkIgnitePaths.aliveInternalId(aliveNodePath),
+ validateRes.err);
+
+ processJoinError(aliveNodePath, nodeId, prefixId, joinErr);
+ }
+ }
+ else {
+ assert data instanceof ZkInternalJoinErrorMessage : data;
+
+ processJoinError(aliveNodePath, nodeId, prefixId, (ZkInternalJoinErrorMessage)data);
+ }
+ }
+
+ /**
+ * @param aliveNodePath Joined node path.
+ * @param nodeId Node ID.
+ * @param prefixId Path prefix ID.
+ * @param joinErr Join error message.
+ * @throws Exception If failed.
+ */
+ private void processJoinError(String aliveNodePath,
+ UUID nodeId,
+ UUID prefixId,
+ ZkInternalJoinErrorMessage joinErr) throws Exception {
+ ZookeeperClient client = rtState.zkClient;
+
+ if (joinErr.notifyNode) {
+ String joinDataPath = zkPaths.joiningNodeDataPath(nodeId, prefixId);
+
+ client.setData(joinDataPath, marshalZip(joinErr), -1);
+
+ client.deleteIfExists(zkPaths.aliveNodesDir + "/" + aliveNodePath, -1);
+ }
+ else {
+ if (log.isInfoEnabled())
+ log.info("Ignore join data, node was failed by previous coordinator: " + aliveNodePath);
+
+ client.deleteIfExists(zkPaths.aliveNodesDir + "/" + aliveNodePath, -1);
+ }
+ }
+
+ /**
+ * @param node Joining node.
+ * @return Validation result.
+ */
+ private ZkNodeValidateResult validateJoiningNode(ZookeeperClusterNode node) {
+ ZookeeperClusterNode node0 = rtState.top.nodesById.get(node.id());
+
+ if (node0 != null) {
+ U.error(log, "Failed to include node in cluster, node with the same ID already exists [joiningNode=" + node +
+ ", existingNode=" + node0 + ']');
+
+ // Note: exception message is checked in tests.
+ return new ZkNodeValidateResult("Node with the same ID already exists: " + node0);
+ }
+
+ ZkNodeValidateResult res = authenticateNode(node);
+
+ if (res.err != null)
+ return res;
+
+ IgniteNodeValidationResult err = spi.getSpiContext().validateNode(node);
+
+ if (err != null) {
+ LT.warn(log, err.message());
+
+ res.err = err.sendMessage();
+ }
+
+ return res;
+ }
+
+ /**
+ * @param node Node.
+ * @return Validation result.
+ */
+ private ZkNodeValidateResult authenticateNode(ZookeeperClusterNode node) {
+ DiscoverySpiNodeAuthenticator nodeAuth = spi.getAuthenticator();
+
+ if (nodeAuth == null)
+ return new ZkNodeValidateResult((byte[])null);
+
+ SecurityCredentials cred;
+
+ try {
+ cred = unmarshalCredentials(node);
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to unmarshal node credentials: " + e, e);
+
+ return new ZkNodeValidateResult("Failed to unmarshal node credentials");
+ }
+
+ SecurityContext subj = nodeAuth.authenticateNode(node, cred);
+
+ if (subj == null) {
+ U.warn(log, "Authentication failed [nodeId=" + node.id() +
+ ", addrs=" + U.addressesAsString(node) + ']',
+ "Authentication failed [nodeId=" + U.id8(node.id()) + ", addrs=" +
+ U.addressesAsString(node) + ']');
+
+ // Note: exception message test is checked in tests.
+ return new ZkNodeValidateResult("Authentication failed");
+ }
+
+ if (!(subj instanceof Serializable)) {
+ U.warn(log, "Authentication subject is not Serializable [nodeId=" + node.id() +
+ ", addrs=" + U.addressesAsString(node) + ']',
+ "Authentication subject is not Serializable [nodeId=" + U.id8(node.id()) +
+ ", addrs=" +
+ U.addressesAsString(node) + ']');
+
+ return new ZkNodeValidateResult("Authentication subject is not serializable");
+ }
+
+ byte[] secSubjZipBytes;
+
+ try {
+ secSubjZipBytes = marshalZip(subj);
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to marshal node security subject: " + e, e);
+
+ return new ZkNodeValidateResult("Failed to marshal node security subject");
+ }
+
+ return new ZkNodeValidateResult(secSubjZipBytes);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void saveAndProcessNewEvents() throws Exception {
+ if (stopping())
+ return;
+
+ long start = System.currentTimeMillis();
+
+ byte[] evtsBytes = marshalZip(rtState.evtsData);
+
+ rtState.zkClient.setData(zkPaths.evtsPath, evtsBytes, -1);
+
+ long time = System.currentTimeMillis() - start;
+
+ if (prevSavedEvtsTopVer != rtState.evtsData.topVer) {
+ if (log.isInfoEnabled()) {
+ log.info("Discovery coordinator saved new topology events [topVer=" + rtState.evtsData.topVer +
+ ", size=" + evtsBytes.length +
+ ", evts=" + rtState.evtsData.evts.size() +
+ ", lastEvt=" + rtState.evtsData.evtIdGen +
+ ", saveTime=" + time + ']');
+ }
+
+ prevSavedEvtsTopVer = rtState.evtsData.topVer;
+ }
+ else if (log.isDebugEnabled()) {
+ log.debug("Discovery coordinator saved new topology events [topVer=" + rtState.evtsData.topVer +
+ ", size=" + evtsBytes.length +
+ ", evts=" + rtState.evtsData.evts.size() +
+ ", lastEvt=" + rtState.evtsData.evtIdGen +
+ ", saveTime=" + time + ']');
+ }
+
+ processNewEvents(rtState.evtsData);
+ }
+
+ /**
+ * @param curTop Current topology.
+ * @param failedNode Failed node.
+ */
+ private void generateNodeFail(TreeMap<Long, ZookeeperClusterNode> curTop, ZookeeperClusterNode failedNode) {
+ Object rmvd = curTop.remove(failedNode.order());
+
+ assert rmvd != null;
+
+ rtState.evtsData.topVer++;
+ rtState.evtsData.evtIdGen++;
+
+ ZkDiscoveryNodeFailEventData evtData = new ZkDiscoveryNodeFailEventData(
+ rtState.evtsData.evtIdGen,
+ rtState.evtsData.topVer,
+ failedNode.internalId());
+
+ rtState.evtsData.addEvent(curTop.values(), evtData);
+
+ if (log.isInfoEnabled())
+ log.info("Generated NODE_FAILED event [evt=" + evtData + ']');
+ }
+
+ /**
+ * @param curTop Current nodes.
+ * @param joiningNodeData Join data.
+ * @param internalId Joined node internal ID.
+ * @param prefixId Unique path prefix.
+ * @param secSubjZipBytes Marshalled security subject.
+ * @throws Exception If failed.
+ */
+ private void addJoinedNode(
+ ZkBulkJoinContext joinCtx,
+ TreeMap<Long, ZookeeperClusterNode> curTop,
+ ZkJoiningNodeData joiningNodeData,
+ long internalId,
+ UUID prefixId,
+ @Nullable byte[] secSubjZipBytes)
+ throws Exception
+ {
+ ZookeeperClusterNode joinedNode = joiningNodeData.node();
+
+ UUID nodeId = joinedNode.id();
+
+ rtState.evtsData.topVer++;
+
+ joinedNode.order(rtState.evtsData.topVer);
+ joinedNode.internalId(internalId);
+
+ DiscoveryDataBag joiningNodeBag = new DiscoveryDataBag(nodeId, joiningNodeData.node().isClient());
+
+ joiningNodeBag.joiningNodeData(joiningNodeData.discoveryData());
+
+ exchange.onExchange(joiningNodeBag);
+
+ DiscoveryDataBag collectBag = new DiscoveryDataBag(nodeId,
+ new HashSet<Integer>(),
+ joiningNodeData.node().isClient());
+
+ collectBag.joiningNodeData(joiningNodeBag.joiningNodeData());
+
+ exchange.collect(collectBag);
+
+ Map<Integer, Serializable> commonData = collectBag.commonData();
+
+ Object old = curTop.put(joinedNode.order(), joinedNode);
+
+ assert old == null;
+
+ int overhead = 5;
+
+ int secSubjPartCnt = 0;
+
+ if (secSubjZipBytes != null) {
+ secSubjPartCnt = saveData(zkPaths.joinEventSecuritySubjectPath(joinedNode.order()),
+ secSubjZipBytes,
+ overhead);
+
+ assert secSubjPartCnt > 0 : secSubjPartCnt;
+
+ setNodeSecuritySubject(joinedNode, secSubjZipBytes);
+ }
+
+ ZkJoinedNodeEvtData nodeEvtData = new ZkJoinedNodeEvtData(
+ rtState.evtsData.topVer,
+ joinedNode.id(),
+ joinedNode.internalId(),
+ prefixId,
+ joiningNodeData.partCount(),
+ secSubjPartCnt);
+
+ nodeEvtData.joiningNodeData = joiningNodeData;
+
+ joinCtx.addJoinedNode(nodeEvtData, commonData);
+
+ rtState.evtsData.onNodeJoin(joinedNode);
+ }
+
+ /**
+ * @param path Path to save.
+ * @param bytes Bytes to save.
+ * @param overhead Extra overhead.
+ * @return Parts count.
+ * @throws Exception If failed.
+ */
+ private int saveData(String path, byte[] bytes, int overhead) throws Exception {
+ int dataForJoinedPartCnt = 1;
+
+ if (rtState.zkClient.needSplitNodeData(path, bytes, overhead)) {
+ dataForJoinedPartCnt = saveMultipleParts(rtState.zkClient,
+ path,
+ rtState.zkClient.splitNodeData(path, bytes, overhead));
+ }
+ else {
+ rtState.zkClient.createIfNeeded(multipartPathName(path, 0),
+ bytes,
+ PERSISTENT);
+ }
+
+ return dataForJoinedPartCnt;
+ }
+
+ /**
+ * @param prevEvts Events from previous cluster.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ private void newClusterStarted(@Nullable ZkDiscoveryEventsData prevEvts) throws Exception {
+ assert !locNode.isClient() : locNode;
+
+ long locInternalId = rtState.internalOrder;
+
+ assert prevEvts == null || prevEvts.maxInternalOrder < locInternalId;
+
+ spi.getSpiContext().removeTimeoutObject(rtState.joinErrTo);
+
+ cleanupPreviousClusterData(prevEvts != null ? prevEvts.maxInternalOrder + 1 : -1L);
+
+ rtState.joined = true;
+ rtState.gridStartTime = System.currentTimeMillis();
+
+ rtState.evtsData = ZkDiscoveryEventsData.createForNewCluster(rtState.gridStartTime);
+
+ if (log.isInfoEnabled()) {
+ log.info("New cluster started [locId=" + locNode.id() +
+ ", clusterId=" + rtState.evtsData.clusterId +
+ ", startTime=" + rtState.evtsData.clusterStartTime + ']');
+ }
+
+ locNode.internalId(locInternalId);
+ locNode.order(1);
+
+ rtState.evtsData.onNodeJoin(locNode);
+
+ rtState.top.addNode(locNode);
+
+ final List<ClusterNode> topSnapshot = Collections.singletonList((ClusterNode)locNode);
+
+ lsnr.onDiscovery(EVT_NODE_JOINED,
+ 1L,
+ locNode,
+ topSnapshot,
+ Collections.<Long, Collection<ClusterNode>>emptyMap(),
+ null);
+
+ // Reset events (this is also notification for clients left from previous cluster).
+ rtState.zkClient.setData(zkPaths.evtsPath, marshalZip(rtState.evtsData), -1);
+
+ joinFut.onDone();
+ }
+
+ /**
+ * @param startInternalOrder Starting internal order for cluster (znodes having lower order belong
+ * to clients from previous cluster and should be removed).
+
+ * @throws Exception If failed.
+ */
+ private void cleanupPreviousClusterData(long startInternalOrder) throws Exception {
+ long start = System.currentTimeMillis();
+
+ ZookeeperClient client = rtState.zkClient;
+
+ // TODO ZK: use multi, better batching + max-size safe + NoNodeException safe.
+ List<String> evtChildren = rtState.zkClient.getChildren(zkPaths.evtsPath);
+
+ for (String evtPath : evtChildren) {
+ String evtDir = zkPaths.evtsPath + "/" + evtPath;
+
+ removeChildren(evtDir);
+ }
+
+ client.deleteAll(zkPaths.evtsPath, evtChildren, -1);
+
+ client.deleteAll(zkPaths.customEvtsDir,
+ client.getChildren(zkPaths.customEvtsDir),
+ -1);
+
+ rtState.zkClient.deleteAll(zkPaths.customEvtsPartsDir,
+ rtState.zkClient.getChildren(zkPaths.customEvtsPartsDir),
+ -1);
+
+ rtState.zkClient.deleteAll(zkPaths.customEvtsAcksDir,
+ rtState.zkClient.getChildren(zkPaths.customEvtsAcksDir),
+ -1);
+
+ if (startInternalOrder > 0) {
+ for (String alive : rtState.zkClient.getChildren(zkPaths.aliveNodesDir)) {
+ if (ZkIgnitePaths.aliveInternalId(alive) < startInternalOrder)
+ rtState.zkClient.deleteIfExists(zkPaths.aliveNodesDir + "/" + alive, -1);
+ }
+ }
+
+ long time = System.currentTimeMillis() - start;
+
+ if (time > 0) {
+ if (log.isInfoEnabled())
+ log.info("Previous cluster data cleanup time: " + time);
+ }
+ }
+
+ /**
+ * @param path Path.
+ * @throws Exception If failed.
+ */
+ private void removeChildren(String path) throws Exception {
+ rtState.zkClient.deleteAll(path, rtState.zkClient.getChildren(path), -1);
+ }
+
+ /**
+ * @param zkClient Client.
+ * @param evtPath Event path.
+ * @param sndNodeId Sender node ID.
+ * @return Event data.
+ * @throws Exception If failed.
+ */
+ private byte[] readCustomEventData(ZookeeperClient zkClient, String evtPath, UUID sndNodeId) throws Exception {
+ int partCnt = ZkIgnitePaths.customEventPartsCount(evtPath);
+
+ if (partCnt > 1) {
+ String partsBasePath = zkPaths.customEventPartsBasePath(
+ ZkIgnitePaths.customEventPrefix(evtPath), sndNodeId);
+
+ return readMultipleParts(zkClient, partsBasePath, partCnt);
+ }
+ else
+ return zkClient.getData(zkPaths.customEvtsDir + "/" + evtPath);
+ }
+
+ /**
+ * @param customEvtNodes ZK nodes representing custom events to process.
+ * @throws Exception If failed.
+ */
+ private void generateCustomEvents(List<String> customEvtNodes) throws Exception {
+ assert rtState.crd;
+
+ ZookeeperClient zkClient = rtState.zkClient;
+ ZkDiscoveryEventsData evtsData = rtState.evtsData;
+
+ TreeMap<Integer, String> unprocessedEvts = null;
+
+ for (int i = 0; i < customEvtNodes.size(); i++) {
+ String evtPath = customEvtNodes.get(i);
+
+ int evtSeq = ZkIgnitePaths.customEventSequence(evtPath);
+
+ if (evtSeq > evtsData.procCustEvt) {
+ if (unprocessedEvts == null)
+ unprocessedEvts = new TreeMap<>();
+
+ unprocessedEvts.put(evtSeq, evtPath);
+ }
+ }
+
+ if (unprocessedEvts == null)
+ return;
+
+ for (Map.Entry<Integer, String> evtE : unprocessedEvts.entrySet()) {
+ evtsData.procCustEvt = evtE.getKey();
+
+ String evtPath = evtE.getValue();
+
+ UUID sndNodeId = ZkIgnitePaths.customEventSendNodeId(evtPath);
+
+ ZookeeperClusterNode sndNode = rtState.top.nodesById.get(sndNodeId);
+
+ if (sndNode != null) {
+ byte[] evtBytes = readCustomEventData(zkClient, evtPath, sndNodeId);
+
+ DiscoverySpiCustomMessage msg;
+
+ try {
+ msg = unmarshalZip(evtBytes);
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to unmarshal custom discovery message: " + e, e);
+
+ deleteCustomEventDataAsync(rtState.zkClient, evtPath);
+
+ continue;
+ }
+
+ generateAndProcessCustomEventOnCoordinator(evtPath, sndNode, msg);
+ }
+ else {
+ U.warn(log, "Ignore custom event from unknown node: " + sndNodeId);
+
+ deleteCustomEventDataAsync(rtState.zkClient, evtPath);
+ }
+ }
+ }
+
+ /**
+ * @param evtPath Event data path.
+ * @param sndNode Sender node.
+ * @param msg Message instance.
+ * @throws Exception If failed.
+ */
+ private void generateAndProcessCustomEventOnCoordinator(String evtPath,
+ ZookeeperClusterNode sndNode,
+ DiscoverySpiCustomMessage msg) throws Exception
+ {
+ ZookeeperClient zkClient = rtState.zkClient;
+ ZkDiscoveryEventsData evtsData = rtState.evtsData;
+
+ ZookeeperClusterNode failedNode = null;
+
+ if (msg instanceof ZkForceNodeFailMessage) {
+ ZkForceNodeFailMessage msg0 = (ZkForceNodeFailMessage)msg;
+
+ failedNode = rtState.top.nodesByInternalId.get(msg0.nodeInternalId);
+
+ if (failedNode != null)
+ evtsData.topVer++;
+ else {
+ if (log.isDebugEnabled())
+ log.debug("Ignore forcible node fail request for unknown node: " + msg0.nodeInternalId);
+
+ deleteCustomEventDataAsync(zkClient, evtPath);
+
+ return;
+ }
+ }
+ else if (msg instanceof ZkCommunicationErrorResolveStartMessage) {
+ ZkCommunicationErrorResolveStartMessage msg0 =
+ (ZkCommunicationErrorResolveStartMessage)msg;
+
+ if (evtsData.communicationErrorResolveFutureId() != null) {
+ if (log.isInfoEnabled()) {
+ log.info("Ignore communication error resolve message, resolve process " +
+ "already started [sndNode=" + sndNode + ']');
+ }
+
+ deleteCustomEventDataAsync(zkClient, evtPath);
+
+ return;
+ }
+ else {
+ if (log.isInfoEnabled()) {
+ log.info("Start cluster-wide communication error resolve [sndNode=" + sndNode +
+ ", reqId=" + msg0.id +
+ ", topVer=" + evtsData.topVer + ']');
+ }
+
+ zkClient.createIfNeeded(zkPaths.distributedFutureBasePath(msg0.id),
+ null,
+ PERSISTENT);
+
+ evtsData.communicationErrorResolveFutureId(msg0.id);
+ }
+ }
+
+ evtsData.evtIdGen++;
+
+ ZkDiscoveryCustomEventData evtData = new ZkDiscoveryCustomEventData(
+ evtsData.evtIdGen,
+ 0L,
+ evtsData.topVer,
+ sndNode.id(),
+ null,
+ evtPath);
+
+ evtData.resolvedMsg = msg;
+
+ if (log.isDebugEnabled())
+ log.debug("Generated CUSTOM event [evt=" + evtData + ", msg=" + msg + ']');
+
+ boolean fastStopProcess = false;
+
+ if (msg instanceof ZkInternalMessage)
+ processInternalMessage(evtData, (ZkInternalMessage)msg);
+ else {
+ notifyCustomEvent(evtData, msg);
+
+ if (msg.stopProcess()) {
+ if (log.isDebugEnabled())
+ log.debug("Fast stop process custom event [evt=" + evtData + ", msg=" + msg + ']');
+
+ fastStopProcess = true;
+
+ // No need to process this event on others nodes, skip this event.
+ evtsData.evts.remove(evtData.eventId());
+
+ evtsData.evtIdGen--;
+
+ DiscoverySpiCustomMessage ack = msg.ackMessage();
+
+ if (ack != null) {
+ evtData = createAckEvent(ack, evtData);
+
+ if (log.isDebugEnabled())
+ log.debug("Generated CUSTOM event (ack for fast stop process) [evt=" + evtData + ", msg=" + msg + ']');
+
+ notifyCustomEvent(evtData, ack);
+ }
+ else
+ evtData = null;
+ }
+ }
+
+ if (evtData != null) {
+ evtsData.addEvent(rtState.top.nodesByOrder.values(), evtData);
+
+ rtState.locNodeInfo.lastProcEvt = evtData.eventId();
+
+ saveAndProcessNewEvents();
+
+ if (fastStopProcess)
+ deleteCustomEventDataAsync(zkClient, evtPath);
+
+ if (failedNode != null) {
+ deleteAliveNode(failedNode.internalId());
+
+ handleProcessedEventsOnNodesFail(Collections.singletonList(failedNode));
+
+ rtState.updateAlives = true;
+ }
+ }
+ }
+
+ /**
+ * @param internalId Node internal ID.
+ * @throws Exception If failed.
+ */
+ private void deleteAliveNode(long internalId) throws Exception {
+ for (String child : rtState.zkClient.getChildren(zkPaths.aliveNodesDir)) {
+ if (ZkIgnitePaths.aliveInternalId(child) == internalId) {
+ // Need use sync delete to do not process again join of this node again.
+ rtState.zkClient.deleteIfExists(zkPaths.aliveNodesDir + "/" + child, -1);
+
+ return;
+ }
+ }
+ }
+
+ /**
+ * @param zkClient Client.
+ * @param evtPath Event path.
+ */
+ private void deleteCustomEventDataAsync(ZookeeperClient zkClient, String evtPath) {
+ if (log.isDebugEnabled())
+ log.debug("Delete custom event data: " + evtPath);
+
+ String prefix = ZkIgnitePaths.customEventPrefix(evtPath);
+ UUID sndNodeId = ZkIgnitePaths.customEventSendNodeId(evtPath);
+ int partCnt = ZkIgnitePaths.customEventPartsCount(evtPath);
+
+ assert partCnt >= 1 : partCnt;
+
+ if (partCnt > 1) {
+ for (int i = 0; i < partCnt; i++) {
+ String path0 = zkPaths.customEventPartPath(prefix, sndNodeId, i);
+
+ zkClient.deleteIfExistsAsync(path0);
+ }
+ }
+
+ zkClient.deleteIfExistsAsync(zkPaths.customEvtsDir + "/" + evtPath);
+ }
+
+ /**
+ * @param data Marshalled events.
+ * @throws Exception If failed.
+ * @return Events.
+ */
+ @Nullable private ZkDiscoveryEventsData processNewEvents(byte[] data) throws Exception {
+ ZkDiscoveryEventsData newEvts = data.length > 0 ? (ZkDiscoveryEventsData)unmarshalZip(data) : null;
+
+ if (rtState.joined && (newEvts == null || !rtState.evtsData.clusterId.equals(newEvts.clusterId))) {
+ assert locNode.isClient() : locNode;
+
+ throw localNodeFail("All server nodes failed, client node disconnected (received events from new custer) " +
+ "[locId=" + locNode.id() + ']', true);
+ }
+
+ if (newEvts == null)
+ return null;
+
+ assert !rtState.crd;
+
+ // Need keep processed custom events since they contain message object which is needed to create ack.
+ if (!locNode.isClient() && rtState.evtsData != null) {
+ for (Map.Entry<Long, ZkDiscoveryEventData> e : rtState.evtsData.evts.entrySet()) {
+ ZkDiscoveryEventData evtData = e.getValue();
+
+ if (evtData.eventType() == ZkDiscoveryEventData.ZK_EVT_CUSTOM_EVT) {
+ ZkDiscoveryCustomEventData evtData0 =
+ (ZkDiscoveryCustomEventData)newEvts.evts.get(evtData.eventId());
+
+ if (evtData0 != null)
+ evtData0.resolvedMsg = ((ZkDiscoveryCustomEventData)evtData).resolvedMsg;
+ }
+ }
+ }
+
+ processNewEvents(newEvts);
+
+ if (rtState.joined)
+ rtState.evtsData = newEvts;
+
+ return newEvts;
+ }
+
+ /**
+ * @param evtsData Ev
<TRUNCATED>
[33/54] [abbrv] ignite git commit: IGNITE-7871 Check local join
future on error. - Fixes #3793.
Posted by ag...@apache.org.
IGNITE-7871 Check local join future on error. - Fixes #3793.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0e73fa2c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0e73fa2c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0e73fa2c
Branch: refs/heads/ignite-6083
Commit: 0e73fa2c10dcd96ff98279018bdd3f8b36568008
Parents: 74d2545
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Wed Apr 11 14:12:50 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Wed Apr 11 14:12:50 2018 +0300
----------------------------------------------------------------------
.../distributed/dht/preloader/latch/ExchangeLatchManager.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0e73fa2c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
index c205cb1..404f88f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -104,7 +104,8 @@ public class ExchangeLatchManager {
// First coordinator initialization.
ctx.discovery().localJoinFuture().listen(f -> {
- this.coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
+ if (f.error() == null)
+ this.coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
});
ctx.event().addDiscoveryEventListener((e, cache) -> {
[15/54] [abbrv] ignite git commit: IGNITE-8059: Integrate decision
tree with partition based dataset.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/VarianceSplitCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/VarianceSplitCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/VarianceSplitCalculator.java
deleted file mode 100644
index 66c54f2..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/VarianceSplitCalculator.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.PrimitiveIterator;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.ContinuousSplitInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-
-/**
- * Calculator of variance in a given region.
- */
-public class VarianceSplitCalculator implements ContinuousSplitCalculator<VarianceSplitCalculator.VarianceData> {
- /**
- * Data used in variance calculations.
- */
- public static class VarianceData extends ContinuousRegionInfo {
- /** Mean value in a given region. */
- double mean;
-
- /**
- * @param var Variance in this region.
- * @param size Size of data for which variance is calculated.
- * @param mean Mean value in this region.
- */
- public VarianceData(double var, int size, double mean) {
- super(var, size);
- this.mean = mean;
- }
-
- /**
- * No-op constructor. For serialization/deserialization.
- */
- public VarianceData() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- super.writeExternal(out);
- out.writeDouble(mean);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- super.readExternal(in);
- mean = in.readDouble();
- }
-
- /**
- * Returns mean.
- */
- public double mean() {
- return mean;
- }
- }
-
- /** {@inheritDoc} */
- @Override public VarianceData calculateRegionInfo(DoubleStream s, int size) {
- PrimitiveIterator.OfDouble itr = s.iterator();
- int i = 0;
-
- double mean = 0.0;
- double m2 = 0.0;
-
- // Here we calculate variance and mean by incremental computation.
- while (itr.hasNext()) {
- i++;
- double x = itr.next();
- double delta = x - mean;
- mean += delta / i;
- double delta2 = x - mean;
- m2 += delta * delta2;
- }
-
- return new VarianceData(m2 / i, size, mean);
- }
-
- /** {@inheritDoc} */
- @Override public SplitInfo<VarianceData> splitRegion(Integer[] s, double[] values, double[] labels, int regionIdx,
- VarianceData d) {
- int size = d.getSize();
-
- double lm2 = 0.0;
- double rm2 = d.impurity() * size;
- int lSize = size;
-
- double lMean = 0.0;
- double rMean = d.mean;
-
- double minImpurity = d.impurity() * size;
- double curThreshold;
- double curImpurity;
- double threshold = Double.NEGATIVE_INFINITY;
-
- int i = 0;
- int nextIdx = s[0];
- i++;
- double[] lrImps = new double[] {lm2, rm2, lMean, rMean};
-
- do {
- // Process all values equal to prev.
- while (i < s.length) {
- moveLeft(labels[nextIdx], lrImps[2], i, lrImps[0], lrImps[3], size - i, lrImps[1], lrImps);
- curImpurity = (lrImps[0] + lrImps[1]);
- curThreshold = values[nextIdx];
-
- if (values[nextIdx] != values[(nextIdx = s[i++])]) {
- if (curImpurity < minImpurity) {
- lSize = i - 1;
-
- lm2 = lrImps[0];
- rm2 = lrImps[1];
-
- lMean = lrImps[2];
- rMean = lrImps[3];
-
- minImpurity = curImpurity;
- threshold = curThreshold;
- }
-
- break;
- }
- }
- }
- while (i < s.length - 1);
-
- if (lSize == size)
- return null;
-
- VarianceData lData = new VarianceData(lm2 / (lSize != 0 ? lSize : 1), lSize, lMean);
- int rSize = size - lSize;
- VarianceData rData = new VarianceData(rm2 / (rSize != 0 ? rSize : 1), rSize, rMean);
-
- return new ContinuousSplitInfo<>(regionIdx, threshold, lData, rData);
- }
-
- /**
- * Add point to the left interval and remove it from the right interval and calculate necessary statistics on
- * intervals with new bounds.
- */
- private void moveLeft(double x, double lMean, int lSize, double lm2, double rMean, int rSize, double rm2,
- double[] data) {
- // We add point to the left interval.
- double lDelta = x - lMean;
- double lMeanNew = lMean + lDelta / lSize;
- double lm2New = lm2 + lDelta * (x - lMeanNew);
-
- // We remove point from the right interval. lSize + 1 is the size of right interval before removal.
- double rMeanNew = (rMean * (rSize + 1) - x) / rSize;
- double rm2New = rm2 - (x - rMean) * (x - rMeanNew);
-
- data[0] = lm2New;
- data[1] = rm2New;
-
- data[2] = lMeanNew;
- data[3] = rMeanNew;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/package-info.java
deleted file mode 100644
index 08c8a75..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Calculators of splits by continuous features.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/package-info.java
deleted file mode 100644
index 8523914..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains column based decision tree algorithms.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/RegionCalculators.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/RegionCalculators.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/RegionCalculators.java
deleted file mode 100644
index 5c4b354..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/RegionCalculators.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.regcalcs;
-
-import it.unimi.dsi.fastutil.doubles.Double2IntOpenHashMap;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.PrimitiveIterator;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-
-/** Some commonly used functions for calculations of regions of space which correspond to decision tree leaf nodes. */
-public class RegionCalculators {
- /** Mean value in the region. */
- public static final IgniteFunction<DoubleStream, Double> MEAN = s -> s.average().orElse(0.0);
-
- /** Most common value in the region. */
- public static final IgniteFunction<DoubleStream, Double> MOST_COMMON =
- s -> {
- PrimitiveIterator.OfDouble itr = s.iterator();
- Map<Double, Integer> voc = new HashMap<>();
-
- while (itr.hasNext())
- voc.compute(itr.next(), (d, i) -> i != null ? i + 1 : 0);
-
- return voc.entrySet().stream().max(Comparator.comparing(Map.Entry::getValue)).map(Map.Entry::getKey).orElse(0.0);
- };
-
- /** Variance of a region. */
- public static final IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> VARIANCE = input ->
- s -> {
- PrimitiveIterator.OfDouble itr = s.iterator();
- int i = 0;
-
- double mean = 0.0;
- double m2 = 0.0;
-
- while (itr.hasNext()) {
- i++;
- double x = itr.next();
- double delta = x - mean;
- mean += delta / i;
- double delta2 = x - mean;
- m2 += delta * delta2;
- }
-
- return i > 0 ? m2 / i : 0.0;
- };
-
- /** Gini impurity of a region. */
- public static final IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> GINI = input ->
- s -> {
- PrimitiveIterator.OfDouble itr = s.iterator();
-
- Double2IntOpenHashMap m = new Double2IntOpenHashMap();
-
- int size = 0;
-
- while (itr.hasNext()) {
- size++;
- m.compute(itr.next(), (a, i) -> i != null ? i + 1 : 1);
- }
-
- double c2 = m.values().stream().mapToDouble(v -> v * v).sum();
-
- return size != 0 ? 1 - c2 / (size * size) : 0.0;
- };
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/package-info.java
deleted file mode 100644
index e8edd8f..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Region calculators.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased.regcalcs;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/CategoricalFeatureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/CategoricalFeatureProcessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/CategoricalFeatureProcessor.java
deleted file mode 100644
index 3232ac2..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/CategoricalFeatureProcessor.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.CategoricalRegionInfo;
-import org.apache.ignite.ml.trees.CategoricalSplitInfo;
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.RegionProjection;
-
-import static org.apache.ignite.ml.trees.trainers.columnbased.vectors.FeatureVectorProcessorUtils.splitByBitSet;
-
-/**
- * Categorical feature vector processor implementation used by {@link ColumnDecisionTreeTrainer}.
- */
-public class CategoricalFeatureProcessor
- implements FeatureProcessor<CategoricalRegionInfo, CategoricalSplitInfo<CategoricalRegionInfo>> {
- /** Count of categories for this feature. */
- private final int catsCnt;
-
- /** Function for calculating impurity of a given region of points. */
- private final IgniteFunction<DoubleStream, Double> calc;
-
- /**
- * @param calc Function for calculating impurity of a given region of points.
- * @param catsCnt Number of categories.
- */
- public CategoricalFeatureProcessor(IgniteFunction<DoubleStream, Double> calc, int catsCnt) {
- this.calc = calc;
- this.catsCnt = catsCnt;
- }
-
- /** */
- private SplitInfo<CategoricalRegionInfo> split(BitSet leftCats, int intervalIdx, Map<Integer, Integer> mapping,
- Integer[] sampleIndexes, double[] values, double[] labels, double impurity) {
- Map<Boolean, List<Integer>> leftRight = Arrays.stream(sampleIndexes).
- collect(Collectors.partitioningBy((smpl) -> leftCats.get(mapping.get((int)values[smpl]))));
-
- List<Integer> left = leftRight.get(true);
- int leftSize = left.size();
- double leftImpurity = calc.apply(left.stream().mapToDouble(s -> labels[s]));
-
- List<Integer> right = leftRight.get(false);
- int rightSize = right.size();
- double rightImpurity = calc.apply(right.stream().mapToDouble(s -> labels[s]));
-
- int totalSize = leftSize + rightSize;
-
- // Result of this call will be sent back to trainer node, we do not need vectors inside of sent data.
- CategoricalSplitInfo<CategoricalRegionInfo> res = new CategoricalSplitInfo<>(intervalIdx,
- new CategoricalRegionInfo(leftImpurity, null), // cats can be computed on the last step.
- new CategoricalRegionInfo(rightImpurity, null),
- leftCats);
-
- res.setInfoGain(impurity - (double)leftSize / totalSize * leftImpurity - (double)rightSize / totalSize * rightImpurity);
- return res;
- }
-
- /**
- * Get a stream of subsets given categories count.
- *
- * @param catsCnt categories count.
- * @return Stream of subsets given categories count.
- */
- private Stream<BitSet> powerSet(int catsCnt) {
- Iterable<BitSet> iterable = () -> new PSI(catsCnt);
- return StreamSupport.stream(iterable.spliterator(), false);
- }
-
- /** {@inheritDoc} */
- @Override public SplitInfo findBestSplit(RegionProjection<CategoricalRegionInfo> regionPrj, double[] values,
- double[] labels, int regIdx) {
- Map<Integer, Integer> mapping = mapping(regionPrj.data().cats());
-
- return powerSet(regionPrj.data().cats().length()).
- map(s -> split(s, regIdx, mapping, regionPrj.sampleIndexes(), values, labels, regionPrj.data().impurity())).
- max(Comparator.comparingDouble(SplitInfo::infoGain)).
- orElse(null);
- }
-
- /** {@inheritDoc} */
- @Override public RegionProjection<CategoricalRegionInfo> createInitialRegion(Integer[] sampleIndexes,
- double[] values, double[] labels) {
- BitSet set = new BitSet();
- set.set(0, catsCnt);
-
- Double impurity = calc.apply(Arrays.stream(labels));
-
- return new RegionProjection<>(sampleIndexes, new CategoricalRegionInfo(impurity, set), 0);
- }
-
- /** {@inheritDoc} */
- @Override public SparseBitSet calculateOwnershipBitSet(RegionProjection<CategoricalRegionInfo> regionPrj,
- double[] values,
- CategoricalSplitInfo<CategoricalRegionInfo> s) {
- SparseBitSet res = new SparseBitSet();
- Arrays.stream(regionPrj.sampleIndexes()).forEach(smpl -> res.set(smpl, s.bitSet().get((int)values[smpl])));
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public IgniteBiTuple<RegionProjection, RegionProjection> performSplit(SparseBitSet bs,
- RegionProjection<CategoricalRegionInfo> reg, CategoricalRegionInfo leftData, CategoricalRegionInfo rightData) {
- return performSplitGeneric(bs, null, reg, leftData, rightData);
- }
-
- /** {@inheritDoc} */
- @Override public IgniteBiTuple<RegionProjection, RegionProjection> performSplitGeneric(
- SparseBitSet bs, double[] values, RegionProjection<CategoricalRegionInfo> reg, RegionInfo leftData,
- RegionInfo rightData) {
- int depth = reg.depth();
-
- int lSize = bs.cardinality();
- int rSize = reg.sampleIndexes().length - lSize;
- IgniteBiTuple<Integer[], Integer[]> lrSamples = splitByBitSet(lSize, rSize, reg.sampleIndexes(), bs);
- BitSet leftCats = calculateCats(lrSamples.get1(), values);
- CategoricalRegionInfo lInfo = new CategoricalRegionInfo(leftData.impurity(), leftCats);
-
- // TODO: IGNITE-5892 Check how it will work with sparse data.
- BitSet rightCats = calculateCats(lrSamples.get2(), values);
- CategoricalRegionInfo rInfo = new CategoricalRegionInfo(rightData.impurity(), rightCats);
-
- RegionProjection<CategoricalRegionInfo> rPrj = new RegionProjection<>(lrSamples.get2(), rInfo, depth + 1);
- RegionProjection<CategoricalRegionInfo> lPrj = new RegionProjection<>(lrSamples.get1(), lInfo, depth + 1);
- return new IgniteBiTuple<>(lPrj, rPrj);
- }
-
- /**
- * Powerset iterator. Iterates not over the whole powerset, but on half of it.
- */
- private static class PSI implements Iterator<BitSet> {
-
- /** Current subset number. */
- private int i = 1; // We are not interested in {emptyset, set} split and therefore start from 1.
-
- /** Size of set, subsets of which we iterate over. */
- final int size;
-
- /**
- * @param bitCnt Size of set, subsets of which we iterate over.
- */
- PSI(int bitCnt) {
- this.size = 1 << (bitCnt - 1);
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasNext() {
- return i < size;
- }
-
- /** {@inheritDoc} */
- @Override public BitSet next() {
- BitSet res = BitSet.valueOf(new long[] {i});
- i++;
- return res;
- }
- }
-
- /** */
- private Map<Integer, Integer> mapping(BitSet bs) {
- int bn = 0;
- Map<Integer, Integer> res = new HashMap<>();
-
- int i = 0;
- while ((bn = bs.nextSetBit(bn)) != -1) {
- res.put(bn, i);
- i++;
- bn++;
- }
-
- return res;
- }
-
- /** Get set of categories of given samples */
- private BitSet calculateCats(Integer[] sampleIndexes, double[] values) {
- BitSet res = new BitSet();
-
- for (int smpl : sampleIndexes)
- res.set((int)values[smpl]);
-
- return res;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousFeatureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousFeatureProcessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousFeatureProcessor.java
deleted file mode 100644
index 4117993..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousFeatureProcessor.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import java.util.Arrays;
-import java.util.Comparator;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.RegionProjection;
-
-import static org.apache.ignite.ml.trees.trainers.columnbased.vectors.FeatureVectorProcessorUtils.splitByBitSet;
-
-/**
- * Container of projection of samples on continuous feature.
- *
- * @param <D> Information about regions. Designed to contain information which will make computations of impurity
- * optimal.
- */
-public class ContinuousFeatureProcessor<D extends ContinuousRegionInfo> implements
- FeatureProcessor<D, ContinuousSplitInfo<D>> {
- /** ContinuousSplitCalculator used for calculating of best split of each region. */
- private final ContinuousSplitCalculator<D> calc;
-
- /**
- * @param splitCalc Calculator used for calculating splits.
- */
- public ContinuousFeatureProcessor(ContinuousSplitCalculator<D> splitCalc) {
- this.calc = splitCalc;
- }
-
- /** {@inheritDoc} */
- @Override public SplitInfo<D> findBestSplit(RegionProjection<D> ri, double[] values, double[] labels, int regIdx) {
- SplitInfo<D> res = calc.splitRegion(ri.sampleIndexes(), values, labels, regIdx, ri.data());
-
- if (res == null)
- return null;
-
- double lWeight = (double)res.leftData.getSize() / ri.sampleIndexes().length;
- double rWeight = (double)res.rightData.getSize() / ri.sampleIndexes().length;
-
- double infoGain = ri.data().impurity() - lWeight * res.leftData().impurity() - rWeight * res.rightData().impurity();
- res.setInfoGain(infoGain);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public RegionProjection<D> createInitialRegion(Integer[] samples, double[] values, double[] labels) {
- Arrays.sort(samples, Comparator.comparingDouble(s -> values[s]));
- return new RegionProjection<>(samples, calc.calculateRegionInfo(Arrays.stream(labels), samples.length), 0);
- }
-
- /** {@inheritDoc} */
- @Override public SparseBitSet calculateOwnershipBitSet(RegionProjection<D> reg, double[] values,
- ContinuousSplitInfo<D> s) {
- SparseBitSet res = new SparseBitSet();
-
- for (int i = 0; i < s.leftData().getSize(); i++)
- res.set(reg.sampleIndexes()[i]);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public IgniteBiTuple<RegionProjection, RegionProjection> performSplit(SparseBitSet bs,
- RegionProjection<D> reg, D leftData, D rightData) {
- int lSize = leftData.getSize();
- int rSize = rightData.getSize();
- int depth = reg.depth();
-
- IgniteBiTuple<Integer[], Integer[]> lrSamples = splitByBitSet(lSize, rSize, reg.sampleIndexes(), bs);
-
- RegionProjection<D> left = new RegionProjection<>(lrSamples.get1(), leftData, depth + 1);
- RegionProjection<D> right = new RegionProjection<>(lrSamples.get2(), rightData, depth + 1);
-
- return new IgniteBiTuple<>(left, right);
- }
-
- /** {@inheritDoc} */
- @Override public IgniteBiTuple<RegionProjection, RegionProjection> performSplitGeneric(SparseBitSet bs,
- double[] labels, RegionProjection<D> reg, RegionInfo leftData, RegionInfo rightData) {
- int lSize = bs.cardinality();
- int rSize = reg.sampleIndexes().length - lSize;
- int depth = reg.depth();
-
- IgniteBiTuple<Integer[], Integer[]> lrSamples = splitByBitSet(lSize, rSize, reg.sampleIndexes(), bs);
-
- D ld = calc.calculateRegionInfo(Arrays.stream(lrSamples.get1()).mapToDouble(s -> labels[s]), lSize);
- D rd = calc.calculateRegionInfo(Arrays.stream(lrSamples.get2()).mapToDouble(s -> labels[s]), rSize);
-
- return new IgniteBiTuple<>(new RegionProjection<>(lrSamples.get1(), ld, depth + 1), new RegionProjection<>(lrSamples.get2(), rd, depth + 1));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousSplitInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousSplitInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousSplitInfo.java
deleted file mode 100644
index 8b45cb5..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousSplitInfo.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.nodes.ContinuousSplitNode;
-import org.apache.ignite.ml.trees.nodes.SplitNode;
-
-/**
- * Information about split of continuous region.
- *
- * @param <D> Class encapsulating information about the region.
- */
-public class ContinuousSplitInfo<D extends RegionInfo> extends SplitInfo<D> {
- /**
- * Threshold used for split.
- * Samples with values less or equal than this go to left region, others go to the right region.
- */
- private final double threshold;
-
- /**
- * @param regionIdx Index of region being split.
- * @param threshold Threshold used for split. Samples with values less or equal than this go to left region, others
- * go to the right region.
- * @param leftData Information about left subregion.
- * @param rightData Information about right subregion.
- */
- public ContinuousSplitInfo(int regionIdx, double threshold, D leftData, D rightData) {
- super(regionIdx, leftData, rightData);
- this.threshold = threshold;
- }
-
- /** {@inheritDoc} */
- @Override public SplitNode createSplitNode(int featureIdx) {
- return new ContinuousSplitNode(threshold, featureIdx);
- }
-
- /**
- * Threshold used for splits.
- * Samples with values less or equal than this go to left region, others go to the right region.
- */
- public double threshold() {
- return threshold;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "ContinuousSplitInfo [" +
- "threshold=" + threshold +
- ", infoGain=" + infoGain +
- ", regionIdx=" + regionIdx +
- ", leftData=" + leftData +
- ", rightData=" + rightData +
- ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureProcessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureProcessor.java
deleted file mode 100644
index 56508e5..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureProcessor.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.RegionProjection;
-
-/**
- * Base interface for feature processors used in {@link ColumnDecisionTreeTrainer}
- *
- * @param <D> Class representing data of regions resulted from split.
- * @param <S> Class representing data of split.
- */
-public interface FeatureProcessor<D extends RegionInfo, S extends SplitInfo<D>> {
- /**
- * Finds best split by this feature among all splits of all regions.
- *
- * @return best split by this feature among all splits of all regions.
- */
- SplitInfo findBestSplit(RegionProjection<D> regionPrj, double[] values, double[] labels, int regIdx);
-
- /**
- * Creates initial region from samples.
- *
- * @param samples samples.
- * @return region.
- */
- RegionProjection<D> createInitialRegion(Integer[] samples, double[] values, double[] labels);
-
- /**
- * Calculates the bitset mapping each data point to left (corresponding bit is set) or right subregion.
- *
- * @param s data used for calculating the split.
- * @return Bitset mapping each data point to left (corresponding bit is set) or right subregion.
- */
- SparseBitSet calculateOwnershipBitSet(RegionProjection<D> regionPrj, double[] values, S s);
-
- /**
- * Splits given region using bitset which maps data point to left or right subregion.
- * This method is present for the vectors of the same type to be able to pass between them information about regions
- * and therefore used iff the optimal split is received on feature of the same type.
- *
- * @param bs Bitset which maps data point to left or right subregion.
- * @param leftData Data of the left subregion.
- * @param rightData Data of the right subregion.
- * @return This feature vector.
- */
- IgniteBiTuple<RegionProjection, RegionProjection> performSplit(SparseBitSet bs, RegionProjection<D> reg, D leftData,
- D rightData);
-
- /**
- * Splits given region using bitset which maps data point to left or right subregion. This method is used iff the
- * optimal split is received on feature of different type, therefore information about regions is limited to the
- * {@link RegionInfo} class which is base for all classes used to represent region data.
- *
- * @param bs Bitset which maps data point to left or right subregion.
- * @param leftData Data of the left subregion.
- * @param rightData Data of the right subregion.
- * @return This feature vector.
- */
- IgniteBiTuple<RegionProjection, RegionProjection> performSplitGeneric(SparseBitSet bs, double[] values,
- RegionProjection<D> reg, RegionInfo leftData,
- RegionInfo rightData);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureVectorProcessorUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureVectorProcessorUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureVectorProcessorUtils.java
deleted file mode 100644
index 69ff019..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureVectorProcessorUtils.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import org.apache.ignite.lang.IgniteBiTuple;
-
-/** Utility class for feature vector processors. */
-public class FeatureVectorProcessorUtils {
- /**
- * Split target array into two (left and right) arrays by bitset.
- *
- * @param lSize Left array size;
- * @param rSize Right array size.
- * @param samples Arrays to split size.
- * @param bs Bitset specifying split.
- * @return BiTuple containing result of split.
- */
- public static IgniteBiTuple<Integer[], Integer[]> splitByBitSet(int lSize, int rSize, Integer[] samples,
- SparseBitSet bs) {
- Integer[] lArr = new Integer[lSize];
- Integer[] rArr = new Integer[rSize];
-
- int lc = 0;
- int rc = 0;
-
- for (int i = 0; i < lSize + rSize; i++) {
- int si = samples[i];
-
- if (bs.get(si)) {
- lArr[lc] = si;
- lc++;
- }
- else {
- rArr[rc] = si;
- rc++;
- }
- }
-
- return new IgniteBiTuple<>(lArr, rArr);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SampleInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SampleInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SampleInfo.java
deleted file mode 100644
index 8aa4f79..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SampleInfo.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-
-/**
- * Information about given sample within given fixed feature.
- */
-public class SampleInfo implements Externalizable {
- /** Value of projection of this sample on given fixed feature. */
- private double val;
-
- /** Sample index. */
- private int sampleIdx;
-
- /**
- * @param val Value of projection of this sample on given fixed feature.
- * @param sampleIdx Sample index.
- */
- public SampleInfo(double val, int sampleIdx) {
- this.val = val;
- this.sampleIdx = sampleIdx;
- }
-
- /**
- * No-op constructor used for serialization/deserialization.
- */
- public SampleInfo() {
- // No-op.
- }
-
- /**
- * Get the value of projection of this sample on given fixed feature.
- *
- * @return Value of projection of this sample on given fixed feature.
- */
- public double val() {
- return val;
- }
-
- /**
- * Get the sample index.
- *
- * @return Sample index.
- */
- public int sampleInd() {
- return sampleIdx;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeDouble(val);
- out.writeInt(sampleIdx);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- val = in.readDouble();
- sampleIdx = in.readInt();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SplitInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SplitInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SplitInfo.java
deleted file mode 100644
index 124e82f..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SplitInfo.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.nodes.SplitNode;
-
-/**
- * Class encapsulating information about the split.
- *
- * @param <D> Class representing information of left and right subregions.
- */
-public abstract class SplitInfo<D extends RegionInfo> {
- /** Information gain of this split. */
- protected double infoGain;
-
- /** Index of the region to split. */
- protected final int regionIdx;
-
- /** Data of left subregion. */
- protected final D leftData;
-
- /** Data of right subregion. */
- protected final D rightData;
-
- /**
- * Construct the split info.
- *
- * @param regionIdx Index of the region to split.
- * @param leftData Data of left subregion.
- * @param rightData Data of right subregion.
- */
- public SplitInfo(int regionIdx, D leftData, D rightData) {
- this.regionIdx = regionIdx;
- this.leftData = leftData;
- this.rightData = rightData;
- }
-
- /**
- * Index of region to split.
- *
- * @return Index of region to split.
- */
- public int regionIndex() {
- return regionIdx;
- }
-
- /**
- * Information gain of the split.
- *
- * @return Information gain of the split.
- */
- public double infoGain() {
- return infoGain;
- }
-
- /**
- * Data of right subregion.
- *
- * @return Data of right subregion.
- */
- public D rightData() {
- return rightData;
- }
-
- /**
- * Data of left subregion.
- *
- * @return Data of left subregion.
- */
- public D leftData() {
- return leftData;
- }
-
- /**
- * Create SplitNode from this split info.
- *
- * @param featureIdx Index of feature by which goes split.
- * @return SplitNode from this split info.
- */
- public abstract SplitNode createSplitNode(int featureIdx);
-
- /**
- * Set information gain.
- *
- * @param infoGain Information gain.
- */
- public void setInfoGain(double infoGain) {
- this.infoGain = infoGain;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/package-info.java
deleted file mode 100644
index 0dea204..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains feature containers needed by column based decision tree trainers.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
index e22a3a5..9900f85 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
@@ -28,7 +28,7 @@ import org.apache.ignite.ml.preprocessing.PreprocessingTestSuite;
import org.apache.ignite.ml.regressions.RegressionsTestSuite;
import org.apache.ignite.ml.svm.SVMTestSuite;
import org.apache.ignite.ml.trainers.group.TrainersGroupTestSuite;
-import org.apache.ignite.ml.trees.DecisionTreesTestSuite;
+import org.apache.ignite.ml.tree.DecisionTreeTestSuite;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -41,7 +41,7 @@ import org.junit.runners.Suite;
RegressionsTestSuite.class,
SVMTestSuite.class,
ClusteringTestSuite.class,
- DecisionTreesTestSuite.class,
+ DecisionTreeTestSuite.class,
KNNTestSuite.class,
LocalModelsTest.class,
MLPTestSuite.class,
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java
index e624004..d68b355 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java
@@ -25,11 +25,10 @@ import java.util.Random;
import java.util.stream.Stream;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.performance.ColumnDecisionTreeTrainerBenchmark;
import org.apache.ignite.ml.util.MnistUtils;
/** */
-class MnistMLPTestUtil {
+public class MnistMLPTestUtil {
/** Name of the property specifying path to training set images. */
private static final String PROP_TRAINING_IMAGES = "mnist.training.images";
@@ -62,7 +61,7 @@ class MnistMLPTestUtil {
* @return List of MNIST images.
* @throws IOException In case of exception.
*/
- static List<MnistUtils.MnistLabeledImage> loadTrainingSet(int cnt) throws IOException {
+ public static List<MnistUtils.MnistLabeledImage> loadTrainingSet(int cnt) throws IOException {
Properties props = loadMNISTProperties();
return MnistUtils.mnistAsList(props.getProperty(PROP_TRAINING_IMAGES), props.getProperty(PROP_TRAINING_LABELS), new Random(123L), cnt);
}
@@ -74,7 +73,7 @@ class MnistMLPTestUtil {
* @return List of MNIST images.
* @throws IOException In case of exception.
*/
- static List<MnistUtils.MnistLabeledImage> loadTestSet(int cnt) throws IOException {
+ public static List<MnistUtils.MnistLabeledImage> loadTestSet(int cnt) throws IOException {
Properties props = loadMNISTProperties();
return MnistUtils.mnistAsList(props.getProperty(PROP_TEST_IMAGES), props.getProperty(PROP_TEST_LABELS), new Random(123L), cnt);
}
@@ -83,7 +82,7 @@ class MnistMLPTestUtil {
private static Properties loadMNISTProperties() throws IOException {
Properties res = new Properties();
- InputStream is = ColumnDecisionTreeTrainerBenchmark.class.getClassLoader().getResourceAsStream("manualrun/trees/columntrees.manualrun.properties");
+ InputStream is = MnistMLPTestUtil.class.getClassLoader().getResourceAsStream("manualrun/trees/columntrees.manualrun.properties");
res.load(is);
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
new file mode 100644
index 0000000..94bca3f
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.Arrays;
+import java.util.Random;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for {@link DecisionTreeClassificationTrainer} that require to start the whole Ignite infrastructure.
+ */
+public class DecisionTreeClassificationTrainerIntegrationTest extends GridCommonAbstractTest {
+ /** Number of nodes in grid */
+ private static final int NODE_COUNT = 3;
+
+ /** Ignite instance. */
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ for (int i = 1; i <= NODE_COUNT; i++)
+ startGrid(i);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() {
+ stopAllGrids();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void beforeTest() throws Exception {
+ /* Grid instance. */
+ ignite = grid(NODE_COUNT);
+ ignite.configuration().setPeerClassLoadingEnabled(true);
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+ }
+
+ /** */
+ public void testFit() {
+ int size = 100;
+
+ CacheConfiguration<Integer, double[]> trainingSetCacheCfg = new CacheConfiguration<>();
+ trainingSetCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+ trainingSetCacheCfg.setName("TRAINING_SET");
+
+ IgniteCache<Integer, double[]> data = ignite.createCache(trainingSetCacheCfg);
+
+ Random rnd = new Random(0);
+ for (int i = 0; i < size; i++) {
+ double x = rnd.nextDouble() - 0.5;
+ data.put(i, new double[]{x, x > 0 ? 1 : 0});
+ }
+
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(1, 0);
+
+ DecisionTreeNode tree = trainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, data),
+ (k, v) -> Arrays.copyOf(v, v.length - 1),
+ (k, v) -> v[v.length - 1]
+ );
+
+ assertTrue(tree instanceof DecisionTreeConditionalNode);
+
+ DecisionTreeConditionalNode node = (DecisionTreeConditionalNode) tree;
+
+ assertEquals(0, node.getThreshold(), 1e-3);
+
+ assertTrue(node.getThenNode() instanceof DecisionTreeLeafNode);
+ assertTrue(node.getElseNode() instanceof DecisionTreeLeafNode);
+
+ DecisionTreeLeafNode thenNode = (DecisionTreeLeafNode) node.getThenNode();
+ DecisionTreeLeafNode elseNode = (DecisionTreeLeafNode) node.getElseNode();
+
+ assertEquals(1, thenNode.getVal(), 1e-10);
+ assertEquals(0, elseNode.getVal(), 1e-10);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
new file mode 100644
index 0000000..2599bfe
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * Tests for {@link DecisionTreeClassificationTrainer}.
+ */
+@RunWith(Parameterized.class)
+public class DecisionTreeClassificationTrainerTest {
+ /** Number of parts to be tested. */
+ private static final int[] partsToBeTested = new int[] {1, 2, 3, 4, 5, 7};
+
+ /** Number of partitions. */
+ @Parameterized.Parameter
+ public int parts;
+
+ @Parameterized.Parameters(name = "Data divided on {0} partitions")
+ public static Iterable<Integer[]> data() {
+ List<Integer[]> res = new ArrayList<>();
+ for (int part : partsToBeTested)
+ res.add(new Integer[] {part});
+
+ return res;
+ }
+
+ /** */
+ @Test
+ public void testFit() {
+ int size = 100;
+
+ Map<Integer, double[]> data = new HashMap<>();
+
+ Random rnd = new Random(0);
+ for (int i = 0; i < size; i++) {
+ double x = rnd.nextDouble() - 0.5;
+ data.put(i, new double[]{x, x > 0 ? 1 : 0});
+ }
+
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(1, 0);
+
+ DecisionTreeNode tree = trainer.fit(
+ new LocalDatasetBuilder<>(data, parts),
+ (k, v) -> Arrays.copyOf(v, v.length - 1),
+ (k, v) -> v[v.length - 1]
+ );
+
+ assertTrue(tree instanceof DecisionTreeConditionalNode);
+
+ DecisionTreeConditionalNode node = (DecisionTreeConditionalNode) tree;
+
+ assertEquals(0, node.getThreshold(), 1e-3);
+
+ assertTrue(node.getThenNode() instanceof DecisionTreeLeafNode);
+ assertTrue(node.getElseNode() instanceof DecisionTreeLeafNode);
+
+ DecisionTreeLeafNode thenNode = (DecisionTreeLeafNode) node.getThenNode();
+ DecisionTreeLeafNode elseNode = (DecisionTreeLeafNode) node.getElseNode();
+
+ assertEquals(1, thenNode.getVal(), 1e-10);
+ assertEquals(0, elseNode.getVal(), 1e-10);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
new file mode 100644
index 0000000..754ff20
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.Arrays;
+import java.util.Random;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for {@link DecisionTreeRegressionTrainer} that require to start the whole Ignite infrastructure.
+ */
+public class DecisionTreeRegressionTrainerIntegrationTest extends GridCommonAbstractTest {
+ /** Number of nodes in grid */
+ private static final int NODE_COUNT = 3;
+
+ /** Ignite instance. */
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ for (int i = 1; i <= NODE_COUNT; i++)
+ startGrid(i);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() {
+ stopAllGrids();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void beforeTest() throws Exception {
+ /* Grid instance. */
+ ignite = grid(NODE_COUNT);
+ ignite.configuration().setPeerClassLoadingEnabled(true);
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+ }
+
+ /** */
+ public void testFit() {
+ int size = 100;
+
+ CacheConfiguration<Integer, double[]> trainingSetCacheCfg = new CacheConfiguration<>();
+ trainingSetCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+ trainingSetCacheCfg.setName("TRAINING_SET");
+
+ IgniteCache<Integer, double[]> data = ignite.createCache(trainingSetCacheCfg);
+
+ Random rnd = new Random(0);
+ for (int i = 0; i < size; i++) {
+ double x = rnd.nextDouble() - 0.5;
+ data.put(i, new double[]{x, x > 0 ? 1 : 0});
+ }
+
+ DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(1, 0);
+
+ DecisionTreeNode tree = trainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, data),
+ (k, v) -> Arrays.copyOf(v, v.length - 1),
+ (k, v) -> v[v.length - 1]
+ );
+
+ assertTrue(tree instanceof DecisionTreeConditionalNode);
+
+ DecisionTreeConditionalNode node = (DecisionTreeConditionalNode) tree;
+
+ assertEquals(0, node.getThreshold(), 1e-3);
+
+ assertTrue(node.getThenNode() instanceof DecisionTreeLeafNode);
+ assertTrue(node.getElseNode() instanceof DecisionTreeLeafNode);
+
+ DecisionTreeLeafNode thenNode = (DecisionTreeLeafNode) node.getThenNode();
+ DecisionTreeLeafNode elseNode = (DecisionTreeLeafNode) node.getElseNode();
+
+ assertEquals(1, thenNode.getVal(), 1e-10);
+ assertEquals(0, elseNode.getVal(), 1e-10);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
new file mode 100644
index 0000000..3bdbf60
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * Tests for {@link DecisionTreeRegressionTrainer}.
+ */
+@RunWith(Parameterized.class)
+public class DecisionTreeRegressionTrainerTest {
+ /** Number of parts to be tested. */
+ private static final int[] partsToBeTested = new int[] {1, 2, 3, 4, 5, 7};
+
+ /** Number of partitions. */
+ @Parameterized.Parameter
+ public int parts;
+
+ @Parameterized.Parameters(name = "Data divided on {0} partitions")
+ public static Iterable<Integer[]> data() {
+ List<Integer[]> res = new ArrayList<>();
+ for (int part : partsToBeTested)
+ res.add(new Integer[] {part});
+
+ return res;
+ }
+
+ /** */
+ @Test
+ public void testFit() {
+ int size = 100;
+
+ Map<Integer, double[]> data = new HashMap<>();
+
+ Random rnd = new Random(0);
+ for (int i = 0; i < size; i++) {
+ double x = rnd.nextDouble() - 0.5;
+ data.put(i, new double[]{x, x > 0 ? 1 : 0});
+ }
+
+ DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(1, 0);
+
+ DecisionTreeNode tree = trainer.fit(
+ new LocalDatasetBuilder<>(data, parts),
+ (k, v) -> Arrays.copyOf(v, v.length - 1),
+ (k, v) -> v[v.length - 1]
+ );
+
+ assertTrue(tree instanceof DecisionTreeConditionalNode);
+
+ DecisionTreeConditionalNode node = (DecisionTreeConditionalNode) tree;
+
+ assertEquals(0, node.getThreshold(), 1e-3);
+
+ assertTrue(node.getThenNode() instanceof DecisionTreeLeafNode);
+ assertTrue(node.getElseNode() instanceof DecisionTreeLeafNode);
+
+ DecisionTreeLeafNode thenNode = (DecisionTreeLeafNode) node.getThenNode();
+ DecisionTreeLeafNode elseNode = (DecisionTreeLeafNode) node.getElseNode();
+
+ assertEquals(1, thenNode.getVal(), 1e-10);
+ assertEquals(0, elseNode.getVal(), 1e-10);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeTestSuite.java
new file mode 100644
index 0000000..2cbb486
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeTestSuite.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import org.apache.ignite.ml.tree.data.DecisionTreeDataTest;
+import org.apache.ignite.ml.tree.impurity.gini.GiniImpurityMeasureCalculatorTest;
+import org.apache.ignite.ml.tree.impurity.gini.GiniImpurityMeasureTest;
+import org.apache.ignite.ml.tree.impurity.mse.MSEImpurityMeasureCalculatorTest;
+import org.apache.ignite.ml.tree.impurity.mse.MSEImpurityMeasureTest;
+import org.apache.ignite.ml.tree.impurity.util.SimpleStepFunctionCompressorTest;
+import org.apache.ignite.ml.tree.impurity.util.StepFunctionTest;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/**
+ * Test suite for all tests located in {@link org.apache.ignite.ml.tree} package.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ DecisionTreeClassificationTrainerTest.class,
+ DecisionTreeRegressionTrainerTest.class,
+ DecisionTreeClassificationTrainerIntegrationTest.class,
+ DecisionTreeRegressionTrainerIntegrationTest.class,
+ DecisionTreeDataTest.class,
+ GiniImpurityMeasureCalculatorTest.class,
+ GiniImpurityMeasureTest.class,
+ MSEImpurityMeasureCalculatorTest.class,
+ MSEImpurityMeasureTest.class,
+ StepFunctionTest.class,
+ SimpleStepFunctionCompressorTest.class
+})
+public class DecisionTreeTestSuite {
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/data/DecisionTreeDataTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/data/DecisionTreeDataTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/data/DecisionTreeDataTest.java
new file mode 100644
index 0000000..0c89d4e
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/data/DecisionTreeDataTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.data;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link DecisionTreeData}.
+ */
+public class DecisionTreeDataTest {
+ /** */
+ @Test
+ public void testFilter() {
+ double[][] features = new double[][]{{0}, {1}, {2}, {3}, {4}, {5}};
+ double[] labels = new double[]{0, 1, 2, 3, 4, 5};
+
+ DecisionTreeData data = new DecisionTreeData(features, labels);
+ DecisionTreeData filteredData = data.filter(obj -> obj[0] > 2);
+
+ assertArrayEquals(new double[][]{{3}, {4}, {5}}, filteredData.getFeatures());
+ assertArrayEquals(new double[]{3, 4, 5}, filteredData.getLabels(), 1e-10);
+ }
+
+ /** */
+ @Test
+ public void testSort() {
+ double[][] features = new double[][]{{4, 1}, {3, 3}, {2, 0}, {1, 4}, {0, 2}};
+ double[] labels = new double[]{0, 1, 2, 3, 4};
+
+ DecisionTreeData data = new DecisionTreeData(features, labels);
+
+ data.sort(0);
+
+ assertArrayEquals(new double[][]{{0, 2}, {1, 4}, {2, 0}, {3, 3}, {4, 1}}, features);
+ assertArrayEquals(new double[]{4, 3, 2, 1, 0}, labels, 1e-10);
+
+ data.sort(1);
+
+ assertArrayEquals(new double[][]{{2, 0}, {4, 1}, {0, 2}, {3, 3}, {1, 4}}, features);
+ assertArrayEquals(new double[]{2, 0, 4, 1, 3}, labels, 1e-10);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculatorTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculatorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculatorTest.java
new file mode 100644
index 0000000..afd81e8
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculatorTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.gini;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link GiniImpurityMeasureCalculator}.
+ */
+public class GiniImpurityMeasureCalculatorTest {
+ /** */
+ @Test
+ public void testCalculate() {
+ double[][] data = new double[][]{{0, 1}, {1, 0}, {2, 2}, {3, 3}};
+ double[] labels = new double[]{0, 1, 1, 1};
+
+ Map<Double, Integer> encoder = new HashMap<>();
+ encoder.put(0.0, 0);
+ encoder.put(1.0, 1);
+ GiniImpurityMeasureCalculator calculator = new GiniImpurityMeasureCalculator(encoder);
+
+ StepFunction<GiniImpurityMeasure>[] impurity = calculator.calculate(new DecisionTreeData(data, labels));
+
+ assertEquals(2, impurity.length);
+
+ // Check Gini calculated for the first column.
+ assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[0].getX(), 1e-10);
+ assertEquals(-2.500, impurity[0].getY()[0].impurity(), 1e-3);
+ assertEquals(-4.000, impurity[0].getY()[1].impurity(),1e-3);
+ assertEquals(-3.000, impurity[0].getY()[2].impurity(),1e-3);
+ assertEquals(-2.666, impurity[0].getY()[3].impurity(),1e-3);
+ assertEquals(-2.500, impurity[0].getY()[4].impurity(),1e-3);
+
+ // Check Gini calculated for the second column.
+ assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[1].getX(), 1e-10);
+ assertEquals(-2.500, impurity[1].getY()[0].impurity(),1e-3);
+ assertEquals(-2.666, impurity[1].getY()[1].impurity(),1e-3);
+ assertEquals(-3.000, impurity[1].getY()[2].impurity(),1e-3);
+ assertEquals(-2.666, impurity[1].getY()[3].impurity(),1e-3);
+ assertEquals(-2.500, impurity[1].getY()[4].impurity(),1e-3);
+ }
+
+ /** */
+ @Test
+ public void testCalculateWithRepeatedData() {
+ double[][] data = new double[][]{{0}, {1}, {2}, {2}, {3}};
+ double[] labels = new double[]{0, 1, 1, 1, 1};
+
+ Map<Double, Integer> encoder = new HashMap<>();
+ encoder.put(0.0, 0);
+ encoder.put(1.0, 1);
+ GiniImpurityMeasureCalculator calculator = new GiniImpurityMeasureCalculator(encoder);
+
+ StepFunction<GiniImpurityMeasure>[] impurity = calculator.calculate(new DecisionTreeData(data, labels));
+
+ assertEquals(1, impurity.length);
+
+ // Check Gini calculated for the first column.
+ assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[0].getX(), 1e-10);
+ assertEquals(-3.400, impurity[0].getY()[0].impurity(), 1e-3);
+ assertEquals(-5.000, impurity[0].getY()[1].impurity(),1e-3);
+ assertEquals(-4.000, impurity[0].getY()[2].impurity(),1e-3);
+ assertEquals(-3.500, impurity[0].getY()[3].impurity(),1e-3);
+ assertEquals(-3.400, impurity[0].getY()[4].impurity(),1e-3);
+ }
+
+ /** */
+ @Test
+ public void testGetLabelCode() {
+ Map<Double, Integer> encoder = new HashMap<>();
+ encoder.put(0.0, 0);
+ encoder.put(1.0, 1);
+ encoder.put(2.0, 2);
+
+ GiniImpurityMeasureCalculator calculator = new GiniImpurityMeasureCalculator(encoder);
+
+ assertEquals(0, calculator.getLabelCode(0.0));
+ assertEquals(1, calculator.getLabelCode(1.0));
+ assertEquals(2, calculator.getLabelCode(2.0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureTest.java
new file mode 100644
index 0000000..35c456a
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.gini;
+
+import java.util.Random;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+
+/**
+ * Tests for {@link GiniImpurityMeasure}.
+ */
+public class GiniImpurityMeasureTest {
+ /** */
+ @Test
+ public void testImpurityOnEmptyData() {
+ long[] left = new long[]{0, 0, 0};
+ long[] right = new long[]{0, 0, 0};
+
+ GiniImpurityMeasure impurity = new GiniImpurityMeasure(left, right);
+
+ assertEquals(0.0, impurity.impurity(), 1e-10);
+ }
+
+ /** */
+ @Test
+ public void testImpurityLeftPart() {
+ long[] left = new long[]{3, 0, 0};
+ long[] right = new long[]{0, 0, 0};
+
+ GiniImpurityMeasure impurity = new GiniImpurityMeasure(left, right);
+
+ assertEquals(-3, impurity.impurity(), 1e-10);
+ }
+
+ /** */
+ @Test
+ public void testImpurityRightPart() {
+ long[] left = new long[]{0, 0, 0};
+ long[] right = new long[]{3, 0, 0};
+
+ GiniImpurityMeasure impurity = new GiniImpurityMeasure(left, right);
+
+ assertEquals(-3, impurity.impurity(), 1e-10);
+ }
+
+ /** */
+ @Test
+ public void testImpurityLeftAndRightPart() {
+ long[] left = new long[]{3, 0, 0};
+ long[] right = new long[]{0, 3, 0};
+
+ GiniImpurityMeasure impurity = new GiniImpurityMeasure(left, right);
+
+ assertEquals(-6, impurity.impurity(), 1e-10);
+ }
+
+ /** */
+ @Test
+ public void testAdd() {
+ Random rnd = new Random(0);
+
+ GiniImpurityMeasure a = new GiniImpurityMeasure(
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)},
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)}
+ );
+
+
+ GiniImpurityMeasure b = new GiniImpurityMeasure(
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)},
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)}
+ );
+
+ GiniImpurityMeasure c = a.add(b);
+
+ assertEquals(a.getLeft()[0] + b.getLeft()[0], c.getLeft()[0]);
+ assertEquals(a.getLeft()[1] + b.getLeft()[1], c.getLeft()[1]);
+ assertEquals(a.getLeft()[2] + b.getLeft()[2], c.getLeft()[2]);
+
+ assertEquals(a.getRight()[0] + b.getRight()[0], c.getRight()[0]);
+ assertEquals(a.getRight()[1] + b.getRight()[1], c.getRight()[1]);
+ assertEquals(a.getRight()[2] + b.getRight()[2], c.getRight()[2]);
+ }
+
+ /** */
+ @Test
+ public void testSubtract() {
+ Random rnd = new Random(0);
+
+ GiniImpurityMeasure a = new GiniImpurityMeasure(
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)},
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)}
+ );
+
+
+ GiniImpurityMeasure b = new GiniImpurityMeasure(
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)},
+ new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)}
+ );
+
+ GiniImpurityMeasure c = a.subtract(b);
+
+ assertEquals(a.getLeft()[0] - b.getLeft()[0], c.getLeft()[0]);
+ assertEquals(a.getLeft()[1] - b.getLeft()[1], c.getLeft()[1]);
+ assertEquals(a.getLeft()[2] - b.getLeft()[2], c.getLeft()[2]);
+
+ assertEquals(a.getRight()[0] - b.getRight()[0], c.getRight()[0]);
+ assertEquals(a.getRight()[1] - b.getRight()[1], c.getRight()[1]);
+ assertEquals(a.getRight()[2] - b.getRight()[2], c.getRight()[2]);
+ }
+
+ /** Generates random count. */
+ private long randCnt(Random rnd) {
+ return Math.abs(rnd.nextInt());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculatorTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculatorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculatorTest.java
new file mode 100644
index 0000000..510c18f
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculatorTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.mse;
+
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link MSEImpurityMeasureCalculator}.
+ */
+public class MSEImpurityMeasureCalculatorTest {
+ /** */
+ @Test
+ public void testCalculate() {
+ double[][] data = new double[][]{{0, 2}, {1, 1}, {2, 0}, {3, 3}};
+ double[] labels = new double[]{1, 2, 2, 1};
+
+ MSEImpurityMeasureCalculator calculator = new MSEImpurityMeasureCalculator();
+
+ StepFunction<MSEImpurityMeasure>[] impurity = calculator.calculate(new DecisionTreeData(data, labels));
+
+ assertEquals(2, impurity.length);
+
+ // Test MSE calculated for the first column.
+ assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[0].getX(), 1e-10);
+ assertEquals(1.000, impurity[0].getY()[0].impurity(), 1e-3);
+ assertEquals(0.666, impurity[0].getY()[1].impurity(),1e-3);
+ assertEquals(1.000, impurity[0].getY()[2].impurity(),1e-3);
+ assertEquals(0.666, impurity[0].getY()[3].impurity(),1e-3);
+ assertEquals(1.000, impurity[0].getY()[4].impurity(),1e-3);
+
+ // Test MSE calculated for the second column.
+ assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[1].getX(), 1e-10);
+ assertEquals(1.000, impurity[1].getY()[0].impurity(),1e-3);
+ assertEquals(0.666, impurity[1].getY()[1].impurity(),1e-3);
+ assertEquals(0.000, impurity[1].getY()[2].impurity(),1e-3);
+ assertEquals(0.666, impurity[1].getY()[3].impurity(),1e-3);
+ assertEquals(1.000, impurity[1].getY()[4].impurity(),1e-3);
+ }
+}
[18/54] [abbrv] ignite git commit: IGNITE-8059: Integrate decision
tree with partition based dataset.
Posted by ag...@apache.org.
IGNITE-8059: Integrate decision tree with partition based dataset.
this closes #3760
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/139c2af6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/139c2af6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/139c2af6
Branch: refs/heads/ignite-6083
Commit: 139c2af66a9f745f89429842810f5d5fe1addf28
Parents: a64b941
Author: dmitrievanthony <dm...@gmail.com>
Authored: Tue Apr 10 12:46:43 2018 +0300
Committer: YuriBabak <y....@gmail.com>
Committed: Tue Apr 10 12:46:44 2018 +0300
----------------------------------------------------------------------
...ecisionTreeClassificationTrainerExample.java | 147 +++++
.../DecisionTreeRegressionTrainerExample.java | 124 ++++
.../ignite/examples/ml/tree/package-info.java | 22 +
.../examples/ml/trees/DecisionTreesExample.java | 354 ------------
.../ignite/examples/ml/trees/package-info.java | 22 -
.../main/java/org/apache/ignite/ml/Trainer.java | 3 -
.../org/apache/ignite/ml/tree/DecisionTree.java | 252 ++++++++
.../tree/DecisionTreeClassificationTrainer.java | 93 +++
.../ml/tree/DecisionTreeConditionalNode.java | 78 +++
.../ignite/ml/tree/DecisionTreeLeafNode.java | 48 ++
.../apache/ignite/ml/tree/DecisionTreeNode.java | 26 +
.../ml/tree/DecisionTreeRegressionTrainer.java | 60 ++
.../org/apache/ignite/ml/tree/TreeFilter.java | 38 ++
.../ignite/ml/tree/data/DecisionTreeData.java | 128 +++++
.../ml/tree/data/DecisionTreeDataBuilder.java | 73 +++
.../ignite/ml/tree/data/package-info.java | 22 +
.../ml/tree/impurity/ImpurityMeasure.java | 55 ++
.../impurity/ImpurityMeasureCalculator.java | 38 ++
.../tree/impurity/gini/GiniImpurityMeasure.java | 115 ++++
.../gini/GiniImpurityMeasureCalculator.java | 110 ++++
.../ml/tree/impurity/gini/package-info.java | 22 +
.../tree/impurity/mse/MSEImpurityMeasure.java | 133 +++++
.../mse/MSEImpurityMeasureCalculator.java | 80 +++
.../ml/tree/impurity/mse/package-info.java | 22 +
.../ignite/ml/tree/impurity/package-info.java | 22 +
.../util/SimpleStepFunctionCompressor.java | 149 +++++
.../ml/tree/impurity/util/StepFunction.java | 162 ++++++
.../impurity/util/StepFunctionCompressor.java | 55 ++
.../ml/tree/impurity/util/package-info.java | 22 +
.../ml/tree/leaf/DecisionTreeLeafBuilder.java | 38 ++
.../tree/leaf/MeanDecisionTreeLeafBuilder.java | 73 +++
.../leaf/MostCommonDecisionTreeLeafBuilder.java | 86 +++
.../ignite/ml/tree/leaf/package-info.java | 22 +
.../org/apache/ignite/ml/tree/package-info.java | 22 +
.../ignite/ml/trees/CategoricalRegionInfo.java | 72 ---
.../ignite/ml/trees/CategoricalSplitInfo.java | 68 ---
.../ignite/ml/trees/ContinuousRegionInfo.java | 74 ---
.../ml/trees/ContinuousSplitCalculator.java | 51 --
.../org/apache/ignite/ml/trees/RegionInfo.java | 62 --
.../ml/trees/models/DecisionTreeModel.java | 44 --
.../ignite/ml/trees/models/package-info.java | 22 -
.../ml/trees/nodes/CategoricalSplitNode.java | 50 --
.../ml/trees/nodes/ContinuousSplitNode.java | 56 --
.../ignite/ml/trees/nodes/DecisionTreeNode.java | 33 --
.../org/apache/ignite/ml/trees/nodes/Leaf.java | 49 --
.../apache/ignite/ml/trees/nodes/SplitNode.java | 100 ----
.../ignite/ml/trees/nodes/package-info.java | 22 -
.../apache/ignite/ml/trees/package-info.java | 22 -
.../ml/trees/trainers/columnbased/BiIndex.java | 113 ----
...exedCacheColumnDecisionTreeTrainerInput.java | 57 --
.../CacheColumnDecisionTreeTrainerInput.java | 141 -----
.../columnbased/ColumnDecisionTreeTrainer.java | 568 -------------------
.../ColumnDecisionTreeTrainerInput.java | 55 --
.../MatrixColumnDecisionTreeTrainerInput.java | 83 ---
.../trainers/columnbased/RegionProjection.java | 109 ----
.../trainers/columnbased/TrainingContext.java | 166 ------
.../columnbased/caches/ContextCache.java | 68 ---
.../columnbased/caches/FeaturesCache.java | 151 -----
.../columnbased/caches/ProjectionsCache.java | 286 ----------
.../trainers/columnbased/caches/SplitCache.java | 206 -------
.../columnbased/caches/package-info.java | 22 -
.../ContinuousSplitCalculators.java | 34 --
.../contsplitcalcs/GiniSplitCalculator.java | 234 --------
.../contsplitcalcs/VarianceSplitCalculator.java | 179 ------
.../contsplitcalcs/package-info.java | 22 -
.../trainers/columnbased/package-info.java | 22 -
.../columnbased/regcalcs/RegionCalculators.java | 85 ---
.../columnbased/regcalcs/package-info.java | 22 -
.../vectors/CategoricalFeatureProcessor.java | 212 -------
.../vectors/ContinuousFeatureProcessor.java | 111 ----
.../vectors/ContinuousSplitInfo.java | 71 ---
.../columnbased/vectors/FeatureProcessor.java | 82 ---
.../vectors/FeatureVectorProcessorUtils.java | 57 --
.../columnbased/vectors/SampleInfo.java | 80 ---
.../trainers/columnbased/vectors/SplitInfo.java | 106 ----
.../columnbased/vectors/package-info.java | 22 -
.../org/apache/ignite/ml/IgniteMLTestSuite.java | 4 +-
.../ml/nn/performance/MnistMLPTestUtil.java | 9 +-
...reeClassificationTrainerIntegrationTest.java | 100 ++++
.../DecisionTreeClassificationTrainerTest.java | 91 +++
...ionTreeRegressionTrainerIntegrationTest.java | 100 ++++
.../tree/DecisionTreeRegressionTrainerTest.java | 91 +++
.../ignite/ml/tree/DecisionTreeTestSuite.java | 48 ++
.../ml/tree/data/DecisionTreeDataTest.java | 59 ++
.../gini/GiniImpurityMeasureCalculatorTest.java | 103 ++++
.../impurity/gini/GiniImpurityMeasureTest.java | 131 +++++
.../mse/MSEImpurityMeasureCalculatorTest.java | 59 ++
.../impurity/mse/MSEImpurityMeasureTest.java | 109 ++++
.../util/SimpleStepFunctionCompressorTest.java | 75 +++
.../ml/tree/impurity/util/StepFunctionTest.java | 71 +++
.../tree/impurity/util/TestImpurityMeasure.java | 88 +++
.../DecisionTreeMNISTIntegrationTest.java | 105 ++++
.../tree/performance/DecisionTreeMNISTTest.java | 74 +++
.../ignite/ml/trees/BaseDecisionTreeTest.java | 70 ---
.../ml/trees/ColumnDecisionTreeTrainerTest.java | 191 -------
.../ignite/ml/trees/DecisionTreesTestSuite.java | 33 --
.../ml/trees/GiniSplitCalculatorTest.java | 141 -----
.../ignite/ml/trees/SplitDataGenerator.java | 390 -------------
.../ml/trees/VarianceSplitCalculatorTest.java | 84 ---
.../ColumnDecisionTreeTrainerBenchmark.java | 456 ---------------
.../IgniteColumnDecisionTreeGiniBenchmark.java | 70 ---
...niteColumnDecisionTreeVarianceBenchmark.java | 71 ---
.../yardstick/ml/trees/SplitDataGenerator.java | 426 --------------
.../ignite/yardstick/ml/trees/package-info.java | 22 -
104 files changed, 3647 insertions(+), 6429 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
new file mode 100644
index 0000000..cef6368
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.tree;
+
+import java.util.Random;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
+import org.apache.ignite.ml.tree.DecisionTreeNode;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * Example of using distributed {@link DecisionTreeClassificationTrainer}.
+ */
+public class DecisionTreeClassificationTrainerExample {
+ /**
+ * Executes example.
+ *
+ * @param args Command line arguments, none required.
+ */
+ public static void main(String... args) throws InterruptedException {
+ System.out.println(">>> Decision tree classification trainer example started.");
+
+ // Start ignite grid.
+ try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+ System.out.println(">>> Ignite grid started.");
+
+ IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+ DecisionTreeClassificationTrainerExample.class.getSimpleName(), () -> {
+
+ // Create cache with training data.
+ CacheConfiguration<Integer, LabeledPoint> trainingSetCfg = new CacheConfiguration<>();
+ trainingSetCfg.setName("TRAINING_SET");
+ trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+ IgniteCache<Integer, LabeledPoint> trainingSet = ignite.createCache(trainingSetCfg);
+
+ Random rnd = new Random(0);
+
+ // Fill training data.
+ for (int i = 0; i < 1000; i++)
+ trainingSet.put(i, generatePoint(rnd));
+
+ // Create classification trainer.
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(4, 0);
+
+ // Train decision tree model.
+ DecisionTreeNode mdl = trainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+ (k, v) -> new double[]{v.x, v.y},
+ (k, v) -> v.lb
+ );
+
+ // Calculate score.
+ int correctPredictions = 0;
+ for (int i = 0; i < 1000; i++) {
+ LabeledPoint pnt = generatePoint(rnd);
+
+ double prediction = mdl.apply(new double[]{pnt.x, pnt.y});
+
+ if (prediction == pnt.lb)
+ correctPredictions++;
+ }
+
+ System.out.println(">>> Accuracy: " + correctPredictions / 10.0 + "%");
+
+ System.out.println(">>> Decision tree classification trainer example completed.");
+ });
+
+ igniteThread.start();
+
+ igniteThread.join();
+ }
+ }
+
+ /**
+ * Generate point with {@code x} in (-0.5, 0.5) and {@code y} in the same interval. If {@code x * y > 0} then label
+ * is 1, otherwise 0.
+ *
+ * @param rnd Random.
+ * @return Point with label.
+ */
+ private static LabeledPoint generatePoint(Random rnd) {
+
+ double x = rnd.nextDouble() - 0.5;
+ double y = rnd.nextDouble() - 0.5;
+
+ return new LabeledPoint(x, y, x * y > 0 ? 1 : 0);
+ }
+
+ /** Point data class. */
+ private static class Point {
+ /** X coordinate. */
+ final double x;
+
+ /** Y coordinate. */
+ final double y;
+
+ /**
+ * Constructs a new instance of point.
+ *
+ * @param x X coordinate.
+ * @param y Y coordinate.
+ */
+ Point(double x, double y) {
+ this.x = x;
+ this.y = y;
+ }
+ }
+
+ /** Labeled point data class. */
+ private static class LabeledPoint extends Point {
+ /** Point label. */
+ final double lb;
+
+ /**
+ * Constructs a new instance of labeled point data.
+ *
+ * @param x X coordinate.
+ * @param y Y coordinate.
+ * @param lb Point label.
+ */
+ LabeledPoint(double x, double y, double lb) {
+ super(x, y);
+ this.lb = lb;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
new file mode 100644
index 0000000..61ba5f9
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.tree;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.tree.DecisionTreeNode;
+import org.apache.ignite.ml.tree.DecisionTreeRegressionTrainer;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * Example of using distributed {@link DecisionTreeRegressionTrainer}.
+ */
+public class DecisionTreeRegressionTrainerExample {
+ /**
+ * Executes example.
+ *
+ * @param args Command line arguments, none required.
+ */
+ public static void main(String... args) throws InterruptedException {
+ System.out.println(">>> Decision tree regression trainer example started.");
+
+ // Start ignite grid.
+ try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+ System.out.println(">>> Ignite grid started.");
+
+ IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+ DecisionTreeRegressionTrainerExample.class.getSimpleName(), () -> {
+
+ // Create cache with training data.
+ CacheConfiguration<Integer, Point> trainingSetCfg = new CacheConfiguration<>();
+ trainingSetCfg.setName("TRAINING_SET");
+ trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+ IgniteCache<Integer, Point> trainingSet = ignite.createCache(trainingSetCfg);
+
+ // Fill training data.
+ generatePoints(trainingSet);
+
+ // Create regression trainer.
+ DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(10, 0);
+
+ // Train decision tree model.
+ DecisionTreeNode mdl = trainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+ (k, v) -> new double[] {v.x},
+ (k, v) -> v.y
+ );
+
+ System.out.println(">>> Linear regression model: " + mdl);
+
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
+
+ // Calculate score.
+ for (int x = 0; x < 10; x++) {
+ double predicted = mdl.apply(new double[] {x});
+
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", predicted, Math.sin(x));
+ }
+
+ System.out.println(">>> ---------------------------------");
+
+ System.out.println(">>> Decision tree regression trainer example completed.");
+ });
+
+ igniteThread.start();
+
+ igniteThread.join();
+ }
+ }
+
+ /**
+ * Generates {@code sin(x)} on interval [0, 10) and loads into the specified cache.
+ */
+ private static void generatePoints(IgniteCache<Integer, Point> trainingSet) {
+ for (int i = 0; i < 1000; i++) {
+ double x = i / 100.0;
+ double y = Math.sin(x);
+
+ trainingSet.put(i, new Point(x, y));
+ }
+ }
+
+ /** Point data class. */
+ private static class Point {
+ /** X coordinate. */
+ final double x;
+
+ /** Y coordinate. */
+ final double y;
+
+ /**
+ * Constructs a new instance of point.
+ *
+ * @param x X coordinate.
+ * @param y Y coordinate.
+ */
+ Point(double x, double y) {
+ this.x = x;
+ this.y = y;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/examples/src/main/java/org/apache/ignite/examples/ml/tree/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/package-info.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/package-info.java
new file mode 100644
index 0000000..d8d9de6
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Decision trees examples.
+ */
+package org.apache.ignite.examples.ml.tree;
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/examples/src/main/java/org/apache/ignite/examples/ml/trees/DecisionTreesExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/trees/DecisionTreesExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/trees/DecisionTreesExample.java
deleted file mode 100644
index b1b2c42..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/trees/DecisionTreesExample.java
+++ /dev/null
@@ -1,354 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.trees;
-
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Random;
-import java.util.Scanner;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.zip.GZIPInputStream;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.examples.ExampleNodeStartup;
-import org.apache.ignite.examples.ml.MLExamplesCommonArgs;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.estimators.Estimators;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteTriFunction;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.BiIndex;
-import org.apache.ignite.ml.trees.trainers.columnbased.BiIndexedCacheColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.GiniSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.ml.util.MnistUtils;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * <p>
- * Example of usage of decision trees algorithm for MNIST dataset
- * (it can be found here: http://yann.lecun.com/exdb/mnist/). </p>
- * <p>
- * Remote nodes should always be started with special configuration file which
- * enables P2P class loading: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will start node
- * with {@code examples/config/example-ignite.xml} configuration.</p>
- * <p>
- * It is recommended to start at least one node prior to launching this example if you intend
- * to run it with default memory settings.</p>
- * <p>
- * This example should be run with program arguments, for example
- * -cfg examples/config/example-ignite.xml.</p>
- * <p>
- * -cfg specifies path to a config path.</p>
- */
-public class DecisionTreesExample {
- /** Name of parameter specifying path of Ignite config. */
- private static final String CONFIG = "cfg";
-
- /** Default config path. */
- private static final String DEFAULT_CONFIG = "examples/config/example-ignite.xml";
-
- /**
- * Folder in which MNIST dataset is expected.
- */
- private static String MNIST_DIR = "examples/src/main/resources/";
-
- /**
- * Key for MNIST training images.
- */
- private static String MNIST_TRAIN_IMAGES = "train_images";
-
- /**
- * Key for MNIST training labels.
- */
- private static String MNIST_TRAIN_LABELS = "train_labels";
-
- /**
- * Key for MNIST test images.
- */
- private static String MNIST_TEST_IMAGES = "test_images";
-
- /**
- * Key for MNIST test labels.
- */
- private static String MNIST_TEST_LABELS = "test_labels";
-
- /**
- * Launches example.
- *
- * @param args Program arguments.
- */
- public static void main(String[] args) throws IOException {
- System.out.println(">>> Decision trees example started.");
-
- String igniteCfgPath;
-
- CommandLineParser parser = new BasicParser();
-
- String trainingImagesPath;
- String trainingLabelsPath;
-
- String testImagesPath;
- String testLabelsPath;
-
- Map<String, String> mnistPaths = new HashMap<>();
-
- mnistPaths.put(MNIST_TRAIN_IMAGES, "train-images-idx3-ubyte");
- mnistPaths.put(MNIST_TRAIN_LABELS, "train-labels-idx1-ubyte");
- mnistPaths.put(MNIST_TEST_IMAGES, "t10k-images-idx3-ubyte");
- mnistPaths.put(MNIST_TEST_LABELS, "t10k-labels-idx1-ubyte");
-
- try {
- // Parse the command line arguments.
- CommandLine line = parser.parse(buildOptions(), args);
-
- if (line.hasOption(MLExamplesCommonArgs.UNATTENDED)) {
- System.out.println(">>> Skipped example execution because 'unattended' mode is used.");
- System.out.println(">>> Decision trees example finished.");
- return;
- }
-
- igniteCfgPath = line.getOptionValue(CONFIG, DEFAULT_CONFIG);
- }
- catch (ParseException e) {
- e.printStackTrace();
- return;
- }
-
- if (!getMNIST(mnistPaths.values())) {
- System.out.println(">>> You should have MNIST dataset in " + MNIST_DIR + " to run this example.");
- return;
- }
-
- trainingImagesPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" +
- mnistPaths.get(MNIST_TRAIN_IMAGES))).getPath();
- trainingLabelsPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" +
- mnistPaths.get(MNIST_TRAIN_LABELS))).getPath();
- testImagesPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" +
- mnistPaths.get(MNIST_TEST_IMAGES))).getPath();
- testLabelsPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" +
- mnistPaths.get(MNIST_TEST_LABELS))).getPath();
-
- try (Ignite ignite = Ignition.start(igniteCfgPath)) {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
- int ptsCnt = 60000;
- int featCnt = 28 * 28;
-
- Stream<DenseLocalOnHeapVector> trainingMnistStream = MnistUtils.mnistAsStream(trainingImagesPath, trainingLabelsPath,
- new Random(123L), ptsCnt);
-
- Stream<DenseLocalOnHeapVector> testMnistStream = MnistUtils.mnistAsStream(testImagesPath, testLabelsPath,
- new Random(123L), 10_000);
-
- IgniteCache<BiIndex, Double> cache = createBiIndexedCache(ignite);
-
- loadVectorsIntoBiIndexedCache(cache.getName(), trainingMnistStream.iterator(), featCnt + 1, ignite);
-
- ColumnDecisionTreeTrainer<GiniSplitCalculator.GiniData> trainer = new ColumnDecisionTreeTrainer<>(10,
- ContinuousSplitCalculators.GINI.apply(ignite),
- RegionCalculators.GINI,
- RegionCalculators.MOST_COMMON,
- ignite);
-
- System.out.println(">>> Training started");
- long before = System.currentTimeMillis();
- DecisionTreeModel mdl = trainer.train(new BiIndexedCacheColumnDecisionTreeTrainerInput(cache, new HashMap<>(), ptsCnt, featCnt));
- System.out.println(">>> Training finished in " + (System.currentTimeMillis() - before));
-
- IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse =
- Estimators.errorsPercentage();
-
- Double accuracy = mse.apply(mdl, testMnistStream.map(v ->
- new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
-
- System.out.println(">>> Errs percentage: " + accuracy);
- }
- catch (IOException e) {
- e.printStackTrace();
- }
-
- System.out.println(">>> Decision trees example finished.");
- }
-
- /**
- * Get MNIST dataset. Value of predicate 'MNIST dataset is present in expected folder' is returned.
- *
- * @param mnistFileNames File names of MNIST dataset.
- * @return Value of predicate 'MNIST dataset is present in expected folder'.
- * @throws IOException In case of file system errors.
- */
- private static boolean getMNIST(Collection<String> mnistFileNames) throws IOException {
- List<String> missing = mnistFileNames.stream().
- filter(f -> IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" + f) == null).
- collect(Collectors.toList());
-
- if (!missing.isEmpty()) {
- System.out.println(">>> You have not fully downloaded MNIST dataset in directory " + MNIST_DIR +
- ", do you want it to be downloaded? [y]/n");
- Scanner s = new Scanner(System.in);
- String str = s.nextLine();
-
- if (!str.isEmpty() && !str.toLowerCase().equals("y"))
- return false;
- }
-
- for (String s : missing) {
- String f = s + ".gz";
- System.out.println(">>> Downloading " + f + "...");
- URL website = new URL("http://yann.lecun.com/exdb/mnistAsStream/" + f);
- ReadableByteChannel rbc = Channels.newChannel(website.openStream());
- FileOutputStream fos = new FileOutputStream(MNIST_DIR + "/" + f);
- fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
- System.out.println(">>> Done.");
-
- System.out.println(">>> Unzipping " + f + "...");
- unzip(MNIST_DIR + "/" + f, MNIST_DIR + "/" + s);
-
- System.out.println(">>> Deleting gzip " + f + ", status: " +
- Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" + f)).delete());
-
- System.out.println(">>> Done.");
- }
-
- return true;
- }
-
- /**
- * Unzip file located in {@code input} to {@code output}.
- *
- * @param input Input file path.
- * @param output Output file path.
- * @throws IOException In case of file system errors.
- */
- private static void unzip(String input, String output) throws IOException {
- byte[] buf = new byte[1024];
-
- try (GZIPInputStream gis = new GZIPInputStream(new FileInputStream(input));
- FileOutputStream out = new FileOutputStream(output)) {
- int sz;
- while ((sz = gis.read(buf)) > 0)
- out.write(buf, 0, sz);
- }
- }
-
- /**
- * Build cli options.
- */
- @NotNull private static Options buildOptions() {
- Options options = new Options();
-
- Option cfgOpt = OptionBuilder
- .withArgName(CONFIG)
- .withLongOpt(CONFIG)
- .hasArg()
- .withDescription("Path to the config.")
- .isRequired(false).create();
-
- Option unattended = OptionBuilder
- .withArgName(MLExamplesCommonArgs.UNATTENDED)
- .withLongOpt(MLExamplesCommonArgs.UNATTENDED)
- .withDescription("Is example run unattended.")
- .isRequired(false).create();
-
- options.addOption(cfgOpt);
- options.addOption(unattended);
-
- return options;
- }
-
- /**
- * Creates cache where data for training is stored.
- *
- * @param ignite Ignite instance.
- * @return cache where data for training is stored.
- */
- private static IgniteCache<BiIndex, Double> createBiIndexedCache(Ignite ignite) {
- CacheConfiguration<BiIndex, Double> cfg = new CacheConfiguration<>();
-
- // Write to primary.
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
- // No copying of values.
- cfg.setCopyOnRead(false);
-
- cfg.setName("TMP_BI_INDEXED_CACHE");
-
- return ignite.getOrCreateCache(cfg);
- }
-
- /**
- * Loads vectors into cache.
- *
- * @param cacheName Name of cache.
- * @param vectorsIter Iterator over vectors to load.
- * @param vectorSize Size of vector.
- * @param ignite Ignite instance.
- */
- private static void loadVectorsIntoBiIndexedCache(String cacheName, Iterator<? extends Vector> vectorsIter,
- int vectorSize, Ignite ignite) {
- try (IgniteDataStreamer<BiIndex, Double> streamer =
- ignite.dataStreamer(cacheName)) {
- int sampleIdx = 0;
-
- streamer.perNodeBufferSize(10000);
-
- while (vectorsIter.hasNext()) {
- org.apache.ignite.ml.math.Vector next = vectorsIter.next();
-
- for (int i = 0; i < vectorSize; i++)
- streamer.addData(new BiIndex(sampleIdx, i), next.getX(i));
-
- sampleIdx++;
-
- if (sampleIdx % 1000 == 0)
- System.out.println(">>> Loaded " + sampleIdx + " vectors.");
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/examples/src/main/java/org/apache/ignite/examples/ml/trees/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/trees/package-info.java b/examples/src/main/java/org/apache/ignite/examples/ml/trees/package-info.java
deleted file mode 100644
index d944f60..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/trees/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Decision trees examples.
- */
-package org.apache.ignite.examples.ml.trees;
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
index 4e0a570..f53b801 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
@@ -17,11 +17,8 @@
package org.apache.ignite.ml;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-
/**
* Interface for Trainers. Trainer is just a function which produces model from the data.
- * See for example {@link ColumnDecisionTreeTrainer}.
*
* @param <M> Type of produced model.
* @param <T> Type of data needed for model producing.
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
new file mode 100644
index 0000000..c0b88fc
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.primitive.builder.context.EmptyContextBuilder;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.trainers.DatasetTrainer;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.data.DecisionTreeDataBuilder;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+import org.apache.ignite.ml.tree.impurity.util.StepFunctionCompressor;
+import org.apache.ignite.ml.tree.leaf.DecisionTreeLeafBuilder;
+
+/**
+ * Distributed decision tree trainer that allows to fit trees using row-partitioned dataset.
+ *
+ * @param <T> Type of impurity measure.
+ */
+abstract class DecisionTree<T extends ImpurityMeasure<T>> implements DatasetTrainer<DecisionTreeNode, Double> {
+ /** Max tree deep. */
+ private final int maxDeep;
+
+ /** Min impurity decrease. */
+ private final double minImpurityDecrease;
+
+ /** Step function compressor. */
+ private final StepFunctionCompressor<T> compressor;
+
+ /** Decision tree leaf builder. */
+ private final DecisionTreeLeafBuilder decisionTreeLeafBuilder;
+
+ /**
+ * Constructs a new distributed decision tree trainer.
+ *
+ * @param maxDeep Max tree deep.
+ * @param minImpurityDecrease Min impurity decrease.
+ * @param compressor Impurity function compressor.
+ * @param decisionTreeLeafBuilder Decision tree leaf builder.
+ */
+ DecisionTree(int maxDeep, double minImpurityDecrease, StepFunctionCompressor<T> compressor, DecisionTreeLeafBuilder decisionTreeLeafBuilder) {
+ this.maxDeep = maxDeep;
+ this.minImpurityDecrease = minImpurityDecrease;
+ this.compressor = compressor;
+ this.decisionTreeLeafBuilder = decisionTreeLeafBuilder;
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> DecisionTreeNode fit(DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+ try (Dataset<EmptyContext, DecisionTreeData> dataset = datasetBuilder.build(
+ new EmptyContextBuilder<>(),
+ new DecisionTreeDataBuilder<>(featureExtractor, lbExtractor)
+ )) {
+ return split(dataset, e -> true, 0, getImpurityMeasureCalculator(dataset));
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Returns impurity measure calculator.
+ *
+ * @param dataset Dataset.
+ * @return Impurity measure calculator.
+ */
+ abstract ImpurityMeasureCalculator<T> getImpurityMeasureCalculator(Dataset<EmptyContext, DecisionTreeData> dataset);
+
+ /**
+ * Splits the node specified by the given dataset and predicate and returns decision tree node.
+ *
+ * @param dataset Dataset.
+ * @param filter Decision tree node predicate.
+ * @param deep Current tree deep.
+ * @param impurityCalc Impurity measure calculator.
+ * @return Decision tree node.
+ */
+ private DecisionTreeNode split(Dataset<EmptyContext, DecisionTreeData> dataset, TreeFilter filter, int deep,
+ ImpurityMeasureCalculator<T> impurityCalc) {
+ if (deep >= maxDeep)
+ return decisionTreeLeafBuilder.createLeafNode(dataset, filter);
+
+ StepFunction<T>[] criterionFunctions = calculateImpurityForAllColumns(dataset, filter, impurityCalc);
+
+ if (criterionFunctions == null)
+ return decisionTreeLeafBuilder.createLeafNode(dataset, filter);
+
+ SplitPoint splitPnt = calculateBestSplitPoint(criterionFunctions);
+
+ if (splitPnt == null)
+ return decisionTreeLeafBuilder.createLeafNode(dataset, filter);
+
+ return new DecisionTreeConditionalNode(
+ splitPnt.col,
+ splitPnt.threshold,
+ split(dataset, updatePredicateForThenNode(filter, splitPnt), deep + 1, impurityCalc),
+ split(dataset, updatePredicateForElseNode(filter, splitPnt), deep + 1, impurityCalc)
+ );
+ }
+
+ /**
+ * Calculates impurity measure functions for all columns for the node specified by the given dataset and predicate.
+ *
+ * @param dataset Dataset.
+ * @param filter Decision tree node predicate.
+ * @param impurityCalc Impurity measure calculator.
+ * @return Array of impurity measure functions for all columns.
+ */
+ private StepFunction<T>[] calculateImpurityForAllColumns(Dataset<EmptyContext, DecisionTreeData> dataset,
+ TreeFilter filter, ImpurityMeasureCalculator<T> impurityCalc) {
+ return dataset.compute(
+ part -> {
+ if (compressor != null)
+ return compressor.compress(impurityCalc.calculate(part.filter(filter)));
+ else
+ return impurityCalc.calculate(part.filter(filter));
+ }, this::reduce
+ );
+ }
+
+ /**
+ * Calculates best split point.
+ *
+ * @param criterionFunctions Array of impurity measure functions for all columns.
+ * @return Best split point.
+ */
+ private SplitPoint calculateBestSplitPoint(StepFunction<T>[] criterionFunctions) {
+ SplitPoint<T> res = null;
+
+ for (int col = 0; col < criterionFunctions.length; col++) {
+ StepFunction<T> criterionFunctionForCol = criterionFunctions[col];
+
+ double[] arguments = criterionFunctionForCol.getX();
+ T[] values = criterionFunctionForCol.getY();
+
+ for (int leftSize = 1; leftSize < values.length - 1; leftSize++) {
+ if ((values[0].impurity() - values[leftSize].impurity()) > minImpurityDecrease
+ && (res == null || values[leftSize].compareTo(res.val) < 0))
+ res = new SplitPoint<>(values[leftSize], col, calculateThreshold(arguments, leftSize));
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Merges two arrays gotten from two partitions.
+ *
+ * @param a First step function.
+ * @param b Second step function.
+ * @return Merged step function.
+ */
+ private StepFunction<T>[] reduce(StepFunction<T>[] a, StepFunction<T>[] b) {
+ if (a == null)
+ return b;
+ if (b == null)
+ return a;
+ else {
+ StepFunction<T>[] res = Arrays.copyOf(a, a.length);
+
+ for (int i = 0; i < res.length; i++)
+ res[i] = res[i].add(b[i]);
+
+ return res;
+ }
+ }
+
+ /**
+ * Calculates threshold based on the given step function arguments and split point (specified left size).
+ *
+ * @param arguments Step function arguments.
+ * @param leftSize Split point (left size).
+ * @return Threshold.
+ */
+ private double calculateThreshold(double[] arguments, int leftSize) {
+ return (arguments[leftSize] + arguments[leftSize + 1]) / 2.0;
+ }
+
+ /**
+ * Constructs a new predicate for "then" node based on the parent node predicate and split point.
+ *
+ * @param filter Parent node predicate.
+ * @param splitPnt Split point.
+ * @return Predicate for "then" node.
+ */
+ private TreeFilter updatePredicateForThenNode(TreeFilter filter, SplitPoint splitPnt) {
+ return filter.and(f -> f[splitPnt.col] > splitPnt.threshold);
+ }
+
+ /**
+ * Constructs a new predicate for "else" node based on the parent node predicate and split point.
+ *
+ * @param filter Parent node predicate.
+ * @param splitPnt Split point.
+ * @return Predicate for "else" node.
+ */
+ private TreeFilter updatePredicateForElseNode(TreeFilter filter, SplitPoint splitPnt) {
+ return filter.and(f -> f[splitPnt.col] <= splitPnt.threshold);
+ }
+
+ /**
+ * Util class that represents split point.
+ */
+ private static class SplitPoint<T extends ImpurityMeasure<T>> implements Serializable {
+ /** */
+ private static final long serialVersionUID = -1758525953544425043L;
+
+ /** Split point impurity measure value. */
+ private final T val;
+
+ /** Column. */
+ private final int col;
+
+ /** Threshold. */
+ private final double threshold;
+
+ /**
+ * Constructs a new instance of split point.
+ *
+ * @param val Split point impurity measure value.
+ * @param col Column.
+ * @param threshold Threshold.
+ */
+ SplitPoint(T val, int col, double threshold) {
+ this.val = val;
+ this.col = col;
+ this.threshold = threshold;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
new file mode 100644
index 0000000..ce75190
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.gini.GiniImpurityMeasure;
+import org.apache.ignite.ml.tree.impurity.gini.GiniImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunctionCompressor;
+import org.apache.ignite.ml.tree.leaf.MostCommonDecisionTreeLeafBuilder;
+
+/**
+ * Decision tree classifier based on distributed decision tree trainer that allows to fit trees using row-partitioned
+ * dataset.
+ */
+public class DecisionTreeClassificationTrainer extends DecisionTree<GiniImpurityMeasure> {
+ /**
+ * Constructs a new decision tree classifier with default impurity function compressor.
+ *
+ * @param maxDeep Max tree deep.
+ * @param minImpurityDecrease Min impurity decrease.
+ */
+ public DecisionTreeClassificationTrainer(int maxDeep, double minImpurityDecrease) {
+ this(maxDeep, minImpurityDecrease, null);
+ }
+
+ /**
+ * Constructs a new instance of decision tree classifier.
+ *
+ * @param maxDeep Max tree deep.
+ * @param minImpurityDecrease Min impurity decrease.
+ */
+ public DecisionTreeClassificationTrainer(int maxDeep, double minImpurityDecrease,
+ StepFunctionCompressor<GiniImpurityMeasure> compressor) {
+ super(maxDeep, minImpurityDecrease, compressor, new MostCommonDecisionTreeLeafBuilder());
+ }
+
+ /** {@inheritDoc} */
+ @Override ImpurityMeasureCalculator<GiniImpurityMeasure> getImpurityMeasureCalculator(
+ Dataset<EmptyContext, DecisionTreeData> dataset) {
+ Set<Double> labels = dataset.compute(part -> {
+
+ if (part.getLabels() != null) {
+ Set<Double> list = new HashSet<>();
+
+ for (double lb : part.getLabels())
+ list.add(lb);
+
+ return list;
+ }
+
+ return null;
+ }, (a, b) -> {
+ if (a == null)
+ return b;
+ else if (b == null)
+ return a;
+ else {
+ a.addAll(b);
+ return a;
+ }
+ });
+
+ Map<Double, Integer> encoder = new HashMap<>();
+
+ int idx = 0;
+ for (Double lb : labels)
+ encoder.put(lb, idx++);
+
+ return new GiniImpurityMeasureCalculator(encoder);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java
new file mode 100644
index 0000000..9818239
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+/**
+ * Decision tree conditional (non-leaf) node.
+ */
+public class DecisionTreeConditionalNode implements DecisionTreeNode {
+ /** */
+ private static final long serialVersionUID = 981630737007982172L;
+
+ /** Column of the value to be tested. */
+ private final int col;
+
+ /** Threshold. */
+ private final double threshold;
+
+ /** Node that will be used in case tested value is greater then threshold. */
+ private final DecisionTreeNode thenNode;
+
+ /** Node that will be used in case tested value is not greater then threshold. */
+ private final DecisionTreeNode elseNode;
+
+ /**
+ * Constructs a new instance of decision tree conditional node.
+ *
+ * @param col Column of the value to be tested.
+ * @param threshold Threshold.
+ * @param thenNode Node that will be used in case tested value is greater then threshold.
+ * @param elseNode Node that will be used in case tested value is not greater then threshold.
+ */
+ DecisionTreeConditionalNode(int col, double threshold, DecisionTreeNode thenNode, DecisionTreeNode elseNode) {
+ this.col = col;
+ this.threshold = threshold;
+ this.thenNode = thenNode;
+ this.elseNode = elseNode;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double apply(double[] features) {
+ return features[col] > threshold ? thenNode.apply(features) : elseNode.apply(features);
+ }
+
+ /** */
+ public int getCol() {
+ return col;
+ }
+
+ /** */
+ public double getThreshold() {
+ return threshold;
+ }
+
+ /** */
+ public DecisionTreeNode getThenNode() {
+ return thenNode;
+ }
+
+ /** */
+ public DecisionTreeNode getElseNode() {
+ return elseNode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java
new file mode 100644
index 0000000..4c6369d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+/**
+ * Decision tree leaf node which contains value.
+ */
+public class DecisionTreeLeafNode implements DecisionTreeNode {
+ /** */
+ private static final long serialVersionUID = -472145568088482206L;
+
+ /** Value of the node. */
+ private final double val;
+
+ /**
+ * Constructs a new decision tree leaf node.
+ *
+ * @param val Value of the node.
+ */
+ public DecisionTreeLeafNode(double val) {
+ this.val = val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double apply(double[] doubles) {
+ return val;
+ }
+
+ /** */
+ public double getVal() {
+ return val;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeNode.java
new file mode 100644
index 0000000..94878eb
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeNode.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import org.apache.ignite.ml.Model;
+
+/**
+ * Base interface for decision tree nodes.
+ */
+public interface DecisionTreeNode extends Model<double[], Double> {
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
new file mode 100644
index 0000000..2bf09d3
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.mse.MSEImpurityMeasure;
+import org.apache.ignite.ml.tree.impurity.mse.MSEImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunctionCompressor;
+import org.apache.ignite.ml.tree.leaf.MeanDecisionTreeLeafBuilder;
+
+/**
+ * Decision tree regressor based on distributed decision tree trainer that allows to fit trees using row-partitioned
+ * dataset.
+ */
+public class DecisionTreeRegressionTrainer extends DecisionTree<MSEImpurityMeasure> {
+ /**
+ * Constructs a new decision tree regressor with default impurity function compressor.
+ *
+ * @param maxDeep Max tree deep.
+ * @param minImpurityDecrease Min impurity decrease.
+ */
+ public DecisionTreeRegressionTrainer(int maxDeep, double minImpurityDecrease) {
+ this(maxDeep, minImpurityDecrease, null);
+ }
+
+ /**
+ * Constructs a new decision tree regressor.
+ *
+ * @param maxDeep Max tree deep.
+ * @param minImpurityDecrease Min impurity decrease.
+ */
+ public DecisionTreeRegressionTrainer(int maxDeep, double minImpurityDecrease,
+ StepFunctionCompressor<MSEImpurityMeasure> compressor) {
+ super(maxDeep, minImpurityDecrease, compressor, new MeanDecisionTreeLeafBuilder());
+ }
+
+ /** {@inheritDoc} */
+ @Override ImpurityMeasureCalculator<MSEImpurityMeasure> getImpurityMeasureCalculator(
+ Dataset<EmptyContext, DecisionTreeData> dataset) {
+ return new MSEImpurityMeasureCalculator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/TreeFilter.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/TreeFilter.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/TreeFilter.java
new file mode 100644
index 0000000..3e4dc00
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/TreeFilter.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.function.Predicate;
+
+/**
+ * Predicate used to define objects that placed in decision tree node.
+ */
+public interface TreeFilter extends Predicate<double[]>, Serializable {
+ /**
+ * Returns a composed predicate.
+ *
+ * @param other Predicate that will be logically-ANDed with this predicate.
+ * @return Returns a composed predicate
+ */
+ default TreeFilter and(TreeFilter other) {
+ Objects.requireNonNull(other);
+ return (t) -> test(t) && other.test(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
new file mode 100644
index 0000000..34deb46
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.data;
+
+import org.apache.ignite.ml.tree.TreeFilter;
+
+/**
+ * A partition {@code data} of the containing matrix of features and vector of labels stored in heap.
+ */
+public class DecisionTreeData implements AutoCloseable {
+ /** Matrix with features. */
+ private final double[][] features;
+
+ /** Vector with labels. */
+ private final double[] labels;
+
+ /**
+ * Constructs a new instance of decision tree data.
+ *
+ * @param features Matrix with features.
+ * @param labels Vector with labels.
+ */
+ public DecisionTreeData(double[][] features, double[] labels) {
+ assert features.length == labels.length : "Features and labels have to be the same length";
+
+ this.features = features;
+ this.labels = labels;
+ }
+
+ /**
+ * Filters objects and returns only data that passed filter.
+ *
+ * @param filter Filter.
+ * @return Data passed filter.
+ */
+ public DecisionTreeData filter(TreeFilter filter) {
+ int size = 0;
+
+ for (int i = 0; i < features.length; i++)
+ if (filter.test(features[i]))
+ size++;
+
+ double[][] newFeatures = new double[size][];
+ double[] newLabels = new double[size];
+
+ int ptr = 0;
+
+ for (int i = 0; i < features.length; i++) {
+ if (filter.test(features[i])) {
+ newFeatures[ptr] = features[i];
+ newLabels[ptr] = labels[i];
+
+ ptr++;
+ }
+ }
+
+ return new DecisionTreeData(newFeatures, newLabels);
+ }
+
+ /**
+ * Sorts data by specified column in ascending order.
+ *
+ * @param col Column.
+ */
+ public void sort(int col) {
+ sort(col, 0, features.length - 1);
+ }
+
+ /** */
+ private void sort(int col, int from, int to) {
+ if (from < to) {
+ double pivot = features[(from + to) / 2][col];
+
+ int i = from, j = to;
+
+ while (i <= j) {
+ while (features[i][col] < pivot) i++;
+ while (features[j][col] > pivot) j--;
+
+ if (i <= j) {
+ double[] tmpFeature = features[i];
+ features[i] = features[j];
+ features[j] = tmpFeature;
+
+ double tmpLb = labels[i];
+ labels[i] = labels[j];
+ labels[j] = tmpLb;
+
+ i++;
+ j--;
+ }
+ }
+
+ sort(col, from, j);
+ sort(col, i, to);
+ }
+ }
+
+ /** */
+ public double[][] getFeatures() {
+ return features;
+ }
+
+ /** */
+ public double[] getLabels() {
+ return labels;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() {
+ // Do nothing, GC will clean up.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeDataBuilder.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeDataBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeDataBuilder.java
new file mode 100644
index 0000000..67109ae
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeDataBuilder.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.data;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import org.apache.ignite.ml.dataset.PartitionDataBuilder;
+import org.apache.ignite.ml.dataset.UpstreamEntry;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+
+/**
+ * A partition {@code data} builder that makes {@link DecisionTreeData}.
+ *
+ * @param <K> Type of a key in <tt>upstream</tt> data.
+ * @param <V> Type of a value in <tt>upstream</tt> data.
+ * @param <C> Type of a partition <tt>context</tt>.
+ */
+public class DecisionTreeDataBuilder<K, V, C extends Serializable>
+ implements PartitionDataBuilder<K, V, C, DecisionTreeData> {
+ /** */
+ private static final long serialVersionUID = 3678784980215216039L;
+
+ /** Function that extracts features from an {@code upstream} data. */
+ private final IgniteBiFunction<K, V, double[]> featureExtractor;
+
+ /** Function that extracts labels from an {@code upstream} data. */
+ private final IgniteBiFunction<K, V, Double> lbExtractor;
+
+ /**
+ * Constructs a new instance of decision tree data builder.
+ *
+ * @param featureExtractor Function that extracts features from an {@code upstream} data.
+ * @param lbExtractor Function that extracts labels from an {@code upstream} data.
+ */
+ public DecisionTreeDataBuilder(IgniteBiFunction<K, V, double[]> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+ this.featureExtractor = featureExtractor;
+ this.lbExtractor = lbExtractor;
+ }
+
+ /** {@inheritDoc} */
+ @Override public DecisionTreeData build(Iterator<UpstreamEntry<K, V>> upstreamData, long upstreamDataSize, C ctx) {
+ double[][] features = new double[Math.toIntExact(upstreamDataSize)][];
+ double[] labels = new double[Math.toIntExact(upstreamDataSize)];
+
+ int ptr = 0;
+ while (upstreamData.hasNext()) {
+ UpstreamEntry<K, V> entry = upstreamData.next();
+
+ features[ptr] = featureExtractor.apply(entry.getKey(), entry.getValue());
+ labels[ptr] = lbExtractor.apply(entry.getKey(), entry.getValue());
+
+ ptr++;
+ }
+
+ return new DecisionTreeData(features, labels);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/package-info.java
new file mode 100644
index 0000000..192b07f
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains data and data builder required for decision tree trainers built on top of partition based dataset.
+ */
+package org.apache.ignite.ml.tree.data;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasure.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasure.java
new file mode 100644
index 0000000..7ad2b80
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasure.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity;
+
+import java.io.Serializable;
+
+/**
+ * Base interface for impurity measures that can be used in distributed decision tree algorithm.
+ *
+ * @param <T> Type of this impurity measure.
+ */
+public interface ImpurityMeasure<T extends ImpurityMeasure<T>> extends Comparable<T>, Serializable {
+ /**
+ * Calculates impurity measure as a single double value.
+ *
+ * @return Impurity measure value.
+ */
+ public double impurity();
+
+ /**
+ * Adds the given impurity to this.
+ *
+ * @param measure Another impurity.
+ * @return Sum of this and the given impurity.
+ */
+ public T add(T measure);
+
+ /**
+ * Subtracts the given impurity for this.
+ *
+ * @param measure Another impurity.
+ * @return Difference of this and the given impurity.
+ */
+ public T subtract(T measure);
+
+ /** {@inheritDoc} */
+ default public int compareTo(T o) {
+ return Double.compare(impurity(), o.impurity());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasureCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasureCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasureCalculator.java
new file mode 100644
index 0000000..2b69356
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasureCalculator.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity;
+
+import java.io.Serializable;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+
+/**
+ * Base interface for impurity measure calculators that calculates all impurity measures required to find a best split.
+ *
+ * @param <T> Type of impurity measure.
+ */
+public interface ImpurityMeasureCalculator<T extends ImpurityMeasure<T>> extends Serializable {
+ /**
+ * Calculates all impurity measures required required to find a best split and returns them as an array of
+ * {@link StepFunction} (for every column).
+ *
+ * @param data Features and labels.
+ * @return Impurity measures as an array of {@link StepFunction} (for every column).
+ */
+ public StepFunction<T>[] calculate(DecisionTreeData data);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasure.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasure.java
new file mode 100644
index 0000000..817baf5
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasure.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.gini;
+
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Gini impurity measure which is calculated the following way:
+ * {@code \-frac{1}{L}\sum_{i=1}^{s}l_i^2 - \frac{1}{R}\sum_{i=s+1}^{n}r_i^2}.
+ */
+public class GiniImpurityMeasure implements ImpurityMeasure<GiniImpurityMeasure> {
+ /** */
+ private static final long serialVersionUID = 5338129703395229970L;
+
+ /** Number of elements of each type in the left part. */
+ private final long[] left;
+
+ /** Number of elements of each type in the right part. */
+ private final long[] right;
+
+ /**
+ * Constructs a new instance of Gini impurity measure.
+ *
+ * @param left Number of elements of each type in the left part.
+ * @param right Number of elements of each type in the right part.
+ */
+ GiniImpurityMeasure(long[] left, long[] right) {
+ assert left.length == right.length : "Left and right parts have to be the same length";
+
+ this.left = left;
+ this.right = right;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double impurity() {
+ long leftCnt = 0;
+ long rightCnt = 0;
+
+ double leftImpurity = 0;
+ double rightImpurity = 0;
+
+ for (long e : left)
+ leftCnt += e;
+
+ for (long e : right)
+ rightCnt += e;
+
+ if (leftCnt > 0)
+ for (long e : left)
+ leftImpurity += Math.pow(e, 2) / leftCnt;
+
+ if (rightCnt > 0)
+ for (long e : right)
+ rightImpurity += Math.pow(e, 2) / rightCnt;
+
+ return -(leftImpurity + rightImpurity);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GiniImpurityMeasure add(GiniImpurityMeasure b) {
+ assert left.length == b.left.length : "Subtracted measure has to have length " + left.length;
+ assert left.length == b.right.length : "Subtracted measure has to have length " + left.length;
+
+ long[] leftRes = new long[left.length];
+ long[] rightRes = new long[left.length];
+
+ for (int i = 0; i < left.length; i++) {
+ leftRes[i] = left[i] + b.left[i];
+ rightRes[i] = right[i] + b.right[i];
+ }
+
+ return new GiniImpurityMeasure(leftRes, rightRes);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GiniImpurityMeasure subtract(GiniImpurityMeasure b) {
+ assert left.length == b.left.length : "Subtracted measure has to have length " + left.length;
+ assert left.length == b.right.length : "Subtracted measure has to have length " + left.length;
+
+ long[] leftRes = new long[left.length];
+ long[] rightRes = new long[left.length];
+
+ for (int i = 0; i < left.length; i++) {
+ leftRes[i] = left[i] - b.left[i];
+ rightRes[i] = right[i] - b.right[i];
+ }
+
+ return new GiniImpurityMeasure(leftRes, rightRes);
+ }
+
+ /** */
+ public long[] getLeft() {
+ return left;
+ }
+
+ /** */
+ public long[] getRight() {
+ return right;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculator.java
new file mode 100644
index 0000000..0dd0a10
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculator.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.gini;
+
+import java.util.Arrays;
+import java.util.Map;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+
+/**
+ * Gini impurity measure calculator.
+ */
+public class GiniImpurityMeasureCalculator implements ImpurityMeasureCalculator<GiniImpurityMeasure> {
+ /** */
+ private static final long serialVersionUID = -522995134128519679L;
+
+ /** Label encoder which defines integer value for every label class. */
+ private final Map<Double, Integer> lbEncoder;
+
+ /**
+ * Constructs a new instance of Gini impurity measure calculator.
+ *
+ * @param lbEncoder Label encoder which defines integer value for every label class.
+ */
+ public GiniImpurityMeasureCalculator(Map<Double, Integer> lbEncoder) {
+ this.lbEncoder = lbEncoder;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public StepFunction<GiniImpurityMeasure>[] calculate(DecisionTreeData data) {
+ double[][] features = data.getFeatures();
+ double[] labels = data.getLabels();
+
+ if (features.length > 0) {
+ StepFunction<GiniImpurityMeasure>[] res = new StepFunction[features[0].length];
+
+ for (int col = 0; col < res.length; col++) {
+ data.sort(col);
+
+ double[] x = new double[features.length + 1];
+ GiniImpurityMeasure[] y = new GiniImpurityMeasure[features.length + 1];
+
+ int xPtr = 0, yPtr = 0;
+
+ long[] left = new long[lbEncoder.size()];
+ long[] right = new long[lbEncoder.size()];
+
+ for (int i = 0; i < labels.length; i++)
+ right[getLabelCode(labels[i])]++;
+
+ x[xPtr++] = Double.NEGATIVE_INFINITY;
+ y[yPtr++] = new GiniImpurityMeasure(
+ Arrays.copyOf(left, left.length),
+ Arrays.copyOf(right, right.length)
+ );
+
+ for (int i = 0; i < features.length; i++) {
+ left[getLabelCode(labels[i])]++;
+ right[getLabelCode(labels[i])]--;
+
+ if (i < (features.length - 1) && features[i + 1][col] == features[i][col])
+ continue;
+
+ x[xPtr++] = features[i][col];
+ y[yPtr++] = new GiniImpurityMeasure(
+ Arrays.copyOf(left, left.length),
+ Arrays.copyOf(right, right.length)
+ );
+ }
+
+ res[col] = new StepFunction<>(Arrays.copyOf(x, xPtr), Arrays.copyOf(y, yPtr));
+ }
+
+ return res;
+ }
+
+ return null;
+ }
+
+ /**
+ * Returns label code.
+ *
+ * @param lb Label.
+ * @return Label code.
+ */
+ int getLabelCode(double lb) {
+ Integer code = lbEncoder.get(lb);
+
+ assert code != null : "Can't find code for label " + lb;
+
+ return code;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/package-info.java
new file mode 100644
index 0000000..d14cd92
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains Gini impurity measure and calculator.
+ */
+package org.apache.ignite.ml.tree.impurity.gini;
\ No newline at end of file
[19/54] [abbrv] ignite git commit: IGNITE-7927 Web Console: Fixed
demo for non-collocated joins.
Posted by ag...@apache.org.
IGNITE-7927 Web Console: Fixed demo for non-collocated joins.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/647620b3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/647620b3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/647620b3
Branch: refs/heads/ignite-6083
Commit: 647620b3ccf204c7bf11c4ff540584d0c99d9a15
Parents: 139c2af
Author: Vasiliy Sisko <vs...@gridgain.com>
Authored: Tue Apr 10 17:48:52 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Apr 10 17:48:52 2018 +0700
----------------------------------------------------------------------
modules/web-console/backend/routes/demo.js | 2 ++
.../demo/service/DemoCachesLoadService.java | 22 ++++++++++++++++++--
2 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/647620b3/modules/web-console/backend/routes/demo.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/demo.js b/modules/web-console/backend/routes/demo.js
index a18fa7a..b081d0c 100644
--- a/modules/web-console/backend/routes/demo.js
+++ b/modules/web-console/backend/routes/demo.js
@@ -95,10 +95,12 @@ module.exports.factory = (errors, settings, mongo, spacesService) => {
domain.space = cacheDoc.space;
domain.caches.push(cacheDoc._id);
+ domain.clusters.push(cluster._id);
return domain.save()
.then((domainDoc) => {
cacheDoc.domains.push(domainDoc._id);
+ cluster.models.push(domainDoc._id);
return cacheDoc.save();
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/647620b3/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
----------------------------------------------------------------------
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
index 6691d1d..2aace06 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
@@ -269,6 +269,14 @@ public class DemoCachesLoadService implements Service {
type.setFields(qryFlds);
+ // Indexes for DEPARTMENT.
+
+ ArrayList<QueryIndex> indexes = new ArrayList<>();
+
+ indexes.add(new QueryIndex("countryId", QueryIndexType.SORTED, false, "DEP_COUNTRY"));
+
+ type.setIndexes(indexes);
+
ccfg.setQueryEntities(qryEntities);
return ccfg;
@@ -312,6 +320,11 @@ public class DemoCachesLoadService implements Service {
// Indexes for EMPLOYEE.
+ Collection<QueryIndex> indexes = new ArrayList<>();
+
+ indexes.add(new QueryIndex("departmentId", QueryIndexType.SORTED, false, "EMP_DEPARTMENT"));
+ indexes.add(new QueryIndex("managerId", QueryIndexType.SORTED, false, "EMP_MANAGER"));
+
QueryIndex idx = new QueryIndex();
idx.setName("EMP_NAMES");
@@ -323,8 +336,6 @@ public class DemoCachesLoadService implements Service {
idx.setFields(indFlds);
- Collection<QueryIndex> indexes = new ArrayList<>();
-
indexes.add(idx);
indexes.add(new QueryIndex("salary", QueryIndexType.SORTED, false, "EMP_SALARY"));
@@ -392,6 +403,13 @@ public class DemoCachesLoadService implements Service {
type.setFields(qryFlds);
+ // Indexes for CAR.
+
+ ArrayList<QueryIndex> indexes = new ArrayList<>();
+
+ indexes.add(new QueryIndex("parkingId", QueryIndexType.SORTED, false, "CAR_PARKING"));
+ type.setIndexes(indexes);
+
ccfg.setQueryEntities(qryEntities);
return ccfg;
[16/54] [abbrv] ignite git commit: IGNITE-8059: Integrate decision
tree with partition based dataset.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainer.java
deleted file mode 100644
index fec0a83..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainer.java
+++ /dev/null
@@ -1,568 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CachePeekMode;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distributed.CacheUtils;
-import org.apache.ignite.ml.math.functions.Functions;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteCurriedBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.nodes.DecisionTreeNode;
-import org.apache.ignite.ml.trees.nodes.Leaf;
-import org.apache.ignite.ml.trees.nodes.SplitNode;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ContextCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache.FeatureKey;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ProjectionsCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ProjectionsCache.RegionKey;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.SplitCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.SplitCache.SplitKey;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.FeatureProcessor;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-import org.jetbrains.annotations.NotNull;
-
-import static org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache.getFeatureCacheKey;
-
-/**
- * This trainer stores observations as columns and features as rows.
- * Ideas from https://github.com/fabuzaid21/yggdrasil are used here.
- */
-public class ColumnDecisionTreeTrainer<D extends ContinuousRegionInfo> implements
- Trainer<DecisionTreeModel, ColumnDecisionTreeTrainerInput> {
- /**
- * Function used to assign a value to a region.
- */
- private final IgniteFunction<DoubleStream, Double> regCalc;
-
- /**
- * Function used to calculate impurity in regions used by categorical features.
- */
- private final IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> continuousCalculatorProvider;
-
- /**
- * Categorical calculator provider.
- **/
- private final IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> categoricalCalculatorProvider;
-
- /**
- * Cache used for storing data for training.
- */
- private IgniteCache<RegionKey, List<RegionProjection>> prjsCache;
-
- /**
- * Minimal information gain.
- */
- private static final double MIN_INFO_GAIN = 1E-10;
-
- /**
- * Maximal depth of the decision tree.
- */
- private final int maxDepth;
-
- /**
- * Size of block which is used for storing regions in cache.
- */
- private static final int BLOCK_SIZE = 1 << 4;
-
- /** Ignite instance. */
- private final Ignite ignite;
-
- /** Logger */
- private final IgniteLogger log;
-
- /**
- * Construct {@link ColumnDecisionTreeTrainer}.
- *
- * @param maxDepth Maximal depth of the decision tree.
- * @param continuousCalculatorProvider Provider of calculator of splits for region projection on continuous
- * features.
- * @param categoricalCalculatorProvider Provider of calculator of splits for region projection on categorical
- * features.
- * @param regCalc Function used to assign a value to a region.
- */
- public ColumnDecisionTreeTrainer(int maxDepth,
- IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> continuousCalculatorProvider,
- IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> categoricalCalculatorProvider,
- IgniteFunction<DoubleStream, Double> regCalc,
- Ignite ignite) {
- this.maxDepth = maxDepth;
- this.continuousCalculatorProvider = continuousCalculatorProvider;
- this.categoricalCalculatorProvider = categoricalCalculatorProvider;
- this.regCalc = regCalc;
- this.ignite = ignite;
- this.log = ignite.log();
- }
-
- /**
- * Utility class used to get index of feature by which split is done and split info.
- */
- private static class IndexAndSplitInfo {
- /**
- * Index of feature by which split is done.
- */
- private final int featureIdx;
-
- /**
- * Split information.
- */
- private final SplitInfo info;
-
- /**
- * @param featureIdx Index of feature by which split is done.
- * @param info Split information.
- */
- IndexAndSplitInfo(int featureIdx, SplitInfo info) {
- this.featureIdx = featureIdx;
- this.info = info;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "IndexAndSplitInfo [featureIdx=" + featureIdx + ", info=" + info + ']';
- }
- }
-
- /**
- * Utility class used to build decision tree. Basically it is pointer to leaf node.
- */
- private static class TreeTip {
- /** */
- private Consumer<DecisionTreeNode> leafSetter;
-
- /** */
- private int depth;
-
- /** */
- TreeTip(Consumer<DecisionTreeNode> leafSetter, int depth) {
- this.leafSetter = leafSetter;
- this.depth = depth;
- }
- }
-
- /**
- * Utility class used as decision tree root node.
- */
- private static class RootNode implements DecisionTreeNode {
- /** */
- private DecisionTreeNode s;
-
- /**
- * {@inheritDoc}
- */
- @Override public double process(Vector v) {
- return s.process(v);
- }
-
- /** */
- void setSplit(DecisionTreeNode s) {
- this.s = s;
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public DecisionTreeModel train(ColumnDecisionTreeTrainerInput i) {
- prjsCache = ProjectionsCache.getOrCreate(ignite);
- IgniteCache<UUID, TrainingContext<D>> ctxtCache = ContextCache.getOrCreate(ignite);
- SplitCache.getOrCreate(ignite);
-
- UUID trainingUUID = UUID.randomUUID();
-
- TrainingContext<D> ct = new TrainingContext<>(i, continuousCalculatorProvider.apply(i), categoricalCalculatorProvider.apply(i), trainingUUID, ignite);
- ctxtCache.put(trainingUUID, ct);
-
- CacheUtils.bcast(prjsCache.getName(), ignite, () -> {
- Ignite ignite = Ignition.localIgnite();
- IgniteCache<RegionKey, List<RegionProjection>> projCache = ProjectionsCache.getOrCreate(ignite);
- IgniteCache<FeatureKey, double[]> featuresCache = FeaturesCache.getOrCreate(ignite);
-
- Affinity<RegionKey> targetAffinity = ignite.affinity(ProjectionsCache.CACHE_NAME);
-
- ClusterNode locNode = ignite.cluster().localNode();
-
- Map<FeatureKey, double[]> fm = new ConcurrentHashMap<>();
- Map<RegionKey, List<RegionProjection>> pm = new ConcurrentHashMap<>();
-
- targetAffinity.
- mapKeysToNodes(IntStream.range(0, i.featuresCount()).
- mapToObj(idx -> ProjectionsCache.key(idx, 0, i.affinityKey(idx, ignite), trainingUUID)).
- collect(Collectors.toSet())).getOrDefault(locNode, Collections.emptyList()).
- forEach(k -> {
- FeatureProcessor vec;
-
- int featureIdx = k.featureIdx();
-
- IgniteCache<UUID, TrainingContext<D>> ctxCache = ContextCache.getOrCreate(ignite);
- TrainingContext ctx = ctxCache.get(trainingUUID);
- double[] vals = new double[ctx.labels().length];
-
- vec = ctx.featureProcessor(featureIdx);
- i.values(featureIdx).forEach(t -> vals[t.get1()] = t.get2());
-
- fm.put(getFeatureCacheKey(featureIdx, trainingUUID, i.affinityKey(featureIdx, ignite)), vals);
-
- List<RegionProjection> newReg = new ArrayList<>(BLOCK_SIZE);
- newReg.add(vec.createInitialRegion(getSamples(i.values(featureIdx), ctx.labels().length), vals, ctx.labels()));
- pm.put(k, newReg);
- });
-
- featuresCache.putAll(fm);
- projCache.putAll(pm);
-
- return null;
- });
-
- return doTrain(i, trainingUUID);
- }
-
- /**
- * Get samples array.
- *
- * @param values Stream of tuples in the form of (index, value).
- * @param size size of stream.
- * @return Samples array.
- */
- private Integer[] getSamples(Stream<IgniteBiTuple<Integer, Double>> values, int size) {
- Integer[] res = new Integer[size];
-
- values.forEach(v -> res[v.get1()] = v.get1());
-
- return res;
- }
-
- /** */
- @NotNull
- private DecisionTreeModel doTrain(ColumnDecisionTreeTrainerInput input, UUID uuid) {
- RootNode root = new RootNode();
-
- // List containing setters of leaves of the tree.
- List<TreeTip> tips = new LinkedList<>();
- tips.add(new TreeTip(root::setSplit, 0));
-
- int curDepth = 0;
- int regsCnt = 1;
-
- int featuresCnt = input.featuresCount();
- IntStream.range(0, featuresCnt).mapToObj(fIdx -> SplitCache.key(fIdx, input.affinityKey(fIdx, ignite), uuid)).
- forEach(k -> SplitCache.getOrCreate(ignite).put(k, new IgniteBiTuple<>(0, 0.0)));
- updateSplitCache(0, regsCnt, featuresCnt, ig -> i -> input.affinityKey(i, ig), uuid);
-
- // TODO: IGNITE-5893 Currently if the best split makes tree deeper than max depth process will be terminated, but actually we should
- // only stop when *any* improving split makes tree deeper than max depth. Can be fixed if we will store which
- // regions cannot be split more and split only those that can.
- while (true) {
- long before = System.currentTimeMillis();
-
- IgniteBiTuple<Integer, IgniteBiTuple<Integer, Double>> b = findBestSplitIndexForFeatures(featuresCnt, input::affinityKey, uuid);
-
- long findBestRegIdx = System.currentTimeMillis() - before;
-
- Integer bestFeatureIdx = b.get1();
-
- Integer regIdx = b.get2().get1();
- Double bestInfoGain = b.get2().get2();
-
- if (regIdx >= 0 && bestInfoGain > MIN_INFO_GAIN) {
- before = System.currentTimeMillis();
-
- SplitInfo bi = ignite.compute().affinityCall(ProjectionsCache.CACHE_NAME,
- input.affinityKey(bestFeatureIdx, ignite),
- () -> {
- TrainingContext<ContinuousRegionInfo> ctx = ContextCache.getOrCreate(ignite).get(uuid);
- Ignite ignite = Ignition.localIgnite();
- RegionKey key = ProjectionsCache.key(bestFeatureIdx,
- regIdx / BLOCK_SIZE,
- input.affinityKey(bestFeatureIdx, Ignition.localIgnite()),
- uuid);
- RegionProjection reg = ProjectionsCache.getOrCreate(ignite).localPeek(key).get(regIdx % BLOCK_SIZE);
- return ctx.featureProcessor(bestFeatureIdx).findBestSplit(reg, ctx.values(bestFeatureIdx, ignite), ctx.labels(), regIdx);
- });
-
- long findBestSplit = System.currentTimeMillis() - before;
-
- IndexAndSplitInfo best = new IndexAndSplitInfo(bestFeatureIdx, bi);
-
- regsCnt++;
-
- if (log.isDebugEnabled())
- log.debug("Globally best: " + best.info + " idx time: " + findBestRegIdx + ", calculate best: " + findBestSplit + " fi: " + best.featureIdx + ", regs: " + regsCnt);
- // Request bitset for split region.
- int ind = best.info.regionIndex();
-
- SparseBitSet bs = ignite.compute().affinityCall(ProjectionsCache.CACHE_NAME,
- input.affinityKey(bestFeatureIdx, ignite),
- () -> {
- Ignite ignite = Ignition.localIgnite();
- IgniteCache<FeatureKey, double[]> featuresCache = FeaturesCache.getOrCreate(ignite);
- IgniteCache<UUID, TrainingContext<D>> ctxCache = ContextCache.getOrCreate(ignite);
- TrainingContext ctx = ctxCache.localPeek(uuid);
-
- double[] values = featuresCache.localPeek(getFeatureCacheKey(bestFeatureIdx, uuid, input.affinityKey(bestFeatureIdx, Ignition.localIgnite())));
- RegionKey key = ProjectionsCache.key(bestFeatureIdx,
- regIdx / BLOCK_SIZE,
- input.affinityKey(bestFeatureIdx, Ignition.localIgnite()),
- uuid);
- RegionProjection reg = ProjectionsCache.getOrCreate(ignite).localPeek(key).get(regIdx % BLOCK_SIZE);
- return ctx.featureProcessor(bestFeatureIdx).calculateOwnershipBitSet(reg, values, best.info);
-
- });
-
- SplitNode sn = best.info.createSplitNode(best.featureIdx);
-
- TreeTip tipToSplit = tips.get(ind);
- tipToSplit.leafSetter.accept(sn);
- tipToSplit.leafSetter = sn::setLeft;
- int d = tipToSplit.depth++;
- tips.add(new TreeTip(sn::setRight, d));
-
- if (d > curDepth) {
- curDepth = d;
- if (log.isDebugEnabled()) {
- log.debug("Depth: " + curDepth);
- log.debug("Cache size: " + prjsCache.size(CachePeekMode.PRIMARY));
- }
- }
-
- before = System.currentTimeMillis();
- // Perform split on all feature vectors.
- IgniteSupplier<Set<RegionKey>> bestRegsKeys = () -> IntStream.range(0, featuresCnt).
- mapToObj(fIdx -> ProjectionsCache.key(fIdx, ind / BLOCK_SIZE, input.affinityKey(fIdx, Ignition.localIgnite()), uuid)).
- collect(Collectors.toSet());
-
- int rc = regsCnt;
-
- // Perform split.
- CacheUtils.update(prjsCache.getName(), ignite,
- (Ignite ign, Cache.Entry<RegionKey, List<RegionProjection>> e) -> {
- RegionKey k = e.getKey();
-
- List<RegionProjection> leftBlock = e.getValue();
-
- int fIdx = k.featureIdx();
- int idxInBlock = ind % BLOCK_SIZE;
-
- IgniteCache<UUID, TrainingContext<D>> ctxCache = ContextCache.getOrCreate(ign);
- TrainingContext<D> ctx = ctxCache.get(uuid);
-
- RegionProjection targetRegProj = leftBlock.get(idxInBlock);
-
- IgniteBiTuple<RegionProjection, RegionProjection> regs = ctx.
- performSplit(input, bs, fIdx, best.featureIdx, targetRegProj, best.info.leftData(), best.info.rightData(), ign);
-
- RegionProjection left = regs.get1();
- RegionProjection right = regs.get2();
-
- leftBlock.set(idxInBlock, left);
- RegionKey rightKey = ProjectionsCache.key(fIdx, (rc - 1) / BLOCK_SIZE, input.affinityKey(fIdx, ign), uuid);
-
- IgniteCache<RegionKey, List<RegionProjection>> c = ProjectionsCache.getOrCreate(ign);
-
- List<RegionProjection> rightBlock = rightKey.equals(k) ? leftBlock : c.localPeek(rightKey);
-
- if (rightBlock == null) {
- List<RegionProjection> newBlock = new ArrayList<>(BLOCK_SIZE);
- newBlock.add(right);
- return Stream.of(new CacheEntryImpl<>(k, leftBlock), new CacheEntryImpl<>(rightKey, newBlock));
- }
- else {
- rightBlock.add(right);
- return rightBlock.equals(k) ?
- Stream.of(new CacheEntryImpl<>(k, leftBlock)) :
- Stream.of(new CacheEntryImpl<>(k, leftBlock), new CacheEntryImpl<>(rightKey, rightBlock));
- }
- },
- bestRegsKeys);
-
- if (log.isDebugEnabled())
- log.debug("Update of projections cache time: " + (System.currentTimeMillis() - before));
-
- before = System.currentTimeMillis();
-
- updateSplitCache(ind, rc, featuresCnt, ig -> i -> input.affinityKey(i, ig), uuid);
-
- if (log.isDebugEnabled())
- log.debug("Update of split cache time: " + (System.currentTimeMillis() - before));
- }
- else {
- if (log.isDebugEnabled())
- log.debug("Best split [bestFeatureIdx=" + bestFeatureIdx + ", bestInfoGain=" + bestInfoGain + "]");
- break;
- }
- }
-
- int rc = regsCnt;
-
- IgniteSupplier<Iterable<Cache.Entry<RegionKey, List<RegionProjection>>>> featZeroRegs = () -> {
- IgniteCache<RegionKey, List<RegionProjection>> projsCache = ProjectionsCache.getOrCreate(Ignition.localIgnite());
-
- return () -> IntStream.range(0, (rc - 1) / BLOCK_SIZE + 1).
- mapToObj(rBIdx -> ProjectionsCache.key(0, rBIdx, input.affinityKey(0, Ignition.localIgnite()), uuid)).
- map(k -> (Cache.Entry<RegionKey, List<RegionProjection>>)new CacheEntryImpl<>(k, projsCache.localPeek(k))).iterator();
- };
-
- Map<Integer, Double> vals = CacheUtils.reduce(prjsCache.getName(), ignite,
- (TrainingContext ctx, Cache.Entry<RegionKey, List<RegionProjection>> e, Map<Integer, Double> m) -> {
- int regBlockIdx = e.getKey().regionBlockIndex();
-
- if (e.getValue() != null) {
- for (int i = 0; i < e.getValue().size(); i++) {
- int regIdx = regBlockIdx * BLOCK_SIZE + i;
- RegionProjection reg = e.getValue().get(i);
-
- Double res = regCalc.apply(Arrays.stream(reg.sampleIndexes()).mapToDouble(s -> ctx.labels()[s]));
- m.put(regIdx, res);
- }
- }
-
- return m;
- },
- () -> ContextCache.getOrCreate(Ignition.localIgnite()).get(uuid),
- featZeroRegs,
- (infos, infos2) -> {
- Map<Integer, Double> res = new HashMap<>();
- res.putAll(infos);
- res.putAll(infos2);
- return res;
- },
- HashMap::new
- );
-
- int i = 0;
- for (TreeTip tip : tips) {
- tip.leafSetter.accept(new Leaf(vals.get(i)));
- i++;
- }
-
- ProjectionsCache.clear(featuresCnt, rc, input::affinityKey, uuid, ignite);
- ContextCache.getOrCreate(ignite).remove(uuid);
- FeaturesCache.clear(featuresCnt, input::affinityKey, uuid, ignite);
- SplitCache.clear(featuresCnt, input::affinityKey, uuid, ignite);
-
- return new DecisionTreeModel(root.s);
- }
-
- /**
- * Find the best split in the form (feature index, (index of region with the best split, impurity of region with the
- * best split)).
- *
- * @param featuresCnt Count of features.
- * @param affinity Affinity function.
- * @param trainingUUID UUID of training.
- * @return Best split in the form (feature index, (index of region with the best split, impurity of region with the
- * best split)).
- */
- private IgniteBiTuple<Integer, IgniteBiTuple<Integer, Double>> findBestSplitIndexForFeatures(int featuresCnt,
- IgniteBiFunction<Integer, Ignite, Object> affinity,
- UUID trainingUUID) {
- Set<Integer> featureIndexes = IntStream.range(0, featuresCnt).boxed().collect(Collectors.toSet());
-
- return CacheUtils.reduce(SplitCache.CACHE_NAME, ignite,
- (Object ctx, Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>> e, IgniteBiTuple<Integer, IgniteBiTuple<Integer, Double>> r) ->
- Functions.MAX_GENERIC(new IgniteBiTuple<>(e.getKey().featureIdx(), e.getValue()), r, comparator()),
- () -> null,
- () -> SplitCache.localEntries(featureIndexes, affinity, trainingUUID),
- (i1, i2) -> Functions.MAX_GENERIC(i1, i2, Comparator.comparingDouble(bt -> bt.get2().get2())),
- () -> new IgniteBiTuple<>(-1, new IgniteBiTuple<>(-1, Double.NEGATIVE_INFINITY))
- );
- }
-
- /** */
- private static Comparator<IgniteBiTuple<Integer, IgniteBiTuple<Integer, Double>>> comparator() {
- return Comparator.comparingDouble(bt -> bt != null && bt.get2() != null ? bt.get2().get2() : Double.NEGATIVE_INFINITY);
- }
-
- /**
- * Update split cache.
- *
- * @param lastSplitRegionIdx Index of region which had last best split.
- * @param regsCnt Count of regions.
- * @param featuresCnt Count of features.
- * @param affinity Affinity function.
- * @param trainingUUID UUID of current training.
- */
- private void updateSplitCache(int lastSplitRegionIdx, int regsCnt, int featuresCnt,
- IgniteCurriedBiFunction<Ignite, Integer, Object> affinity,
- UUID trainingUUID) {
- CacheUtils.update(SplitCache.CACHE_NAME, ignite,
- (Ignite ign, Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>> e) -> {
- Integer bestRegIdx = e.getValue().get1();
- int fIdx = e.getKey().featureIdx();
- TrainingContext ctx = ContextCache.getOrCreate(ign).get(trainingUUID);
-
- Map<Integer, RegionProjection> toCompare;
-
- // Fully recalculate best.
- if (bestRegIdx == lastSplitRegionIdx)
- toCompare = ProjectionsCache.projectionsOfFeature(fIdx, maxDepth, regsCnt, BLOCK_SIZE, affinity.apply(ign), trainingUUID, ign);
- // Just compare previous best and two regions which are produced by split.
- else
- toCompare = ProjectionsCache.projectionsOfRegions(fIdx, maxDepth,
- IntStream.of(bestRegIdx, lastSplitRegionIdx, regsCnt - 1), BLOCK_SIZE, affinity.apply(ign), trainingUUID, ign);
-
- double[] values = ctx.values(fIdx, ign);
- double[] labels = ctx.labels();
-
- Optional<IgniteBiTuple<Integer, Double>> max = toCompare.entrySet().stream().
- map(ent -> {
- SplitInfo bestSplit = ctx.featureProcessor(fIdx).findBestSplit(ent.getValue(), values, labels, ent.getKey());
- return new IgniteBiTuple<>(ent.getKey(), bestSplit != null ? bestSplit.infoGain() : Double.NEGATIVE_INFINITY);
- }).
- max(Comparator.comparingDouble(IgniteBiTuple::get2));
-
- return max.<Stream<Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>>>>
- map(objects -> Stream.of(new CacheEntryImpl<>(e.getKey(), objects))).orElseGet(Stream::empty);
- },
- () -> IntStream.range(0, featuresCnt).mapToObj(fIdx -> SplitCache.key(fIdx, affinity.apply(ignite).apply(fIdx), trainingUUID)).collect(Collectors.toSet())
- );
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainerInput.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainerInput.java
deleted file mode 100644
index bf8790b..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainerInput.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.util.Map;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.lang.IgniteBiTuple;
-
-/**
- * Input for {@link ColumnDecisionTreeTrainer}.
- */
-public interface ColumnDecisionTreeTrainerInput {
- /**
- * Projection of data on feature with the given index.
- *
- * @param idx Feature index.
- * @return Projection of data on feature with the given index.
- */
- Stream<IgniteBiTuple<Integer, Double>> values(int idx);
-
- /**
- * Labels.
- *
- * @param ignite Ignite instance.
- */
- double[] labels(Ignite ignite);
-
- /** Information about which features are categorical in the form of feature index -> number of categories. */
- Map<Integer, Integer> catFeaturesInfo();
-
- /** Number of features. */
- int featuresCount();
-
- /**
- * Get affinity key for the given column index.
- * Affinity key should be pure-functionally dependent from idx.
- */
- Object affinityKey(int idx, Ignite ignite);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/MatrixColumnDecisionTreeTrainerInput.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/MatrixColumnDecisionTreeTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/MatrixColumnDecisionTreeTrainerInput.java
deleted file mode 100644
index 3da6bad..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/MatrixColumnDecisionTreeTrainerInput.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey;
-import org.apache.ignite.ml.math.distributed.keys.impl.SparseMatrixKey;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * Adapter of SparseDistributedMatrix to ColumnDecisionTreeTrainerInput.
- * Sparse SparseDistributedMatrix should be in {@link StorageConstants#COLUMN_STORAGE_MODE} and
- * should contain samples in rows last position in row being label of this sample.
- */
-public class MatrixColumnDecisionTreeTrainerInput extends CacheColumnDecisionTreeTrainerInput<RowColMatrixKey, Map<Integer, Double>> {
- /**
- * @param m Sparse SparseDistributedMatrix should be in {@link StorageConstants#COLUMN_STORAGE_MODE}
- * containing samples in rows last position in row being label of this sample.
- * @param catFeaturesInfo Information about which features are categorical in form of feature index -> number of
- * categories.
- */
- public MatrixColumnDecisionTreeTrainerInput(SparseDistributedMatrix m, Map<Integer, Integer> catFeaturesInfo) {
- super(((SparseDistributedMatrixStorage)m.getStorage()).cache(),
- () -> Stream.of(new SparseMatrixKey(m.columnSize() - 1, m.getUUID(), m.columnSize() - 1)),
- valuesMapper(m),
- labels(m),
- keyMapper(m),
- catFeaturesInfo,
- m.columnSize() - 1,
- m.rowSize());
- }
-
- /** Values mapper. See {@link CacheColumnDecisionTreeTrainerInput#valuesMapper} */
- @NotNull
- private static IgniteFunction<Cache.Entry<RowColMatrixKey, Map<Integer, Double>>, Stream<IgniteBiTuple<Integer, Double>>> valuesMapper(
- SparseDistributedMatrix m) {
- return ent -> {
- Map<Integer, Double> map = ent.getValue() != null ? ent.getValue() : new HashMap<>();
- return IntStream.range(0, m.rowSize()).mapToObj(k -> new IgniteBiTuple<>(k, map.getOrDefault(k, 0.0)));
- };
- }
-
- /** Key mapper. See {@link CacheColumnDecisionTreeTrainerInput#keyMapper} */
- @NotNull private static IgniteFunction<Integer, Stream<RowColMatrixKey>> keyMapper(SparseDistributedMatrix m) {
- return i -> Stream.of(new SparseMatrixKey(i, ((SparseDistributedMatrixStorage)m.getStorage()).getUUID(), i));
- }
-
- /** Labels mapper. See {@link CacheColumnDecisionTreeTrainerInput#labelsMapper} */
- @NotNull private static IgniteFunction<Map<Integer, Double>, DoubleStream> labels(SparseDistributedMatrix m) {
- return mp -> IntStream.range(0, m.rowSize()).mapToDouble(k -> mp.getOrDefault(k, 0.0));
- }
-
- /** {@inheritDoc} */
- @Override public Object affinityKey(int idx, Ignite ignite) {
- return idx;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/RegionProjection.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/RegionProjection.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/RegionProjection.java
deleted file mode 100644
index e95f57b..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/RegionProjection.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import org.apache.ignite.ml.trees.RegionInfo;
-
-/**
- * Projection of region on given feature.
- *
- * @param <D> Data of region.
- */
-public class RegionProjection<D extends RegionInfo> implements Externalizable {
- /** Samples projections. */
- protected Integer[] sampleIndexes;
-
- /** Region data */
- protected D data;
-
- /** Depth of this region. */
- protected int depth;
-
- /**
- * @param sampleIndexes Samples indexes.
- * @param data Region data.
- * @param depth Depth of this region.
- */
- public RegionProjection(Integer[] sampleIndexes, D data, int depth) {
- this.data = data;
- this.depth = depth;
- this.sampleIndexes = sampleIndexes;
- }
-
- /**
- * No-op constructor used for serialization/deserialization.
- */
- public RegionProjection() {
- // No-op.
- }
-
- /**
- * Get samples indexes.
- *
- * @return Samples indexes.
- */
- public Integer[] sampleIndexes() {
- return sampleIndexes;
- }
-
- /**
- * Get region data.
- *
- * @return Region data.
- */
- public D data() {
- return data;
- }
-
- /**
- * Get region depth.
- *
- * @return Region depth.
- */
- public int depth() {
- return depth;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeInt(sampleIndexes.length);
-
- for (Integer sampleIndex : sampleIndexes)
- out.writeInt(sampleIndex);
-
- out.writeObject(data);
- out.writeInt(depth);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- int size = in.readInt();
-
- sampleIndexes = new Integer[size];
-
- for (int i = 0; i < size; i++)
- sampleIndexes[i] = in.readInt();
-
- data = (D)in.readObject();
- depth = in.readInt();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/TrainingContext.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/TrainingContext.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/TrainingContext.java
deleted file mode 100644
index 6415dab..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/TrainingContext.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import java.util.Map;
-import java.util.UUID;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.CategoricalFeatureProcessor;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.ContinuousFeatureProcessor;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.FeatureProcessor;
-
-import static org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache.COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME;
-
-/**
- * Context of training with {@link ColumnDecisionTreeTrainer}.
- *
- * @param <D> Class for storing of information used in calculation of impurity of continuous feature region.
- */
-public class TrainingContext<D extends ContinuousRegionInfo> {
- /** Input for training with {@link ColumnDecisionTreeTrainer}. */
- private final ColumnDecisionTreeTrainerInput input;
-
- /** Labels. */
- private final double[] labels;
-
- /** Calculator used for finding splits of region of continuous features. */
- private final ContinuousSplitCalculator<D> continuousSplitCalculator;
-
- /** Calculator used for finding splits of region of categorical feature. */
- private final IgniteFunction<DoubleStream, Double> categoricalSplitCalculator;
-
- /** UUID of current training. */
- private final UUID trainingUUID;
-
- /**
- * Construct context for training with {@link ColumnDecisionTreeTrainer}.
- *
- * @param input Input for training.
- * @param continuousSplitCalculator Calculator used for calculations of splits of continuous features regions.
- * @param categoricalSplitCalculator Calculator used for calculations of splits of categorical features regions.
- * @param trainingUUID UUID of the current training.
- * @param ignite Ignite instance.
- */
- public TrainingContext(ColumnDecisionTreeTrainerInput input,
- ContinuousSplitCalculator<D> continuousSplitCalculator,
- IgniteFunction<DoubleStream, Double> categoricalSplitCalculator,
- UUID trainingUUID,
- Ignite ignite) {
- this.input = input;
- this.labels = input.labels(ignite);
- this.continuousSplitCalculator = continuousSplitCalculator;
- this.categoricalSplitCalculator = categoricalSplitCalculator;
- this.trainingUUID = trainingUUID;
- }
-
- /**
- * Get processor used for calculating splits of categorical features.
- *
- * @param catsCnt Count of categories.
- * @return Processor used for calculating splits of categorical features.
- */
- public CategoricalFeatureProcessor categoricalFeatureProcessor(int catsCnt) {
- return new CategoricalFeatureProcessor(categoricalSplitCalculator, catsCnt);
- }
-
- /**
- * Get processor used for calculating splits of continuous features.
- *
- * @return Processor used for calculating splits of continuous features.
- */
- public ContinuousFeatureProcessor<D> continuousFeatureProcessor() {
- return new ContinuousFeatureProcessor<>(continuousSplitCalculator);
- }
-
- /**
- * Get labels.
- *
- * @return Labels.
- */
- public double[] labels() {
- return labels;
- }
-
- /**
- * Get values of feature with given index.
- *
- * @param featIdx Feature index.
- * @param ignite Ignite instance.
- * @return Values of feature with given index.
- */
- public double[] values(int featIdx, Ignite ignite) {
- IgniteCache<FeaturesCache.FeatureKey, double[]> featuresCache = ignite.getOrCreateCache(COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME);
- return featuresCache.localPeek(FeaturesCache.getFeatureCacheKey(featIdx, trainingUUID, input.affinityKey(featIdx, ignite)));
- }
-
- /**
- * Perform best split on the given region projection.
- *
- * @param input Input of {@link ColumnDecisionTreeTrainer} performing split.
- * @param bitSet Bit set specifying split.
- * @param targetFeatIdx Index of feature for performing split.
- * @param bestFeatIdx Index of feature with best split.
- * @param targetRegionPrj Projection of region to split on feature with index {@code featureIdx}.
- * @param leftData Data of left region of split.
- * @param rightData Data of right region of split.
- * @param ignite Ignite instance.
- * @return Perform best split on the given region projection.
- */
- public IgniteBiTuple<RegionProjection, RegionProjection> performSplit(ColumnDecisionTreeTrainerInput input,
- SparseBitSet bitSet, int targetFeatIdx, int bestFeatIdx, RegionProjection targetRegionPrj, RegionInfo leftData,
- RegionInfo rightData, Ignite ignite) {
-
- Map<Integer, Integer> catFeaturesInfo = input.catFeaturesInfo();
-
- if (!catFeaturesInfo.containsKey(targetFeatIdx) && !catFeaturesInfo.containsKey(bestFeatIdx))
- return continuousFeatureProcessor().performSplit(bitSet, targetRegionPrj, (D)leftData, (D)rightData);
- else if (catFeaturesInfo.containsKey(targetFeatIdx))
- return categoricalFeatureProcessor(catFeaturesInfo.get(targetFeatIdx)).performSplitGeneric(bitSet, values(targetFeatIdx, ignite), targetRegionPrj, leftData, rightData);
- return continuousFeatureProcessor().performSplitGeneric(bitSet, labels, targetRegionPrj, leftData, rightData);
- }
-
- /**
- * Processor used for calculating splits for feature with the given index.
- *
- * @param featureIdx Index of feature to process.
- * @return Processor used for calculating splits for feature with the given index.
- */
- public FeatureProcessor featureProcessor(int featureIdx) {
- return input.catFeaturesInfo().containsKey(featureIdx) ? categoricalFeatureProcessor(input.catFeaturesInfo().get(featureIdx)) : continuousFeatureProcessor();
- }
-
- /**
- * Shortcut for affinity key.
- *
- * @param idx Feature index.
- * @return Affinity key.
- */
- public Object affinityKey(int idx) {
- return input.affinityKey(idx, Ignition.localIgnite());
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ContextCache.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ContextCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ContextCache.java
deleted file mode 100644
index 51ea359..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ContextCache.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
-
-import java.util.UUID;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.TrainingContext;
-
-/**
- * Class for operations related to cache containing training context for {@link ColumnDecisionTreeTrainer}.
- */
-public class ContextCache {
- /**
- * Name of cache containing training context for {@link ColumnDecisionTreeTrainer}.
- */
- public static final String COLUMN_DECISION_TREE_TRAINER_CONTEXT_CACHE_NAME = "COLUMN_DECISION_TREE_TRAINER_CONTEXT_CACHE_NAME";
-
- /**
- * Get or create cache for training context.
- *
- * @param ignite Ignite instance.
- * @param <D> Class storing information about continuous regions.
- * @return Cache for training context.
- */
- public static <D extends ContinuousRegionInfo> IgniteCache<UUID, TrainingContext<D>> getOrCreate(Ignite ignite) {
- CacheConfiguration<UUID, TrainingContext<D>> cfg = new CacheConfiguration<>();
-
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
-
- cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
- cfg.setEvictionPolicy(null);
-
- cfg.setCopyOnRead(false);
-
- cfg.setCacheMode(CacheMode.REPLICATED);
-
- cfg.setOnheapCacheEnabled(true);
-
- cfg.setReadFromBackup(true);
-
- cfg.setName(COLUMN_DECISION_TREE_TRAINER_CONTEXT_CACHE_NAME);
-
- return ignite.getOrCreateCache(cfg);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/FeaturesCache.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/FeaturesCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/FeaturesCache.java
deleted file mode 100644
index fcc1f16..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/FeaturesCache.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
-
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-
-/**
- * Cache storing features for {@link ColumnDecisionTreeTrainer}.
- */
-public class FeaturesCache {
- /**
- * Name of cache which is used for storing features for {@link ColumnDecisionTreeTrainer}.
- */
- public static final String COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME = "COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME";
-
- /**
- * Key of features cache.
- */
- public static class FeatureKey {
- /** Column key of cache used as input for {@link ColumnDecisionTreeTrainer}. */
- @AffinityKeyMapped
- private Object parentColKey;
-
- /** Index of feature. */
- private final int featureIdx;
-
- /** UUID of training. */
- private final UUID trainingUUID;
-
- /**
- * Construct FeatureKey.
- *
- * @param featureIdx Feature index.
- * @param trainingUUID UUID of training.
- * @param parentColKey Column key of cache used as input.
- */
- public FeatureKey(int featureIdx, UUID trainingUUID, Object parentColKey) {
- this.parentColKey = parentColKey;
- this.featureIdx = featureIdx;
- this.trainingUUID = trainingUUID;
- this.parentColKey = parentColKey;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- FeatureKey key = (FeatureKey)o;
-
- if (featureIdx != key.featureIdx)
- return false;
- return trainingUUID != null ? trainingUUID.equals(key.trainingUUID) : key.trainingUUID == null;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = trainingUUID != null ? trainingUUID.hashCode() : 0;
- res = 31 * res + featureIdx;
- return res;
- }
- }
-
- /**
- * Create new projections cache for ColumnDecisionTreeTrainer if needed.
- *
- * @param ignite Ignite instance.
- */
- public static IgniteCache<FeatureKey, double[]> getOrCreate(Ignite ignite) {
- CacheConfiguration<FeatureKey, double[]> cfg = new CacheConfiguration<>();
-
- // Write to primary.
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
- // Atomic transactions only.
- cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
- // No eviction.
- cfg.setEvictionPolicy(null);
-
- // No copying of values.
- cfg.setCopyOnRead(false);
-
- // Cache is partitioned.
- cfg.setCacheMode(CacheMode.PARTITIONED);
-
- cfg.setOnheapCacheEnabled(true);
-
- cfg.setBackups(0);
-
- cfg.setName(COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME);
-
- return ignite.getOrCreateCache(cfg);
- }
-
- /**
- * Construct FeatureKey from index, uuid and affinity key.
- *
- * @param idx Feature index.
- * @param uuid UUID of training.
- * @param aff Affinity key.
- * @return FeatureKey.
- */
- public static FeatureKey getFeatureCacheKey(int idx, UUID uuid, Object aff) {
- return new FeatureKey(idx, uuid, aff);
- }
-
- /**
- * Clear all data from features cache related to given training.
- *
- * @param featuresCnt Count of features.
- * @param affinity Affinity function.
- * @param uuid Training uuid.
- * @param ignite Ignite instance.
- */
- public static void clear(int featuresCnt, IgniteBiFunction<Integer, Ignite, Object> affinity, UUID uuid,
- Ignite ignite) {
- Set<FeatureKey> toRmv = IntStream.range(0, featuresCnt).boxed().map(fIdx -> getFeatureCacheKey(fIdx, uuid, affinity.apply(fIdx, ignite))).collect(Collectors.toSet());
-
- getOrCreate(ignite).removeAll(toRmv);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ProjectionsCache.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ProjectionsCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ProjectionsCache.java
deleted file mode 100644
index 080cb66..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ProjectionsCache.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.PrimitiveIterator;
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.RegionProjection;
-
-/**
- * Cache used for storing data of region projections on features.
- */
-public class ProjectionsCache {
- /**
- * Name of cache which is used for storing data of region projections on features of {@link
- * ColumnDecisionTreeTrainer}.
- */
- public static final String CACHE_NAME = "COLUMN_DECISION_TREE_TRAINER_PROJECTIONS_CACHE_NAME";
-
- /**
- * Key of region projections cache.
- */
- public static class RegionKey {
- /** Column key of cache used as input for {@link ColumnDecisionTreeTrainer}. */
- @AffinityKeyMapped
- private final Object parentColKey;
-
- /** Feature index. */
- private final int featureIdx;
-
- /** Region index. */
- private final int regBlockIdx;
-
- /** Training UUID. */
- private final UUID trainingUUID;
-
- /**
- * Construct a RegionKey from feature index, index of block, key of column in input cache and UUID of training.
- *
- * @param featureIdx Feature index.
- * @param regBlockIdx Index of block.
- * @param parentColKey Key of column in input cache.
- * @param trainingUUID UUID of training.
- */
- public RegionKey(int featureIdx, int regBlockIdx, Object parentColKey, UUID trainingUUID) {
- this.featureIdx = featureIdx;
- this.regBlockIdx = regBlockIdx;
- this.trainingUUID = trainingUUID;
- this.parentColKey = parentColKey;
- }
-
- /**
- * Feature index.
- *
- * @return Feature index.
- */
- public int featureIdx() {
- return featureIdx;
- }
-
- /**
- * Region block index.
- *
- * @return Region block index.
- */
- public int regionBlockIndex() {
- return regBlockIdx;
- }
-
- /**
- * UUID of training.
- *
- * @return UUID of training.
- */
- public UUID trainingUUID() {
- return trainingUUID;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- RegionKey key = (RegionKey)o;
-
- if (featureIdx != key.featureIdx)
- return false;
- if (regBlockIdx != key.regBlockIdx)
- return false;
- return trainingUUID != null ? trainingUUID.equals(key.trainingUUID) : key.trainingUUID == null;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = trainingUUID != null ? trainingUUID.hashCode() : 0;
- res = 31 * res + featureIdx;
- res = 31 * res + regBlockIdx;
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "RegionKey [" +
- "parentColKey=" + parentColKey +
- ", featureIdx=" + featureIdx +
- ", regBlockIdx=" + regBlockIdx +
- ", trainingUUID=" + trainingUUID +
- ']';
- }
- }
-
- /**
- * Affinity service for region projections cache.
- *
- * @return Affinity service for region projections cache.
- */
- public static Affinity<RegionKey> affinity() {
- return Ignition.localIgnite().affinity(CACHE_NAME);
- }
-
- /**
- * Get or create region projections cache.
- *
- * @param ignite Ignite instance.
- * @return Region projections cache.
- */
- public static IgniteCache<RegionKey, List<RegionProjection>> getOrCreate(Ignite ignite) {
- CacheConfiguration<RegionKey, List<RegionProjection>> cfg = new CacheConfiguration<>();
-
- // Write to primary.
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
- // Atomic transactions only.
- cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
- // No eviction.
- cfg.setEvictionPolicy(null);
-
- // No copying of values.
- cfg.setCopyOnRead(false);
-
- // Cache is partitioned.
- cfg.setCacheMode(CacheMode.PARTITIONED);
-
- cfg.setBackups(0);
-
- cfg.setOnheapCacheEnabled(true);
-
- cfg.setName(CACHE_NAME);
-
- return ignite.getOrCreateCache(cfg);
- }
-
- /**
- * Get region projections in the form of map (regionIndex -> regionProjections).
- *
- * @param featureIdx Feature index.
- * @param maxDepth Max depth of decision tree.
- * @param regionIndexes Indexes of regions for which we want get projections.
- * @param blockSize Size of regions block.
- * @param affinity Affinity function.
- * @param trainingUUID UUID of training.
- * @param ignite Ignite instance.
- * @return Region projections in the form of map (regionIndex -> regionProjections).
- */
- public static Map<Integer, RegionProjection> projectionsOfRegions(int featureIdx, int maxDepth,
- IntStream regionIndexes, int blockSize, IgniteFunction<Integer, Object> affinity, UUID trainingUUID,
- Ignite ignite) {
- HashMap<Integer, RegionProjection> regsForSearch = new HashMap<>();
- IgniteCache<RegionKey, List<RegionProjection>> cache = getOrCreate(ignite);
-
- PrimitiveIterator.OfInt itr = regionIndexes.iterator();
-
- int curBlockIdx = -1;
- List<RegionProjection> block = null;
-
- Object affinityKey = affinity.apply(featureIdx);
-
- while (itr.hasNext()) {
- int i = itr.nextInt();
-
- int blockIdx = i / blockSize;
-
- if (blockIdx != curBlockIdx) {
- block = cache.localPeek(key(featureIdx, blockIdx, affinityKey, trainingUUID));
- curBlockIdx = blockIdx;
- }
-
- if (block == null)
- throw new IllegalStateException("Unexpected null block at index " + i);
-
- RegionProjection reg = block.get(i % blockSize);
-
- if (reg.depth() < maxDepth)
- regsForSearch.put(i, reg);
- }
-
- return regsForSearch;
- }
-
- /**
- * Returns projections of regions on given feature filtered by maximal depth in the form of (region index -> region
- * projection).
- *
- * @param featureIdx Feature index.
- * @param maxDepth Maximal depth of the tree.
- * @param regsCnt Count of regions.
- * @param blockSize Size of regions blocks.
- * @param affinity Affinity function.
- * @param trainingUUID UUID of training.
- * @param ignite Ignite instance.
- * @return Projections of regions on given feature filtered by maximal depth in the form of (region index -> region
- * projection).
- */
- public static Map<Integer, RegionProjection> projectionsOfFeature(int featureIdx, int maxDepth, int regsCnt,
- int blockSize, IgniteFunction<Integer, Object> affinity, UUID trainingUUID, Ignite ignite) {
- return projectionsOfRegions(featureIdx, maxDepth, IntStream.range(0, regsCnt), blockSize, affinity, trainingUUID, ignite);
- }
-
- /**
- * Construct key for projections cache.
- *
- * @param featureIdx Feature index.
- * @param regBlockIdx Region block index.
- * @param parentColKey Column key of cache used as input for {@link ColumnDecisionTreeTrainer}.
- * @param uuid UUID of training.
- * @return Key for projections cache.
- */
- public static RegionKey key(int featureIdx, int regBlockIdx, Object parentColKey, UUID uuid) {
- return new RegionKey(featureIdx, regBlockIdx, parentColKey, uuid);
- }
-
- /**
- * Clear data from projections cache related to given training.
- *
- * @param featuresCnt Features count.
- * @param regs Regions count.
- * @param aff Affinity function.
- * @param uuid UUID of training.
- * @param ignite Ignite instance.
- */
- public static void clear(int featuresCnt, int regs, IgniteBiFunction<Integer, Ignite, Object> aff, UUID uuid,
- Ignite ignite) {
- Set<RegionKey> toRmv = IntStream.range(0, featuresCnt).boxed().
- flatMap(fIdx -> IntStream.range(0, regs).boxed().map(reg -> new IgniteBiTuple<>(fIdx, reg))).
- map(t -> key(t.get1(), t.get2(), aff.apply(t.get1(), ignite), uuid)).
- collect(Collectors.toSet());
-
- getOrCreate(ignite).removeAll(toRmv);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/SplitCache.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/SplitCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/SplitCache.java
deleted file mode 100644
index ecbc861..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/SplitCache.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Set;
-import java.util.UUID;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-
-/**
- * Class for working with cache used for storing of best splits during training with {@link ColumnDecisionTreeTrainer}.
- */
-public class SplitCache {
- /** Name of splits cache. */
- public static final String CACHE_NAME = "COLUMN_DECISION_TREE_TRAINER_SPLIT_CACHE_NAME";
-
- /**
- * Class used for keys in the splits cache.
- */
- public static class SplitKey {
- /** UUID of current training. */
- private final UUID trainingUUID;
-
- /** Affinity key of input data. */
- @AffinityKeyMapped
- private final Object parentColKey;
-
- /** Index of feature by which the split is made. */
- private final int featureIdx;
-
- /**
- * Construct SplitKey.
- *
- * @param trainingUUID UUID of the training.
- * @param parentColKey Affinity key used to ensure that cache entry for given feature will be on the same node
- * as column with that feature in input.
- * @param featureIdx Feature index.
- */
- public SplitKey(UUID trainingUUID, Object parentColKey, int featureIdx) {
- this.trainingUUID = trainingUUID;
- this.featureIdx = featureIdx;
- this.parentColKey = parentColKey;
- }
-
- /** Get UUID of current training. */
- public UUID trainingUUID() {
- return trainingUUID;
- }
-
- /**
- * Get feature index.
- *
- * @return Feature index.
- */
- public int featureIdx() {
- return featureIdx;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- SplitKey splitKey = (SplitKey)o;
-
- if (featureIdx != splitKey.featureIdx)
- return false;
- return trainingUUID != null ? trainingUUID.equals(splitKey.trainingUUID) : splitKey.trainingUUID == null;
-
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = trainingUUID != null ? trainingUUID.hashCode() : 0;
- res = 31 * res + featureIdx;
- return res;
- }
- }
-
- /**
- * Construct the key for splits cache.
- *
- * @param featureIdx Feature index.
- * @param parentColKey Affinity key used to ensure that cache entry for given feature will be on the same node as
- * column with that feature in input.
- * @param uuid UUID of current training.
- * @return Key for splits cache.
- */
- public static SplitKey key(int featureIdx, Object parentColKey, UUID uuid) {
- return new SplitKey(uuid, parentColKey, featureIdx);
- }
-
- /**
- * Get or create splits cache.
- *
- * @param ignite Ignite instance.
- * @return Splits cache.
- */
- public static IgniteCache<SplitKey, IgniteBiTuple<Integer, Double>> getOrCreate(Ignite ignite) {
- CacheConfiguration<SplitKey, IgniteBiTuple<Integer, Double>> cfg = new CacheConfiguration<>();
-
- // Write to primary.
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
- // Atomic transactions only.
- cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
- // No eviction.
- cfg.setEvictionPolicy(null);
-
- // No copying of values.
- cfg.setCopyOnRead(false);
-
- // Cache is partitioned.
- cfg.setCacheMode(CacheMode.PARTITIONED);
-
- cfg.setBackups(0);
-
- cfg.setOnheapCacheEnabled(true);
-
- cfg.setName(CACHE_NAME);
-
- return ignite.getOrCreateCache(cfg);
- }
-
- /**
- * Affinity function used in splits cache.
- *
- * @return Affinity function used in splits cache.
- */
- public static Affinity<SplitKey> affinity() {
- return Ignition.localIgnite().affinity(CACHE_NAME);
- }
-
- /**
- * Returns local entries for keys corresponding to {@code featureIndexes}.
- *
- * @param featureIndexes Index of features.
- * @param affinity Affinity function.
- * @param trainingUUID UUID of training.
- * @return local entries for keys corresponding to {@code featureIndexes}.
- */
- public static Iterable<Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>>> localEntries(
- Set<Integer> featureIndexes,
- IgniteBiFunction<Integer, Ignite, Object> affinity,
- UUID trainingUUID) {
- Ignite ignite = Ignition.localIgnite();
- Set<SplitKey> keys = featureIndexes.stream().map(fIdx -> new SplitKey(trainingUUID, affinity.apply(fIdx, ignite), fIdx)).collect(Collectors.toSet());
-
- Collection<SplitKey> locKeys = affinity().mapKeysToNodes(keys).getOrDefault(ignite.cluster().localNode(), Collections.emptyList());
-
- return () -> {
- Function<SplitKey, Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>>> f = k -> (new CacheEntryImpl<>(k, getOrCreate(ignite).localPeek(k)));
- return locKeys.stream().map(f).iterator();
- };
- }
-
- /**
- * Clears data related to current training from splits cache related to given training.
- *
- * @param featuresCnt Count of features.
- * @param affinity Affinity function.
- * @param uuid UUID of the given training.
- * @param ignite Ignite instance.
- */
- public static void clear(int featuresCnt, IgniteBiFunction<Integer, Ignite, Object> affinity, UUID uuid,
- Ignite ignite) {
- Set<SplitKey> toRmv = IntStream.range(0, featuresCnt).boxed().map(fIdx -> new SplitKey(uuid, affinity.apply(fIdx, ignite), fIdx)).collect(Collectors.toSet());
-
- getOrCreate(ignite).removeAll(toRmv);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/package-info.java
deleted file mode 100644
index 0a488ab..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains cache configurations for columnbased decision tree trainer with some related logic.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/ContinuousSplitCalculators.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/ContinuousSplitCalculators.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/ContinuousSplitCalculators.java
deleted file mode 100644
index 9fd4c66..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/ContinuousSplitCalculators.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.functions.IgniteCurriedBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-
-/** Continuous Split Calculators. */
-public class ContinuousSplitCalculators {
- /** Variance split calculator. */
- public static IgniteFunction<ColumnDecisionTreeTrainerInput, VarianceSplitCalculator> VARIANCE = input ->
- new VarianceSplitCalculator();
-
- /** Gini split calculator. */
- public static IgniteCurriedBiFunction<Ignite, ColumnDecisionTreeTrainerInput, GiniSplitCalculator> GINI = ignite ->
- input -> new GiniSplitCalculator(input.labels(ignite));
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/GiniSplitCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/GiniSplitCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/GiniSplitCalculator.java
deleted file mode 100644
index 259c84c..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/GiniSplitCalculator.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs;
-
-import it.unimi.dsi.fastutil.doubles.Double2IntArrayMap;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.PrimitiveIterator;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.ContinuousSplitInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-
-/**
- * Calculator for Gini impurity.
- */
-public class GiniSplitCalculator implements ContinuousSplitCalculator<GiniSplitCalculator.GiniData> {
- /** Mapping assigning index to each member value */
- private final Map<Double, Integer> mapping = new Double2IntArrayMap();
-
- /**
- * Create Gini split calculator from labels.
- *
- * @param labels Labels.
- */
- public GiniSplitCalculator(double[] labels) {
- int i = 0;
-
- for (double label : labels) {
- if (!mapping.containsKey(label)) {
- mapping.put(label, i);
- i++;
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public GiniData calculateRegionInfo(DoubleStream s, int l) {
- PrimitiveIterator.OfDouble itr = s.iterator();
-
- Map<Double, Integer> m = new HashMap<>();
-
- int size = 0;
-
- while (itr.hasNext()) {
- size++;
- m.compute(itr.next(), (a, i) -> i != null ? i + 1 : 1);
- }
-
- double c2 = m.values().stream().mapToDouble(v -> v * v).sum();
-
- int[] cnts = new int[mapping.size()];
-
- m.forEach((key, value) -> cnts[mapping.get(key)] = value);
-
- return new GiniData(size != 0 ? 1 - c2 / (size * size) : 0.0, size, cnts, c2);
- }
-
- /** {@inheritDoc} */
- @Override public SplitInfo<GiniData> splitRegion(Integer[] s, double[] values, double[] labels, int regionIdx,
- GiniData d) {
- int size = d.getSize();
-
- double lg = 0.0;
- double rg = d.impurity();
-
- double lc2 = 0.0;
- double rc2 = d.c2;
- int lSize = 0;
-
- double minImpurity = d.impurity() * size;
- double curThreshold;
- double curImpurity;
- double threshold = Double.NEGATIVE_INFINITY;
-
- int i = 0;
- int nextIdx = s[0];
- i++;
- double[] lrImps = new double[] {0.0, d.impurity(), lc2, rc2};
-
- int[] lMapCur = new int[d.counts().length];
- int[] rMapCur = new int[d.counts().length];
-
- System.arraycopy(d.counts(), 0, rMapCur, 0, d.counts().length);
-
- int[] lMap = new int[d.counts().length];
- int[] rMap = new int[d.counts().length];
-
- System.arraycopy(d.counts(), 0, rMap, 0, d.counts().length);
-
- do {
- // Process all values equal to prev.
- while (i < s.length) {
- moveLeft(labels[nextIdx], i, size - i, lMapCur, rMapCur, lrImps);
- curImpurity = (i * lrImps[0] + (size - i) * lrImps[1]);
- curThreshold = values[nextIdx];
-
- if (values[nextIdx] != values[(nextIdx = s[i++])]) {
- if (curImpurity < minImpurity) {
- lSize = i - 1;
-
- lg = lrImps[0];
- rg = lrImps[1];
-
- lc2 = lrImps[2];
- rc2 = lrImps[3];
-
- System.arraycopy(lMapCur, 0, lMap, 0, lMapCur.length);
- System.arraycopy(rMapCur, 0, rMap, 0, rMapCur.length);
-
- minImpurity = curImpurity;
- threshold = curThreshold;
- }
-
- break;
- }
- }
- }
- while (i < s.length - 1);
-
- if (lSize == size || lSize == 0)
- return null;
-
- GiniData lData = new GiniData(lg, lSize, lMap, lc2);
- int rSize = size - lSize;
- GiniData rData = new GiniData(rg, rSize, rMap, rc2);
-
- return new ContinuousSplitInfo<>(regionIdx, threshold, lData, rData);
- }
-
- /**
- * Add point to the left interval and remove it from the right interval and calculate necessary statistics on
- * intervals with new bounds.
- */
- private void moveLeft(double x, int lSize, int rSize, int[] lMap, int[] rMap, double[] data) {
- double lc2 = data[2];
- double rc2 = data[3];
-
- Integer idx = mapping.get(x);
-
- int cxl = lMap[idx];
- int cxr = rMap[idx];
-
- lc2 += 2 * cxl + 1;
- rc2 -= 2 * cxr - 1;
-
- lMap[idx] += 1;
- rMap[idx] -= 1;
-
- data[0] = 1 - lc2 / (lSize * lSize);
- data[1] = 1 - rc2 / (rSize * rSize);
-
- data[2] = lc2;
- data[3] = rc2;
- }
-
- /**
- * Data used for gini impurity calculations.
- */
- public static class GiniData extends ContinuousRegionInfo {
- /** Sum of squares of counts of each label. */
- private double c2;
-
- /** Counts of each label. On i-th position there is count of label which is mapped to index i. */
- private int[] m;
-
- /**
- * Create Gini data.
- *
- * @param impurity Impurity (i.e. Gini impurity).
- * @param size Count of samples.
- * @param m Counts of each label.
- * @param c2 Sum of squares of counts of each label.
- */
- public GiniData(double impurity, int size, int[] m, double c2) {
- super(impurity, size);
- this.m = m;
- this.c2 = c2;
- }
-
- /**
- * No-op constructor for serialization/deserialization..
- */
- public GiniData() {
- // No-op.
- }
-
- /** Get counts of each label. */
- public int[] counts() {
- return m;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- super.writeExternal(out);
- out.writeDouble(c2);
- out.writeInt(m.length);
- for (int i : m)
- out.writeInt(i);
-
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- super.readExternal(in);
-
- c2 = in.readDouble();
- int size = in.readInt();
- m = new int[size];
-
- for (int i = 0; i < size; i++)
- m[i] = in.readInt();
- }
- }
-}
[32/54] [abbrv] ignite git commit: IGNITE-4091 Web Console:
Refactored using of internal Angular API.
Posted by ag...@apache.org.
IGNITE-4091 Web Console: Refactored using of internal Angular API.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/74d25456
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/74d25456
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/74d25456
Branch: refs/heads/ignite-6083
Commit: 74d254564a44a95db9945652c9b579ed6b431ee9
Parents: 9752466
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Wed Apr 11 17:09:41 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Apr 11 17:09:41 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/app/app.config.js | 14 +++---
.../components/modal-import-models/component.js | 4 +-
.../app/components/page-profile/controller.js | 4 +-
.../frontend/app/modules/ace.module.js | 47 ++++++++++----------
.../services/AngularStrapSelect.decorator.js | 5 ++-
.../services/AngularStrapTooltip.decorator.js | 8 ++--
.../frontend/app/services/FormUtils.service.js | 3 +-
7 files changed, 45 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/74d25456/modules/web-console/frontend/app/app.config.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/app.config.js b/modules/web-console/frontend/app/app.config.js
index 9d8dc99..e2bc057 100644
--- a/modules/web-console/frontend/app/app.config.js
+++ b/modules/web-console/frontend/app/app.config.js
@@ -43,7 +43,7 @@ igniteConsoleCfg.config(['$animateProvider', ($animateProvider) => {
// AngularStrap modal popup configuration.
igniteConsoleCfg.config(['$modalProvider', ($modalProvider) => {
- angular.extend($modalProvider.defaults, {
+ Object.assign($modalProvider.defaults, {
animation: 'am-fade-and-scale',
placement: 'center',
html: true
@@ -52,7 +52,7 @@ igniteConsoleCfg.config(['$modalProvider', ($modalProvider) => {
// AngularStrap popover configuration.
igniteConsoleCfg.config(['$popoverProvider', ($popoverProvider) => {
- angular.extend($popoverProvider.defaults, {
+ Object.assign($popoverProvider.defaults, {
trigger: 'manual',
placement: 'right',
container: 'body',
@@ -62,7 +62,7 @@ igniteConsoleCfg.config(['$popoverProvider', ($popoverProvider) => {
// AngularStrap tooltips configuration.
igniteConsoleCfg.config(['$tooltipProvider', ($tooltipProvider) => {
- angular.extend($tooltipProvider.defaults, {
+ Object.assign($tooltipProvider.defaults, {
container: 'body',
delay: {show: 150, hide: 150},
placement: 'right',
@@ -73,7 +73,7 @@ igniteConsoleCfg.config(['$tooltipProvider', ($tooltipProvider) => {
// AngularStrap select (combobox) configuration.
igniteConsoleCfg.config(['$selectProvider', ($selectProvider) => {
- angular.extend($selectProvider.defaults, {
+ Object.assign($selectProvider.defaults, {
container: 'body',
maxLength: '5',
allText: 'Select All',
@@ -87,7 +87,7 @@ igniteConsoleCfg.config(['$selectProvider', ($selectProvider) => {
// AngularStrap alerts configuration.
igniteConsoleCfg.config(['$alertProvider', ($alertProvider) => {
- angular.extend($alertProvider.defaults, {
+ Object.assign($alertProvider.defaults, {
container: 'body',
placement: 'top-right',
duration: '5',
@@ -99,7 +99,7 @@ igniteConsoleCfg.config(['$alertProvider', ($alertProvider) => {
// AngularStrap dropdowns () configuration.
igniteConsoleCfg.config(['$dropdownProvider', ($dropdownProvider) => {
- angular.extend($dropdownProvider.defaults, {
+ Object.assign($dropdownProvider.defaults, {
templateUrl: dropdownTemplateUrl,
animation: ''
});
@@ -107,7 +107,7 @@ igniteConsoleCfg.config(['$dropdownProvider', ($dropdownProvider) => {
// AngularStrap dropdowns () configuration.
igniteConsoleCfg.config(['$datepickerProvider', ($datepickerProvider) => {
- angular.extend($datepickerProvider.defaults, {
+ Object.assign($datepickerProvider.defaults, {
autoclose: true,
iconLeft: 'icon-datepicker-left',
iconRight: 'icon-datepicker-right'
http://git-wip-us.apache.org/repos/asf/ignite/blob/74d25456/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
index 7f852b0..813c998 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
@@ -84,7 +84,7 @@ const DFLT_REPLICATED_CACHE = {
const CACHE_TEMPLATES = [DFLT_PARTITIONED_CACHE, DFLT_REPLICATED_CACHE];
export class ModalImportModels {
- /**
+ /**
* Cluster ID to import models into
* @type {string}
*/
@@ -771,7 +771,7 @@ export class ModalImportModels {
// Prepare caches for generation.
if (table.action === IMPORT_DM_NEW_CACHE) {
- const newCache = angular.copy(this.loadedCaches[table.cacheOrTemplate]);
+ const newCache = _.cloneDeep(this.loadedCaches[table.cacheOrTemplate]);
batchAction.newCache = newCache;
http://git-wip-us.apache.org/repos/asf/ignite/blob/74d25456/modules/web-console/frontend/app/components/page-profile/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-profile/controller.js b/modules/web-console/frontend/app/components/page-profile/controller.js
index 05fe118..c67a603 100644
--- a/modules/web-console/frontend/app/components/page-profile/controller.js
+++ b/modules/web-console/frontend/app/components/page-profile/controller.js
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import _ from 'lodash';
+
export default class PageProfileController {
static $inject = [
'$rootScope', '$scope', '$http', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteCountries', 'User'
@@ -28,7 +30,7 @@ export default class PageProfileController {
this.ui = {};
this.User.read()
- .then((user) => this.ui.user = angular.copy(user));
+ .then((user) => this.ui.user = _.cloneDeep(user));
this.ui.countries = this.Countries.getAll();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/74d25456/modules/web-console/frontend/app/modules/ace.module.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/ace.module.js b/modules/web-console/frontend/app/modules/ace.module.js
index a28536a..6a6e70a 100644
--- a/modules/web-console/frontend/app/modules/ace.module.js
+++ b/modules/web-console/frontend/app/modules/ace.module.js
@@ -16,12 +16,13 @@
*/
import angular from 'angular';
+import _ from 'lodash';
angular
.module('ignite-console.ace', [])
.constant('igniteAceConfig', {})
.directive('igniteAce', ['igniteAceConfig', (aceConfig) => {
- if (angular.isUndefined(window.ace))
+ if (_.isUndefined(window.ace))
throw new Error('ignite-ace need ace to work... (o rly?)');
/**
@@ -43,7 +44,7 @@ angular
*/
const setOptions = (acee, session, opts) => {
// Sets the ace worker path, if running from concatenated or minified source.
- if (angular.isDefined(opts.workerPath)) {
+ if (!_.isUndefined(opts.workerPath)) {
const config = window.ace.acequire('ace/config');
config.set('workerPath', opts.workerPath);
@@ -53,26 +54,26 @@ angular
_.forEach(opts.require, (n) => window.ace.acequire(n));
// Boolean options.
- if (angular.isDefined(opts.showGutter))
+ if (!_.isUndefined(opts.showGutter))
acee.renderer.setShowGutter(opts.showGutter);
- if (angular.isDefined(opts.useWrapMode))
+ if (!_.isUndefined(opts.useWrapMode))
session.setUseWrapMode(opts.useWrapMode);
- if (angular.isDefined(opts.showInvisibles))
+ if (!_.isUndefined(opts.showInvisibles))
acee.renderer.setShowInvisibles(opts.showInvisibles);
- if (angular.isDefined(opts.showIndentGuides))
+ if (!_.isUndefined(opts.showIndentGuides))
acee.renderer.setDisplayIndentGuides(opts.showIndentGuides);
- if (angular.isDefined(opts.useSoftTabs))
+ if (!_.isUndefined(opts.useSoftTabs))
session.setUseSoftTabs(opts.useSoftTabs);
- if (angular.isDefined(opts.showPrintMargin))
+ if (!_.isUndefined(opts.showPrintMargin))
acee.setShowPrintMargin(opts.showPrintMargin);
// Commands.
- if (angular.isDefined(opts.disableSearch) && opts.disableSearch) {
+ if (!_.isUndefined(opts.disableSearch) && opts.disableSearch) {
acee.commands.addCommands([{
name: 'unfind',
bindKey: {
@@ -85,21 +86,21 @@ angular
}
// Base options.
- if (angular.isString(opts.theme))
+ if (_.isString(opts.theme))
acee.setTheme('ace/theme/' + opts.theme);
- if (angular.isString(opts.mode))
+ if (_.isString(opts.mode))
session.setMode('ace/mode/' + opts.mode);
- if (angular.isDefined(opts.firstLineNumber)) {
- if (angular.isNumber(opts.firstLineNumber))
+ if (!_.isUndefined(opts.firstLineNumber)) {
+ if (_.isNumber(opts.firstLineNumber))
session.setOption('firstLineNumber', opts.firstLineNumber);
- else if (angular.isFunction(opts.firstLineNumber))
+ else if (_.isFunction(opts.firstLineNumber))
session.setOption('firstLineNumber', opts.firstLineNumber());
}
// Advanced options.
- if (angular.isDefined(opts.advanced)) {
+ if (!_.isUndefined(opts.advanced)) {
for (const key in opts.advanced) {
if (opts.advanced.hasOwnProperty(key)) {
// Create a javascript object with the key and value.
@@ -112,7 +113,7 @@ angular
}
// Advanced options for the renderer.
- if (angular.isDefined(opts.rendererOptions)) {
+ if (!_.isUndefined(opts.rendererOptions)) {
for (const key in opts.rendererOptions) {
if (opts.rendererOptions.hasOwnProperty(key)) {
// Create a javascript object with the key and value.
@@ -126,7 +127,7 @@ angular
// onLoad callbacks.
_.forEach(opts.callbacks, (cb) => {
- if (angular.isFunction(cb))
+ if (_.isFunction(cb))
cb(acee);
});
};
@@ -147,7 +148,7 @@ angular
*
* @type object
*/
- let opts = angular.extend({}, options, scope.$eval(attrs.igniteAce));
+ let opts = Object.assign({}, options, scope.$eval(attrs.igniteAce));
/**
* ACE editor.
@@ -191,9 +192,9 @@ angular
!scope.$$phase && !scope.$root.$$phase)
scope.$eval(() => ngModel.$setViewValue(newValue));
- if (angular.isDefined(callback)) {
+ if (!_.isUndefined(callback)) {
scope.$evalAsync(() => {
- if (angular.isFunction(callback))
+ if (_.isFunction(callback))
callback([e, acee]);
else
throw new Error('ignite-ace use a function as callback');
@@ -210,10 +211,10 @@ angular
form && form.$removeControl(ngModel);
ngModel.$formatters.push((value) => {
- if (angular.isUndefined(value) || value === null)
+ if (_.isUndefined(value) || value === null)
return '';
- if (angular.isObject(value) || angular.isArray(value))
+ if (_.isObject(value) || _.isArray(value))
throw new Error('ignite-ace cannot use an object or an array as a model');
return value;
@@ -229,7 +230,7 @@ angular
if (current === previous)
return;
- opts = angular.extend({}, options, scope.$eval(attrs.igniteAce));
+ opts = Object.assign({}, options, scope.$eval(attrs.igniteAce));
opts.callbacks = [opts.onLoad];
http://git-wip-us.apache.org/repos/asf/ignite/blob/74d25456/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js b/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js
index 39f7ccd..32fa167 100644
--- a/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js
+++ b/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js
@@ -16,6 +16,7 @@
*/
import angular from 'angular';
+import _ from 'lodash';
/**
* Special decorator that fix problem in AngularStrap selectAll / deselectAll methods.
@@ -27,12 +28,12 @@ export default angular.module('mgcrea.ngStrap.select')
const delegate = $delegate(element, controller, config);
// Common vars.
- const options = angular.extend({}, $delegate.defaults, config);
+ const options = Object.assign({}, $delegate.defaults, config);
const scope = delegate.$scope;
const valueByIndex = (index) => {
- if (angular.isUndefined(scope.$matches[index]))
+ if (_.isUndefined(scope.$matches[index]))
return null;
return scope.$matches[index].value;
http://git-wip-us.apache.org/repos/asf/ignite/blob/74d25456/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js b/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js
index d01a450..fa59f32 100644
--- a/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js
+++ b/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js
@@ -16,7 +16,7 @@
*/
import angular from 'angular';
-import flow from 'lodash/flow';
+import _ from 'lodash';
/**
* Decorator that fix problem in AngularStrap $tooltip.
@@ -62,7 +62,7 @@ export default angular
scope.$emit(options.prefixEvent + '.hide.before', $tooltip);
- if (angular.isDefined(options.onBeforeHide) && angular.isFunction(options.onBeforeHide))
+ if (!_.isUndefined(options.onBeforeHide) && _.isFunction(options.onBeforeHide))
options.onBeforeHide($tooltip);
$tooltip.$isShown = scope.$isShown = false;
@@ -82,8 +82,8 @@ export default angular
const $tooltip = $delegate(el, config);
$tooltip.$referenceElement = el;
- $tooltip.destroy = flow($tooltip.destroy, () => $tooltip.$referenceElement = null);
- $tooltip.$applyPlacement = flow($tooltip.$applyPlacement, () => {
+ $tooltip.destroy = _.flow($tooltip.destroy, () => $tooltip.$referenceElement = null);
+ $tooltip.$applyPlacement = _.flow($tooltip.$applyPlacement, () => {
if (!$tooltip.$element)
return;
http://git-wip-us.apache.org/repos/asf/ignite/blob/74d25456/modules/web-console/frontend/app/services/FormUtils.service.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/services/FormUtils.service.js b/modules/web-console/frontend/app/services/FormUtils.service.js
index f22d4bc..da1d737 100644
--- a/modules/web-console/frontend/app/services/FormUtils.service.js
+++ b/modules/web-console/frontend/app/services/FormUtils.service.js
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+import _ from 'lodash';
export default ['IgniteFormUtils', ['$window', 'IgniteFocus', ($window, Focus) => {
function ensureActivePanel(ui, pnl, focusId) {
@@ -41,7 +42,7 @@ export default ['IgniteFormUtils', ['$window', 'IgniteFocus', ($window, Focus) =
if (!activePanels || activePanels.length < 1)
ui.activePanels = [idx];
else if (!_.includes(activePanels, idx)) {
- const newActivePanels = angular.copy(activePanels);
+ const newActivePanels = _.cloneDeep(activePanels);
newActivePanels.push(idx);
[46/54] [abbrv] ignite git commit: IGNITE-7871 Fixed condition for
cache partitions validation. - Fixes #3804.
Posted by ag...@apache.org.
IGNITE-7871 Fixed condition for cache partitions validation. - Fixes #3804.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7a1d0ea7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7a1d0ea7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7a1d0ea7
Branch: refs/heads/ignite-6083
Commit: 7a1d0ea70127f73409071d7f9581a8207be97491
Parents: 706e7f8
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Thu Apr 12 14:29:43 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Thu Apr 12 14:29:43 2018 +0300
----------------------------------------------------------------------
.../distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a1d0ea7/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index dd4a571..af5acd6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -2760,7 +2760,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
|| grpCtx.config().isReadThrough()
|| grpCtx.config().isWriteThrough()
|| grpCtx.config().getCacheStoreFactory() != null
- || grpCtx.config().getRebalanceDelay() != -1
+ || grpCtx.config().getRebalanceDelay() == -1
|| grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE)
continue;
[03/54] [abbrv] ignite git commit: IGNITE-7222 Added ZooKeeper
discovery SPI
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/test/java/org/apache/ZookeeperNodeStart.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ZookeeperNodeStart.java b/modules/zookeeper/src/test/java/org/apache/ZookeeperNodeStart.java
new file mode 100644
index 0000000..ef4d5f4
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ZookeeperNodeStart.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache;
+
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
+
+/**
+ *
+ */
+public class ZookeeperNodeStart {
+ public static void main(String[] args) throws Exception {
+ try {
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ ZookeeperDiscoverySpi spi = new ZookeeperDiscoverySpi();
+
+ spi.setZkConnectionString("localhost:2181");
+
+ cfg.setDiscoverySpi(spi);
+
+ Ignition.start(cfg);
+ }
+ catch (Throwable e) {
+ e.printStackTrace(System.out);
+
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/IgniteCacheEntryListenerWithZkDiscoAtomicTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/IgniteCacheEntryListenerWithZkDiscoAtomicTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/IgniteCacheEntryListenerWithZkDiscoAtomicTest.java
new file mode 100644
index 0000000..754a6bf
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/IgniteCacheEntryListenerWithZkDiscoAtomicTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery.zk;
+
+import org.apache.ignite.internal.processors.cache.IgniteCacheEntryListenerAtomicTest;
+
+/**
+ * Class is added to mute {@link #testConcurrentRegisterDeregister} test in ZooKeeper suite
+ * (see related ticket).
+ *
+ * When slow down is tracked down and fixed this class can be replaced back with its parent.
+ */
+public class IgniteCacheEntryListenerWithZkDiscoAtomicTest extends IgniteCacheEntryListenerAtomicTest {
+ /** {@inheritDoc} */
+ @Override public void testConcurrentRegisterDeregister() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8109");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
new file mode 100644
index 0000000..766635c
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import junit.framework.TestSuite;
+import org.apache.curator.test.InstanceSpec;
+import org.apache.curator.test.TestingCluster;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.testframework.config.GridTestProperties;
+
+/**
+ * Allows to run regular Ignite tests with {@link org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi}.
+ */
+public abstract class ZookeeperDiscoverySpiAbstractTestSuite extends TestSuite {
+ /** */
+ private static TestingCluster testingCluster;
+
+ /**
+ * @throws Exception If failed.
+ */
+ public static void initSuite() throws Exception {
+ System.setProperty("zookeeper.forceSync", "false");
+
+ testingCluster = createTestingCluster(3);
+
+ testingCluster.start();
+
+ System.setProperty(GridTestProperties.IGNITE_CFG_PREPROCESSOR_CLS, ZookeeperDiscoverySpiAbstractTestSuite.class.getName());
+ }
+
+ /**
+ * Called via reflection by {@link org.apache.ignite.testframework.junits.GridAbstractTest}.
+ *
+ * @param cfg Configuration to change.
+ */
+ public synchronized static void preprocessConfiguration(IgniteConfiguration cfg) {
+ if (testingCluster == null)
+ throw new IllegalStateException("Test Zookeeper cluster is not started.");
+
+ ZookeeperDiscoverySpi zkSpi = new ZookeeperDiscoverySpi();
+
+ DiscoverySpi spi = cfg.getDiscoverySpi();
+
+ if (spi instanceof TcpDiscoverySpi)
+ zkSpi.setClientReconnectDisabled(((TcpDiscoverySpi)spi).isClientReconnectDisabled());
+
+ zkSpi.setSessionTimeout(30_000);
+ zkSpi.setZkConnectionString(testingCluster.getConnectString());
+
+ cfg.setDiscoverySpi(zkSpi);
+ }
+
+ /**
+ * @param instances Number of instances in
+ * @return Test cluster.
+ */
+ public static TestingCluster createTestingCluster(int instances) {
+ String tmpDir = System.getProperty("java.io.tmpdir");
+
+ List<InstanceSpec> specs = new ArrayList<>();
+
+ for (int i = 0; i < instances; i++) {
+ File file = new File(tmpDir, "apacheIgniteTestZk-" + i);
+
+ if (file.isDirectory())
+ deleteRecursively0(file);
+ else {
+ if (!file.mkdirs())
+ throw new IgniteException("Failed to create directory for test Zookeeper server: " + file.getAbsolutePath());
+ }
+
+ specs.add(new InstanceSpec(file, -1, -1, -1, true, -1, -1, 500));
+ }
+
+ return new TestingCluster(specs);
+ }
+
+ /**
+ * @param file File or directory to delete.
+ */
+ private static void deleteRecursively0(File file) {
+ File[] files = file.listFiles();
+
+ if (files == null)
+ return;
+
+ for (File f : files) {
+ if (f.isDirectory())
+ deleteRecursively0(f);
+ else {
+ if (!f.delete())
+ throw new IgniteException("Failed to delete file: " + f.getAbsolutePath());
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java
new file mode 100644
index 0000000..860488b
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperClientTest;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoverySpiSaslSuccessfulAuthTest;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoverySpiTest;
+
+/**
+ *
+ */
+public class ZookeeperDiscoverySpiTestSuite1 extends TestSuite {
+ /**
+ * @return Test suite.
+ * @throws Exception Thrown in case of the failure.
+ */
+ public static TestSuite suite() throws Exception {
+ System.setProperty("zookeeper.forceSync", "false");
+
+ TestSuite suite = new TestSuite("ZookeeperDiscoverySpi Test Suite");
+
+ suite.addTestSuite(ZookeeperClientTest.class);
+ suite.addTestSuite(ZookeeperDiscoverySpiTest.class);
+ suite.addTestSuite(ZookeeperDiscoverySpiSaslSuccessfulAuthTest.class);
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
new file mode 100644
index 0000000..3775aa1
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk;
+
+import junit.framework.TestSuite;
+import org.apache.curator.test.TestingCluster;
+import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest;
+import org.apache.ignite.internal.IgniteClientReconnectCacheTest;
+import org.apache.ignite.internal.processors.cache.IgniteCacheEntryListenerAtomicTest;
+import org.apache.ignite.internal.processors.cache.datastructures.IgniteClientDataStructuresTest;
+import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedNodeRestartTxSelfTest;
+import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedSequenceApiSelfTest;
+import org.apache.ignite.internal.processors.cache.datastructures.replicated.GridCacheReplicatedSequenceApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePutRetryAtomicSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePutRetryTransactionalSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNodeRestartTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedNodeRestartSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedQuerySelfTest;
+import org.apache.ignite.internal.processors.cache.multijvm.GridCacheAtomicMultiJvmFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.multijvm.GridCachePartitionedMultiJvmFullApiSelfTest;
+import org.apache.ignite.internal.processors.continuous.GridEventConsumeSelfTest;
+
+/**
+ * Regular Ignite tests executed with {@link org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi}.
+ */
+public class ZookeeperDiscoverySpiTestSuite2 extends ZookeeperDiscoverySpiAbstractTestSuite {
+ /** */
+ private static TestingCluster testingCluster;
+
+ /**
+ * @return Test suite.
+ * @throws Exception Thrown in case of the failure.
+ */
+ public static TestSuite suite() throws Exception {
+ System.setProperty("H2_JDBC_CONNECTIONS", "500"); // For multi-jvm tests.
+
+ initSuite();
+
+ TestSuite suite = new TestSuite("ZookeeperDiscoverySpi Test Suite");
+
+ suite.addTestSuite(ZookeeperDiscoverySuitePreprocessorTest.class);
+
+ suite.addTestSuite(GridCacheReplicatedNodeRestartSelfTest.class);
+ suite.addTestSuite(GridCachePartitionedNodeRestartTest.class);
+
+ suite.addTestSuite(IgniteCacheEntryListenerWithZkDiscoAtomicTest.class);
+
+ suite.addTestSuite(GridEventConsumeSelfTest.class);
+
+ suite.addTestSuite(IgniteClientReconnectCacheTest.class);
+
+ suite.addTestSuite(IgniteCachePutRetryAtomicSelfTest.class);
+ suite.addTestSuite(IgniteCachePutRetryTransactionalSelfTest.class);
+
+ suite.addTestSuite(ClusterNodeMetricsUpdateTest.class);
+
+ suite.addTestSuite(GridCachePartitionedMultiNodeFullApiSelfTest.class);
+ suite.addTestSuite(GridCacheReplicatedMultiNodeFullApiSelfTest.class);
+
+ suite.addTestSuite(GridCacheAtomicMultiNodeFullApiSelfTest.class);
+ suite.addTestSuite(GridCacheReplicatedAtomicMultiNodeFullApiSelfTest.class);
+
+ suite.addTestSuite(GridCachePartitionedNodeRestartTxSelfTest.class);
+ suite.addTestSuite(IgniteClientDataStructuresTest.class);
+ suite.addTestSuite(GridCacheReplicatedSequenceApiSelfTest.class);
+ suite.addTestSuite(GridCachePartitionedSequenceApiSelfTest.class);
+
+ suite.addTestSuite(IgniteCacheReplicatedQuerySelfTest.class);
+
+ suite.addTestSuite(GridCacheAtomicMultiJvmFullApiSelfTest.class);
+ suite.addTestSuite(GridCachePartitionedMultiJvmFullApiSelfTest.class);
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySuitePreprocessorTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySuitePreprocessorTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySuitePreprocessorTest.java
new file mode 100644
index 0000000..28cf17f
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySuitePreprocessorTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk;
+
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.config.GridTestProperties;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Sanity test verifying that configuration callback specified via
+ * {@link GridTestProperties#IGNITE_CFG_PREPROCESSOR_CLS} really works.
+ * <p>
+ * This test should be run as part of {@link ZookeeperDiscoverySpiTestSuite2}.
+ */
+public class ZookeeperDiscoverySuitePreprocessorTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ // Test sets TcpDiscoverySpi, but it should be automatically changed to ZookeeperDiscoverySpi.
+ TcpDiscoverySpi spi = new TcpDiscoverySpi();
+
+ spi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(spi);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ super.afterTest();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSpiConfigurationIsChanged() throws Exception {
+ startGrid(0);
+
+ checkDiscoverySpi(1);
+
+ startGrid(1);
+
+ checkDiscoverySpi(2);
+
+ startGridsMultiThreaded(2, 2);
+
+ checkDiscoverySpi(4);
+
+ startGrid();
+
+ checkDiscoverySpi(5);
+ }
+
+ /**
+ * @param expNodes Expected nodes number.
+ * @throws Exception If failed.
+ */
+ private void checkDiscoverySpi(int expNodes) throws Exception {
+ List<Ignite> nodes = G.allGrids();
+
+ assertEquals(expNodes, nodes.size());
+
+ for (Ignite node : nodes) {
+ DiscoverySpi spi = node.configuration().getDiscoverySpi();
+
+ assertTrue("Node should be started with " + ZookeeperDiscoverySpi.class.getName(),
+ spi instanceof ZookeeperDiscoverySpi);
+ }
+
+ waitForTopology(expNodes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
new file mode 100644
index 0000000..e7cb97a
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.curator.test.TestingCluster;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ *
+ */
+public class ZookeeperClientTest extends GridCommonAbstractTest {
+ /** */
+ private static final int SES_TIMEOUT = 60_000;
+
+ /** */
+ private TestingCluster zkCluster;
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ closeZK();
+
+ super.afterTest();
+ }
+
+ /**
+ * @param sesTimeout Session timeout.
+ * @return Client.
+ * @throws Exception If failed.
+ */
+ private ZookeeperClient createClient(int sesTimeout) throws Exception {
+ return new ZookeeperClient(log, zkCluster.getConnectString(), sesTimeout, null);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSaveLargeValue() throws Exception {
+ startZK(1);
+
+ final ZookeeperClient client = createClient(SES_TIMEOUT);
+
+ byte[] data = new byte[1024 * 1024];
+
+ String basePath = "/ignite";
+
+ assertTrue(client.needSplitNodeData(basePath, data, 2));
+
+ List<byte[]> parts = client.splitNodeData(basePath, data, 2);
+
+ assertTrue(parts.size() > 1);
+
+ ZooKeeper zk = client.zk();
+
+ for (int i = 0; i < parts.size(); i++) {
+ byte[] part = parts.get(i);
+
+ assertTrue(part.length > 0);
+
+ String path0 = basePath + ":" + i;
+
+ zk.create(path0, part, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testClose() throws Exception {
+ startZK(1);
+
+ final ZookeeperClient client = createClient(SES_TIMEOUT);
+
+ client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+ client.zk().close();
+
+ GridTestUtils.assertThrows(log, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+
+ return null;
+ }
+ }, ZookeeperClientFailedException.class, null);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCreateAll() throws Exception {
+ startZK(1);
+
+ ZookeeperClient client = createClient(SES_TIMEOUT);
+
+ client.createIfNeeded("/apacheIgnite", null, CreateMode.PERSISTENT);
+
+ List<String> paths = new ArrayList<>();
+
+ paths.add("/apacheIgnite/1");
+ paths.add("/apacheIgnite/2");
+ paths.add("/apacheIgnite/3");
+
+ client.createAll(paths, CreateMode.PERSISTENT);
+
+ assertEquals(3, client.getChildren("/apacheIgnite").size());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeleteAll() throws Exception {
+ startZK(1);
+
+ ZookeeperClient client = createClient(SES_TIMEOUT);
+
+ client.createIfNeeded("/apacheIgnite", null, CreateMode.PERSISTENT);
+ client.createIfNeeded("/apacheIgnite/1", null, CreateMode.PERSISTENT);
+ client.createIfNeeded("/apacheIgnite/2", null, CreateMode.PERSISTENT);
+
+ client.deleteAll("/apacheIgnite", Arrays.asList("1", "2"), -1);
+
+ assertTrue(client.getChildren("/apacheIgnite").isEmpty());
+
+ client.createIfNeeded("/apacheIgnite/1", null, CreateMode.PERSISTENT);
+ client.deleteAll("/apacheIgnite", Collections.singletonList("1"), -1);
+
+ assertTrue(client.getChildren("/apacheIgnite").isEmpty());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionLoss1() throws Exception {
+ ZookeeperClient client = new ZookeeperClient(log, "localhost:2200", 3000, null);
+
+ try {
+ client.createIfNeeded("/apacheIgnite", null, CreateMode.PERSISTENT);
+
+ fail();
+ }
+ catch (ZookeeperClientFailedException e) {
+ info("Expected error: " + e);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionLoss2() throws Exception {
+ startZK(1);
+
+ ZookeeperClient client = createClient(3000);
+
+ client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+ closeZK();
+
+ try {
+ client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+
+ fail();
+ }
+ catch (ZookeeperClientFailedException e) {
+ info("Expected error: " + e);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionLoss3() throws Exception {
+ startZK(1);
+
+ CallbackFuture cb = new CallbackFuture();
+
+ ZookeeperClient client = new ZookeeperClient(log, zkCluster.getConnectString(), 3000, cb);
+
+ client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+ closeZK();
+
+ final AtomicBoolean res = new AtomicBoolean();
+
+ client.getChildrenAsync("/apacheIgnite1", null, new AsyncCallback.Children2Callback() {
+ @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+ if (rc == 0)
+ res.set(true);
+ }
+ });
+
+ cb.get(60_000);
+
+ assertFalse(res.get());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionLoss4() throws Exception {
+ startZK(1);
+
+ CallbackFuture cb = new CallbackFuture();
+
+ final ZookeeperClient client = new ZookeeperClient(log, zkCluster.getConnectString(), 3000, cb);
+
+ client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+ final CountDownLatch l = new CountDownLatch(1);
+
+ client.getChildrenAsync("/apacheIgnite1", null, new AsyncCallback.Children2Callback() {
+ @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+ closeZK();
+
+ try {
+ client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+ }
+ catch (ZookeeperClientFailedException e) {
+ info("Expected error: " + e);
+
+ l.countDown();
+ }
+ catch (Exception e) {
+ fail("Unexpected error: " + e);
+ }
+ }
+ });
+
+ assertTrue(l.await(10, TimeUnit.SECONDS));
+
+ cb.get();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReconnect1() throws Exception {
+ startZK(1);
+
+ ZookeeperClient client = createClient(SES_TIMEOUT);
+
+ client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+ zkCluster.getServers().get(0).stop();
+
+ IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ U.sleep(2000);
+
+ info("Restart zookeeper server");
+
+ zkCluster.getServers().get(0).restart();
+
+ info("Zookeeper server restarted");
+
+ return null;
+ }
+ }, "start-zk");
+
+ client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+
+ fut.get();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReconnect1_Callback() throws Exception {
+ startZK(1);
+
+ ZookeeperClient client = createClient(SES_TIMEOUT);
+
+ client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+ zkCluster.getServers().get(0).stop();
+
+ final CountDownLatch l = new CountDownLatch(1);
+
+ client.getChildrenAsync("/apacheIgnite1", null, new AsyncCallback.Children2Callback() {
+ @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+ info("Callback: " + rc);
+
+ if (rc == 0)
+ l.countDown();
+ }
+ });
+
+ IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ U.sleep(2000);
+
+ info("Restart zookeeper server");
+
+ zkCluster.getServers().get(0).restart();
+
+ info("Zookeeper server restarted");
+
+ return null;
+ }
+ }, "start-zk");
+
+ assertTrue(l.await(10, TimeUnit.SECONDS));
+
+ fut.get();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReconnect1_InCallback() throws Exception {
+ startZK(1);
+
+ final ZookeeperClient client = createClient(SES_TIMEOUT);
+
+ client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+ final CountDownLatch l = new CountDownLatch(1);
+
+ client.getChildrenAsync("/apacheIgnite1", null, new AsyncCallback.Children2Callback() {
+ @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+ try {
+ zkCluster.getServers().get(0).stop();
+
+ IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ U.sleep(2000);
+
+ info("Restart zookeeper server");
+
+ zkCluster.getServers().get(0).restart();
+
+ info("Zookeeper server restarted");
+
+ return null;
+ }
+ }, "start-zk");
+
+ client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+
+ l.countDown();
+
+ fut.get();
+ }
+ catch (Exception e) {
+ fail("Unexpected error: " + e);
+ }
+ }
+ });
+
+ assertTrue(l.await(10, TimeUnit.SECONDS));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReconnect2() throws Exception {
+ startZK(1);
+
+ ZookeeperClient client = createClient(SES_TIMEOUT);
+
+ client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+ zkCluster.getServers().get(0).restart();
+
+ client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReconnect3() throws Exception {
+ startZK(3);
+
+ ZookeeperClient client = createClient(SES_TIMEOUT);
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 30; i++) {
+ info("Iteration: " + i);
+
+ int idx = rnd.nextInt(3);
+
+ zkCluster.getServers().get(idx).restart();
+
+ doSleep(rnd.nextLong(100) + 1);
+
+ client.createIfNeeded("/apacheIgnite" + i, null, CreateMode.PERSISTENT);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReconnect4() throws Exception {
+ startZK(3);
+
+ ZookeeperClient client = new ZookeeperClient(log,
+ zkCluster.getServers().get(2).getInstanceSpec().getConnectString(),
+ 60_000,
+ null);
+
+ client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+ zkCluster.getServers().get(0).stop();
+ zkCluster.getServers().get(1).stop();
+
+ IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ U.sleep(2000);
+
+ info("Restart zookeeper server");
+
+ zkCluster.getServers().get(0).restart();
+
+ info("Zookeeper server restarted");
+
+ return null;
+ }
+ }, "start-zk");
+
+ client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+
+ fut.get();
+ }
+
+ /**
+ * @param instances Number of servers in ZK ensemble.
+ * @throws Exception If failed.
+ */
+ private void startZK(int instances) throws Exception {
+ assert zkCluster == null;
+
+ zkCluster = new TestingCluster(instances);
+
+ zkCluster.start();
+ }
+
+ /**
+ *
+ */
+ private void closeZK() {
+ if (zkCluster != null) {
+ try {
+ zkCluster.close();
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to stop Zookeeper client: " + e, e);
+ }
+
+ zkCluster = null;
+ }
+ }
+
+ /**
+ *
+ */
+ private static class CallbackFuture extends GridFutureAdapter<Void> implements IgniteRunnable {
+ /** {@inheritDoc} */
+ @Override public void run() {
+ onDone();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslAuthAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslAuthAbstractTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslAuthAbstractTest.java
new file mode 100644
index 0000000..ac94bf2
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslAuthAbstractTest.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Paths;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
+import org.apache.zookeeper.server.ServerCnxnFactory;
+import org.apache.zookeeper.server.ZKDatabase;
+import org.apache.zookeeper.server.ZooKeeperServer;
+
+import static org.apache.curator.test.DirectoryUtils.deleteRecursively;
+
+/**
+ * Implements methods to prepare SASL tests infrastructure: jaas.conf files, starting up ZooKeeper server,
+ * clean up procedures when the test has finished etc.
+ */
+public abstract class ZookeeperDiscoverySpiSaslAuthAbstractTest extends GridCommonAbstractTest {
+ /** */
+ private File tmpDir = createTmpDir();
+
+ /** */
+ private static final String JAAS_CONF_FILE = "jaas.conf";
+
+ /** */
+ private static final String AUTH_PROVIDER = "zookeeper.authProvider.1";
+
+ /** */
+ private static final String SASL_CONFIG = "java.security.auth.login.config";
+
+ /** */
+ private long joinTimeout = 2_000;
+
+ /** */
+ private long sesTimeout = 10_000;
+
+ /** */
+ private ServerCnxnFactory serverFactory;
+
+ /** */
+ private String hostPort = "localhost:2181";
+
+ /** */
+ private int maxCnxns;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String instanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(instanceName);
+
+ ZookeeperDiscoverySpi zkSpi = new ZookeeperDiscoverySpi();
+
+ if (joinTimeout != 0)
+ zkSpi.setJoinTimeout(joinTimeout);
+
+ zkSpi.setSessionTimeout(sesTimeout > 0 ? sesTimeout : 10_000);
+
+ zkSpi.setZkConnectionString(hostPort);
+
+ cfg.setDiscoverySpi(zkSpi);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ prepareJaasConfigFile();
+
+ prepareSaslSystemProperties();
+
+ startZooKeeperServer();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopZooKeeperServer();
+
+ stopAllGrids();
+
+ clearSaslSystemProperties();
+
+ clearTmpDir();
+ }
+
+ /** */
+ private void clearTmpDir() throws Exception {
+ deleteRecursively(tmpDir);
+ }
+
+ /** */
+ protected void clearSaslSystemProperties() {
+ System.clearProperty(AUTH_PROVIDER);
+
+ System.clearProperty(SASL_CONFIG);
+
+ System.clearProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void prepareJaasConfigFile() throws Exception {
+ U.ensureDirectory(tmpDir, "Temp directory for JAAS configuration file wasn't created", null);
+
+ File saslConfFile = new File(tmpDir, JAAS_CONF_FILE);
+
+ FileWriter fwriter = new FileWriter(saslConfFile);
+
+ writeServerConfigSection(fwriter, "validPassword");
+
+ writeClientConfigSection(fwriter, "ValidZookeeperClient", "validPassword");
+
+ writeClientConfigSection(fwriter, "InvalidZookeeperClient", "invalidPassword");
+
+ fwriter.close();
+ }
+
+ /** */
+ private void prepareSaslSystemProperties() {
+ System.setProperty(SASL_CONFIG, Paths.get(tmpDir.getPath().toString(), JAAS_CONF_FILE).toString());
+
+ System.setProperty(AUTH_PROVIDER, "org.apache.zookeeper.server.auth.SASLAuthenticationProvider");
+ }
+
+ /** */
+ private void writeClientConfigSection(FileWriter fwriter, String clientName, String pass) throws IOException {
+ fwriter.write(clientName + "{\n" +
+ " org.apache.zookeeper.server.auth.DigestLoginModule required\n" +
+ " username=\"zkUser\"\n" +
+ " password=\"" + pass + "\";\n" +
+ "};" + "\n");
+ }
+
+ /** */
+ private void writeServerConfigSection(FileWriter fwriter, String pass) throws IOException {
+ fwriter.write("Server {\n" +
+ " org.apache.zookeeper.server.auth.DigestLoginModule required\n" +
+ " user_zkUser=\"" + pass + "\";\n" +
+ "};\n");
+ }
+
+ /** */
+ private File createTmpDir() {
+ File jaasConfDir = Paths.get(System.getProperty("java.io.tmpdir"), "zk_disco_spi_test").toFile();
+
+ try {
+ U.ensureDirectory(jaasConfDir, "", null);
+ }
+ catch (IgniteCheckedException e) {
+ // ignored
+ }
+
+ return jaasConfDir;
+ }
+
+ /** */
+ private void stopZooKeeperServer() throws Exception {
+ shutdownServerInstance(serverFactory);
+ serverFactory = null;
+ }
+
+ /** */
+ private void shutdownServerInstance(ServerCnxnFactory factory)
+ {
+ if (factory != null) {
+ ZKDatabase zkDb = null;
+ {
+ ZooKeeperServer zs = getServer(factory);
+ if (zs != null)
+ zkDb = zs.getZKDatabase();
+ }
+ factory.shutdown();
+ try {
+ if (zkDb != null)
+ zkDb.close();
+ } catch (IOException ie) {
+ // ignore
+ }
+ }
+ }
+
+ /** */
+ private ZooKeeperServer getServer(ServerCnxnFactory fac) {
+ ZooKeeperServer zs = U.field(fac, "zkServer");
+
+ return zs;
+ }
+
+ /** */
+ private void startZooKeeperServer() throws Exception {
+ serverFactory = createNewServerInstance(serverFactory, hostPort,
+ maxCnxns);
+ startServerInstance(tmpDir, serverFactory);
+ }
+
+ /** */
+ private ServerCnxnFactory createNewServerInstance(
+ ServerCnxnFactory factory, String hostPort, int maxCnxns)
+ throws IOException {
+ final int port = getPort(hostPort);
+
+ if (factory == null)
+ factory = ServerCnxnFactory.createFactory(port, maxCnxns);
+
+ return factory;
+ }
+
+ /** */
+ private void startServerInstance(File dataDir,
+ ServerCnxnFactory factory) throws IOException,
+ InterruptedException {
+ ZooKeeperServer zks = new ZooKeeperServer(dataDir, dataDir, 3000);
+ factory.startup(zks);
+ }
+
+ /** */
+ private int getPort(String hostPort) {
+ String[] split = hostPort.split(":");
+ String portstr = split[split.length-1];
+ String[] pc = portstr.split("/");
+
+ if (pc.length > 1)
+ portstr = pc[0];
+
+ return Integer.parseInt(portstr);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslFailedAuthTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslFailedAuthTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslFailedAuthTest.java
new file mode 100644
index 0000000..864ac96
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslFailedAuthTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class ZookeeperDiscoverySpiSaslFailedAuthTest extends ZookeeperDiscoverySpiSaslAuthAbstractTest {
+ /**
+ * @throws Exception If failed.
+ */
+ public void testIgniteNodeWithInvalidPasswordFailsToJoin() throws Exception {
+ System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
+ "InvalidZookeeperClient");
+
+ System.setProperty("IGNITE_ZOOKEEPER_DISCOVERY_MAX_RETRY_COUNT", Integer.toString(1));
+
+ try {
+ startGrid(0);
+
+ Assert.fail("Ignite node with invalid password should fail on join.");
+ }
+ catch (Exception e) {
+ //ignored
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslSuccessfulAuthTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslSuccessfulAuthTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslSuccessfulAuthTest.java
new file mode 100644
index 0000000..5ee0a43
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslSuccessfulAuthTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
+
+/**
+ *
+ */
+public class ZookeeperDiscoverySpiSaslSuccessfulAuthTest extends ZookeeperDiscoverySpiSaslAuthAbstractTest {
+ /**
+ * @throws Exception If failed.
+ */
+ public void testIgniteNodesWithValidPasswordSuccessfullyJoins() throws Exception {
+ System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
+ "ValidZookeeperClient");
+
+ startGrids(3);
+
+ waitForTopology(3);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testIgniteNodeWithoutSaslConfigurationSuccessfullyJoins() throws Exception {
+ //clearing SASL-related system properties that were set in beforeTest
+ clearSaslSystemProperties();
+
+ startGrid(0);
+
+ waitForTopology(1);
+ }
+}
[36/54] [abbrv] ignite git commit: IGNITE-8204: SQL: fixed hangs when
lazy flag is enabled. This closes #3785.
Posted by ag...@apache.org.
IGNITE-8204: SQL: fixed hangs when lazy flag is enabled. This closes #3785.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/747e6c5f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/747e6c5f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/747e6c5f
Branch: refs/heads/ignite-6083
Commit: 747e6c5f9c635a5b9c6856efd2b94b05297b7f25
Parents: 98ef925
Author: Alexander Paschenko <al...@gmail.com>
Authored: Wed Apr 11 16:20:16 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Apr 11 16:20:16 2018 +0300
----------------------------------------------------------------------
.../query/h2/twostep/GridMapQueryExecutor.java | 7 +++++++
.../query/h2/twostep/MapQueryLazyWorker.java | 13 +++++++++++--
2 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/747e6c5f/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index 9b1e4fa..930ada2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -234,6 +234,13 @@ public class GridMapQueryExecutor {
}
/**
+ * @return Busy lock for lazy workers to guard their operations with.
+ */
+ GridSpinBusyLock busyLock() {
+ return busyLock;
+ }
+
+ /**
* @param node Node.
* @param msg Message.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/747e6c5f/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryLazyWorker.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryLazyWorker.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryLazyWorker.java
index 59c050f..98f3df9 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryLazyWorker.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryLazyWorker.java
@@ -80,8 +80,17 @@ public class MapQueryLazyWorker extends GridWorker {
while (!isCancelled()) {
Runnable task = tasks.take();
- if (task != null)
- task.run();
+ if (task != null) {
+ if (!exec.busyLock().enterBusy())
+ return;
+
+ try {
+ task.run();
+ }
+ finally {
+ exec.busyLock().leaveBusy();
+ }
+ }
}
}
finally {
[13/54] [abbrv] ignite git commit: IGNITE-8059: Integrate decision
tree with partition based dataset.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
deleted file mode 100644
index f9117f4..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.trees;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.MatrixColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.util.Utils;
-
-/** */
-class SplitDataGenerator<V extends Vector> {
- /** */
- private static final Random rnd = new Random(12349L);
-
- /** */
- private static final double DELTA = 100.0;
-
- /** Map of the form of (is categorical -> list of region indexes). */
- private final Map<Boolean, List<Integer>> di;
-
- /** List of regions. */
- private final List<Region> regs;
-
- /** Data of bounds of regions. */
- private final Map<Integer, IgniteBiTuple<Double, Double>> boundsData;
-
- /** */
- private final Map<Integer, Integer> catFeaturesInfo;
-
- /** Supplier of vectors. */
- private final Supplier<V> supplier;
-
- /** Features count. */
- private final int featCnt;
-
- /**
- * Create SplitDataGenerator.
- *
- * @param featCnt Features count.
- * @param catFeaturesInfo Information about categorical features in form of map (feature index -> categories
- * count).
- * @param supplier Supplier of vectors.
- */
- SplitDataGenerator(int featCnt, Map<Integer, Integer> catFeaturesInfo, Supplier<V> supplier) {
- regs = new LinkedList<>();
- boundsData = new HashMap<>();
- this.supplier = supplier;
- this.featCnt = featCnt;
- this.catFeaturesInfo = catFeaturesInfo;
-
- // Divide indexes into indexes of categorical coordinates and indexes of continuous coordinates.
- di = IntStream.range(0, featCnt).
- boxed().
- collect(Collectors.partitioningBy(catFeaturesInfo::containsKey));
-
- // Categorical coordinates info.
- Map<Integer, CatCoordInfo> catCoords = new HashMap<>();
- di.get(true).forEach(i -> {
- BitSet bs = new BitSet();
- bs.set(0, catFeaturesInfo.get(i));
- catCoords.put(i, new CatCoordInfo(bs));
- });
-
- // Continuous coordinates info.
- Map<Integer, ContCoordInfo> contCoords = new HashMap<>();
- di.get(false).forEach(i -> {
- contCoords.put(i, new ContCoordInfo());
- boundsData.put(i, new IgniteBiTuple<>(-1.0, 1.0));
- });
-
- Region firstReg = new Region(catCoords, contCoords, 0);
- regs.add(firstReg);
- }
-
- /** */
- <D extends ContinuousRegionInfo> void testByGen(int totalPts,
- IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> calc,
- IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> catImpCalc,
- IgniteFunction<DoubleStream, Double> regCalc, Ignite ignite) {
-
- List<IgniteBiTuple<Integer, V>> lst = points(totalPts, (i, rn) -> i).collect(Collectors.toList());
-
- Collections.shuffle(lst, rnd);
-
- SparseDistributedMatrix m = new SparseDistributedMatrix(totalPts,
- featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
-
- int i = 0;
- for (IgniteBiTuple<Integer, V> bt : lst) {
- byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
- byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
- m.setRow(i, bt.get2().getStorage().data());
- i++;
- }
-
- ColumnDecisionTreeTrainer<D> trainer =
- new ColumnDecisionTreeTrainer<>(3, calc, catImpCalc, regCalc, ignite);
-
- DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catFeaturesInfo));
-
- byRegion.keySet().forEach(k -> mdl.apply(byRegion.get(k).get(0).features()));
- }
-
- /**
- * Split region by continuous coordinate using given threshold.
- *
- * @param regIdx Region index.
- * @param coordIdx Coordinate index.
- * @param threshold Threshold.
- * @return {@code this}.
- */
- SplitDataGenerator<V> split(int regIdx, int coordIdx, double threshold) {
- Region regToSplit = regs.get(regIdx);
- ContCoordInfo cci = regToSplit.contCoords.get(coordIdx);
-
- double left = cci.left;
- double right = cci.right;
-
- if (threshold < left || threshold > right)
- throw new MathIllegalArgumentException("Threshold is out of region bounds.");
-
- regToSplit.incTwoPow();
-
- Region newReg = Utils.copy(regToSplit);
- newReg.contCoords.get(coordIdx).left = threshold;
-
- regs.add(regIdx + 1, newReg);
- cci.right = threshold;
-
- IgniteBiTuple<Double, Double> bounds = boundsData.get(coordIdx);
- double min = bounds.get1();
- double max = bounds.get2();
- boundsData.put(coordIdx, new IgniteBiTuple<>(Math.min(threshold, min), Math.max(max, threshold)));
-
- return this;
- }
-
- /**
- * Split region by categorical coordinate.
- *
- * @param regIdx Region index.
- * @param coordIdx Coordinate index.
- * @param cats Categories allowed for the left sub region.
- * @return {@code this}.
- */
- SplitDataGenerator<V> split(int regIdx, int coordIdx, int[] cats) {
- BitSet subset = new BitSet();
- Arrays.stream(cats).forEach(subset::set);
- Region regToSplit = regs.get(regIdx);
- CatCoordInfo cci = regToSplit.catCoords.get(coordIdx);
-
- BitSet ssc = (BitSet)subset.clone();
- BitSet set = cci.bs;
- ssc.and(set);
- if (ssc.length() != subset.length())
- throw new MathIllegalArgumentException("Splitter set is not a subset of a parent subset.");
-
- ssc.xor(set);
- set.and(subset);
-
- regToSplit.incTwoPow();
- Region newReg = Utils.copy(regToSplit);
- newReg.catCoords.put(coordIdx, new CatCoordInfo(ssc));
-
- regs.add(regIdx + 1, newReg);
-
- return this;
- }
-
- /**
- * Get stream of points generated by this generator.
- *
- * @param ptsCnt Points count.
- */
- private Stream<IgniteBiTuple<Integer, V>> points(int ptsCnt, BiFunction<Double, Random, Double> f) {
- return IntStream.range(0, regs.size()).
- boxed().
- map(i -> regs.get(i).generatePoints(ptsCnt, f.apply((double)i, rnd), boundsData, di, supplier, rnd)
- .map(v -> new IgniteBiTuple<>(i, v))).flatMap(Function.identity());
- }
-
- /**
- * Convert double array to {@link LabeledVectorDouble}
- *
- * @param arr Array for conversion.
- * @return LabeledVectorDouble.
- */
- private static LabeledVectorDouble<DenseLocalOnHeapVector> asLabeledVector(double arr[]) {
- return new LabeledVectorDouble<>(new DenseLocalOnHeapVector(
- Arrays.copyOf(arr, arr.length - 1)), arr[arr.length - 1]);
- }
-
- /**
- * Categorical coordinate info.
- */
- private static class CatCoordInfo implements Serializable {
- /**
- * Defines categories which are included in this region
- */
- private final BitSet bs;
-
- /**
- * Construct CatCoordInfo.
- *
- * @param bs Bitset.
- */
- CatCoordInfo(BitSet bs) {
- this.bs = bs;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "CatCoordInfo [" +
- "bs=" + bs +
- ']';
- }
- }
-
- /**
- * Continuous coordinate info.
- */
- private static class ContCoordInfo implements Serializable {
- /**
- * Left (min) bound of region.
- */
- private double left;
-
- /**
- * Right (max) bound of region.
- */
- private double right;
-
- /**
- * Construct ContCoordInfo.
- */
- ContCoordInfo() {
- left = Double.NEGATIVE_INFINITY;
- right = Double.POSITIVE_INFINITY;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "ContCoordInfo [" +
- "left=" + left +
- ", right=" + right +
- ']';
- }
- }
-
- /**
- * Class representing information about region.
- */
- private static class Region implements Serializable {
- /**
- * Information about categorical coordinates restrictions of this region in form of
- * (coordinate index -> restriction)
- */
- private final Map<Integer, CatCoordInfo> catCoords;
-
- /**
- * Information about continuous coordinates restrictions of this region in form of
- * (coordinate index -> restriction)
- */
- private final Map<Integer, ContCoordInfo> contCoords;
-
- /**
- * Region should contain {@code 1/2^twoPow * totalPoints} points.
- */
- private int twoPow;
-
- /**
- * Construct region by information about restrictions on coordinates (features) values.
- *
- * @param catCoords Restrictions on categorical coordinates.
- * @param contCoords Restrictions on continuous coordinates
- * @param twoPow Region should contain {@code 1/2^twoPow * totalPoints} points.
- */
- Region(Map<Integer, CatCoordInfo> catCoords, Map<Integer, ContCoordInfo> contCoords, int twoPow) {
- this.catCoords = catCoords;
- this.contCoords = contCoords;
- this.twoPow = twoPow;
- }
-
- /** */
- int divideBy() {
- return 1 << twoPow;
- }
-
- /** */
- void incTwoPow() {
- twoPow++;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "Region [" +
- "catCoords=" + catCoords +
- ", contCoords=" + contCoords +
- ", twoPow=" + twoPow +
- ']';
- }
-
- /**
- * Generate continuous coordinate for this region.
- *
- * @param coordIdx Coordinate index.
- * @param boundsData Data with bounds
- * @param rnd Random numbers generator.
- * @return Categorical coordinate value.
- */
- double generateContCoord(int coordIdx, Map<Integer, IgniteBiTuple<Double, Double>> boundsData,
- Random rnd) {
- ContCoordInfo cci = contCoords.get(coordIdx);
- double left = cci.left;
- double right = cci.right;
-
- if (left == Double.NEGATIVE_INFINITY)
- left = boundsData.get(coordIdx).get1() - DELTA;
-
- if (right == Double.POSITIVE_INFINITY)
- right = boundsData.get(coordIdx).get2() + DELTA;
-
- double size = right - left;
-
- return left + rnd.nextDouble() * size;
- }
-
- /**
- * Generate categorical coordinate value for this region.
- *
- * @param coordIdx Coordinate index.
- * @param rnd Random numbers generator.
- * @return Categorical coordinate value.
- */
- double generateCatCoord(int coordIdx, Random rnd) {
- // Pick random bit.
- BitSet bs = catCoords.get(coordIdx).bs;
- int j = rnd.nextInt(bs.length());
-
- int i = 0;
- int bn = 0;
- int bnp = 0;
-
- while ((bn = bs.nextSetBit(bn)) != -1 && i <= j) {
- i++;
- bnp = bn;
- bn++;
- }
-
- return bnp;
- }
-
- /**
- * Generate points for this region.
- *
- * @param ptsCnt Count of points to generate.
- * @param val Label for all points in this region.
- * @param boundsData Data about bounds of continuous coordinates.
- * @param catCont Data about which categories can be in this region in the form (coordinate index -> list of
- * categories indexes).
- * @param s Vectors supplier.
- * @param rnd Random numbers generator.
- * @param <V> Type of vectors.
- * @return Stream of generated points for this region.
- */
- <V extends Vector> Stream<V> generatePoints(int ptsCnt, double val,
- Map<Integer, IgniteBiTuple<Double, Double>> boundsData, Map<Boolean, List<Integer>> catCont,
- Supplier<V> s,
- Random rnd) {
- return IntStream.range(0, ptsCnt / divideBy()).mapToObj(i -> {
- V v = s.get();
- int coordsCnt = v.size();
- catCont.get(false).forEach(ci -> v.setX(ci, generateContCoord(ci, boundsData, rnd)));
- catCont.get(true).forEach(ci -> v.setX(ci, generateCatCoord(ci, rnd)));
-
- v.setX(coordsCnt - 1, val);
- return v;
- });
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/package-info.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/package-info.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/package-info.java
deleted file mode 100644
index fc379a6..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * ML Grid decision tree benchmarks.
- */
-package org.apache.ignite.yardstick.ml.trees;
\ No newline at end of file
[38/54] [abbrv] ignite git commit: IGNITE-8148: JDBC thin: semicolon
as delimiter for properties. This closes #3794.
Posted by ag...@apache.org.
IGNITE-8148: JDBC thin: semicolon as delimiter for properties. This closes #3794.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/14402e40
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/14402e40
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/14402e40
Branch: refs/heads/ignite-6083
Commit: 14402e4030cf0257b72c636894f140f346266299
Parents: 5a29276
Author: devozerov <vo...@gridgain.com>
Authored: Wed Apr 11 16:44:33 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Apr 11 16:44:33 2018 +0300
----------------------------------------------------------------------
.../jdbc/thin/JdbcThinConnectionSelfTest.java | 233 +++++++++++++++----
.../jdbc/thin/ConnectionPropertiesImpl.java | 161 +++++++++----
2 files changed, 300 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/14402e40/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
index 14b91b2..ed0b324 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
@@ -185,86 +185,136 @@ public class JdbcThinConnectionSelfTest extends JdbcThinAbstractSelfTest {
}
/**
+ * Test invalid socket buffer sizes with semicolon.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSocketBuffersSemicolon() throws Exception {
+ final int dfltDufSize = 64 * 1024;
+
+ assertInvalid("jdbc:ignite:thin://127.0.0.1;socketSendBuffer=-1",
+ "Property cannot be lower than 0 [name=socketSendBuffer, value=-1]");
+
+ assertInvalid("jdbc:ignite:thin://127.0.0.1;socketReceiveBuffer=-1",
+ "Property cannot be lower than 0 [name=socketReceiveBuffer, value=-1]");
+
+ // Note that SO_* options are hints, so we check that value is equals to either what we set or to default.
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;socketSendBuffer=1024")) {
+ assertEquals(1024, io(conn).connectionProperties().getSocketSendBuffer());
+ assertEquals(dfltDufSize, io(conn).connectionProperties().getSocketReceiveBuffer());
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;socketReceiveBuffer=1024")) {
+ assertEquals(dfltDufSize, io(conn).connectionProperties().getSocketSendBuffer());
+ assertEquals(1024, io(conn).connectionProperties().getSocketReceiveBuffer());
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;" +
+ "socketSendBuffer=1024;socketReceiveBuffer=2048")) {
+ assertEquals(1024, io(conn).connectionProperties().getSocketSendBuffer());
+ assertEquals(2048, io(conn).connectionProperties().getSocketReceiveBuffer());
+ }
+ }
+
+ /**
* Test SQL hints.
*
* @throws Exception If failed.
*/
public void testSqlHints() throws Exception {
try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1")) {
- assertFalse(io(conn).connectionProperties().isDistributedJoins());
- assertFalse(io(conn).connectionProperties().isEnforceJoinOrder());
- assertFalse(io(conn).connectionProperties().isCollocated());
- assertFalse(io(conn).connectionProperties().isReplicatedOnly());
- assertFalse(io(conn).connectionProperties().isLazy());
- assertFalse(io(conn).connectionProperties().isSkipReducerOnUpdate());
+ assertHints(conn, false, false, false, false, false, false);
}
try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?distributedJoins=true")) {
- assertTrue(io(conn).connectionProperties().isDistributedJoins());
- assertFalse(io(conn).connectionProperties().isEnforceJoinOrder());
- assertFalse(io(conn).connectionProperties().isCollocated());
- assertFalse(io(conn).connectionProperties().isReplicatedOnly());
- assertFalse(io(conn).connectionProperties().isLazy());
- assertFalse(io(conn).connectionProperties().isSkipReducerOnUpdate());
+ assertHints(conn, true, false, false, false, false, false);
}
try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?enforceJoinOrder=true")) {
- assertFalse(io(conn).connectionProperties().isDistributedJoins());
- assertTrue(io(conn).connectionProperties().isEnforceJoinOrder());
- assertFalse(io(conn).connectionProperties().isCollocated());
- assertFalse(io(conn).connectionProperties().isReplicatedOnly());
- assertFalse(io(conn).connectionProperties().isLazy());
- assertFalse(io(conn).connectionProperties().isSkipReducerOnUpdate());
+ assertHints(conn, false, true, false, false, false, false);
}
try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?collocated=true")) {
- assertFalse(io(conn).connectionProperties().isDistributedJoins());
- assertFalse(io(conn).connectionProperties().isEnforceJoinOrder());
- assertTrue(io(conn).connectionProperties().isCollocated());
- assertFalse(io(conn).connectionProperties().isReplicatedOnly());
- assertFalse(io(conn).connectionProperties().isLazy());
- assertFalse(io(conn).connectionProperties().isSkipReducerOnUpdate());
+ assertHints(conn, false, false, true, false, false, false);
}
try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?replicatedOnly=true")) {
- assertFalse(io(conn).connectionProperties().isDistributedJoins());
- assertFalse(io(conn).connectionProperties().isEnforceJoinOrder());
- assertFalse(io(conn).connectionProperties().isCollocated());
- assertTrue(io(conn).connectionProperties().isReplicatedOnly());
- assertFalse(io(conn).connectionProperties().isLazy());
- assertFalse(io(conn).connectionProperties().isSkipReducerOnUpdate());
+ assertHints(conn, false, false, false, true, false, false);
}
try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?lazy=true")) {
- assertFalse(io(conn).connectionProperties().isDistributedJoins());
- assertFalse(io(conn).connectionProperties().isEnforceJoinOrder());
- assertFalse(io(conn).connectionProperties().isCollocated());
- assertFalse(io(conn).connectionProperties().isReplicatedOnly());
- assertTrue(io(conn).connectionProperties().isLazy());
- assertFalse(io(conn).connectionProperties().isSkipReducerOnUpdate());
+ assertHints(conn, false, false, false, false, true, false);
}
try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?skipReducerOnUpdate=true")) {
- assertFalse(io(conn).connectionProperties().isDistributedJoins());
- assertFalse(io(conn).connectionProperties().isEnforceJoinOrder());
- assertFalse(io(conn).connectionProperties().isCollocated());
- assertFalse(io(conn).connectionProperties().isReplicatedOnly());
- assertFalse(io(conn).connectionProperties().isLazy());
- assertTrue(io(conn).connectionProperties().isSkipReducerOnUpdate());
+ assertHints(conn, false, false, false, false, false, true);
}
try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?distributedJoins=true&" +
"enforceJoinOrder=true&collocated=true&replicatedOnly=true&lazy=true&skipReducerOnUpdate=true")) {
- assertTrue(io(conn).connectionProperties().isDistributedJoins());
- assertTrue(io(conn).connectionProperties().isEnforceJoinOrder());
- assertTrue(io(conn).connectionProperties().isCollocated());
- assertTrue(io(conn).connectionProperties().isReplicatedOnly());
- assertTrue(io(conn).connectionProperties().isLazy());
- assertTrue(io(conn).connectionProperties().isSkipReducerOnUpdate());
+ assertHints(conn, true, true, true, true, true, true);
}
}
/**
+ * Test SQL hints with semicolon.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSqlHintsSemicolon() throws Exception {
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;distributedJoins=true")) {
+ assertHints(conn, true, false, false, false, false, false);
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;enforceJoinOrder=true")) {
+ assertHints(conn, false, true, false, false, false, false);
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;collocated=true")) {
+ assertHints(conn, false, false, true, false, false, false);
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;replicatedOnly=true")) {
+ assertHints(conn, false, false, false, true, false, false);
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;lazy=true")) {
+ assertHints(conn, false, false, false, false, true, false);
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;skipReducerOnUpdate=true")) {
+ assertHints(conn, false, false, false, false, false, true);
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;distributedJoins=true;" +
+ "enforceJoinOrder=true;collocated=true;replicatedOnly=true;lazy=true;skipReducerOnUpdate=true")) {
+ assertHints(conn, true, true, true, true, true, true);
+ }
+ }
+
+ /**
+ * Assert hints.
+ *
+ * @param conn Connection.
+ * @param distributedJoins Distributed joins.
+ * @param enforceJoinOrder Enforce join order.
+ * @param collocated Co-located.
+ * @param replicatedOnly Replicated only.
+ * @param lazy Lazy.
+ * @param skipReducerOnUpdate Skip reducer on update.
+ * @throws Exception If failed.
+ */
+ private void assertHints(Connection conn, boolean distributedJoins, boolean enforceJoinOrder, boolean collocated,
+ boolean replicatedOnly, boolean lazy, boolean skipReducerOnUpdate)throws Exception {
+ assertEquals(distributedJoins, io(conn).connectionProperties().isDistributedJoins());
+ assertEquals(enforceJoinOrder, io(conn).connectionProperties().isEnforceJoinOrder());
+ assertEquals(collocated, io(conn).connectionProperties().isCollocated());
+ assertEquals(replicatedOnly, io(conn).connectionProperties().isReplicatedOnly());
+ assertEquals(lazy, io(conn).connectionProperties().isLazy());
+ assertEquals(skipReducerOnUpdate, io(conn).connectionProperties().isSkipReducerOnUpdate());
+ }
+
+ /**
* Test TCP no delay property handling.
*
* @throws Exception If failed.
@@ -304,6 +354,41 @@ public class JdbcThinConnectionSelfTest extends JdbcThinAbstractSelfTest {
}
/**
+ * Test TCP no delay property handling with semicolon.
+ *
+ * @throws Exception If failed.
+ */
+ public void testTcpNoDelaySemicolon() throws Exception {
+ assertInvalid("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=0",
+ "Invalid property value. [name=tcpNoDelay, val=0, choices=[true, false]]");
+
+ assertInvalid("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=1",
+ "Invalid property value. [name=tcpNoDelay, val=1, choices=[true, false]]");
+
+ assertInvalid("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=false1",
+ "Invalid property value. [name=tcpNoDelay, val=false1, choices=[true, false]]");
+
+ assertInvalid("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=true1",
+ "Invalid property value. [name=tcpNoDelay, val=true1, choices=[true, false]]");
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=true")) {
+ assertTrue(io(conn).connectionProperties().isTcpNoDelay());
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=True")) {
+ assertTrue(io(conn).connectionProperties().isTcpNoDelay());
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=false")) {
+ assertFalse(io(conn).connectionProperties().isTcpNoDelay());
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=False")) {
+ assertFalse(io(conn).connectionProperties().isTcpNoDelay());
+ }
+ }
+
+ /**
* Test autoCloseServerCursor property handling.
*
* @throws Exception If failed.
@@ -340,6 +425,38 @@ public class JdbcThinConnectionSelfTest extends JdbcThinAbstractSelfTest {
}
/**
+ * Test autoCloseServerCursor property handling with semicolon.
+ *
+ * @throws Exception If failed.
+ */
+ public void testAutoCloseServerCursorPropertySemicolon() throws Exception {
+ String url = "jdbc:ignite:thin://127.0.0.1;autoCloseServerCursor";
+
+ String err = "Invalid property value. [name=autoCloseServerCursor";
+
+ assertInvalid(url + "=0", err);
+ assertInvalid(url + "=1", err);
+ assertInvalid(url + "=false1", err);
+ assertInvalid(url + "=true1", err);
+
+ try (Connection conn = DriverManager.getConnection(url + "=true")) {
+ assertTrue(io(conn).connectionProperties().isAutoCloseServerCursor());
+ }
+
+ try (Connection conn = DriverManager.getConnection(url + "=True")) {
+ assertTrue(io(conn).connectionProperties().isAutoCloseServerCursor());
+ }
+
+ try (Connection conn = DriverManager.getConnection(url + "=false")) {
+ assertFalse(io(conn).connectionProperties().isAutoCloseServerCursor());
+ }
+
+ try (Connection conn = DriverManager.getConnection(url + "=False")) {
+ assertFalse(io(conn).connectionProperties().isAutoCloseServerCursor());
+ }
+ }
+
+ /**
* Test schema property in URL.
*
* @throws Exception If failed.
@@ -362,6 +479,25 @@ public class JdbcThinConnectionSelfTest extends JdbcThinAbstractSelfTest {
}
/**
+ * Test schema property in URL with semicolon.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSchemaSemicolon() throws Exception {
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;schema=public")) {
+ assertEquals("Invalid schema", "PUBLIC", conn.getSchema());
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;schema=\"" + DEFAULT_CACHE_NAME + '"')) {
+ assertEquals("Invalid schema", DEFAULT_CACHE_NAME, conn.getSchema());
+ }
+
+ try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;schema=_not_exist_schema_")) {
+ assertEquals("Invalid schema", "_NOT_EXIST_SCHEMA_", conn.getSchema());
+ }
+ }
+
+ /**
* Get client socket for connection.
*
* @param conn Connection.
@@ -1010,6 +1146,7 @@ public class JdbcThinConnectionSelfTest extends JdbcThinAbstractSelfTest {
// Invalid parameter value
GridTestUtils.assertThrows(log,
new Callable<Object>() {
+ @SuppressWarnings("MagicConstant")
@Override public Object call() throws Exception {
conn.setTransactionIsolation(-1);
http://git-wip-us.apache.org/repos/asf/ignite/blob/14402e40/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
index 5d77005..86dc298 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
@@ -23,8 +23,6 @@ import java.sql.SQLException;
import java.util.Arrays;
import java.util.Properties;
import java.util.StringTokenizer;
-import javax.naming.RefAddr;
-import javax.naming.Reference;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.ClientConnectorConfiguration;
import org.apache.ignite.internal.processors.odbc.SqlStateCode;
@@ -44,6 +42,9 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
/** Default socket buffer size. */
private static final int DFLT_SOCK_BUFFER_SIZE = 64 * 1024;
+ /** Property: schema. */
+ private static final String PROP_SCHEMA = "schema";
+
/** Connection URL. */
private String url;
@@ -51,7 +52,7 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
private HostAndPortRange [] addrs;
/** Schema name. Hidden property. Is used to set default schema name part of the URL. */
- private StringProperty schema = new StringProperty("schema",
+ private StringProperty schema = new StringProperty(PROP_SCHEMA,
"Schema name of the connection", "PUBLIC", null, false, null);
/** Distributed joins property. */
@@ -487,21 +488,113 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
String nakedUrl = url.substring(JdbcThinUtils.URL_PREFIX.length()).trim();
- int pathPartEndPos = nakedUrl.indexOf('?');
+ parseUrl0(nakedUrl, props);
+ }
+
+ /**
+ * Parse naked URL (i.e. without {@link JdbcThinUtils#URL_PREFIX}).
+ *
+ * @param url Naked URL.
+ * @param props Properties.
+ * @throws SQLException If failed.
+ */
+ private void parseUrl0(String url, Properties props) throws SQLException {
+ // Determine mode - semicolon or ampersand.
+ int semicolonPos = url.indexOf(";");
+ int slashPos = url.indexOf("/");
+ int queryPos = url.indexOf("?");
+
+ boolean semicolonMode;
+
+ if (semicolonPos == -1 && slashPos == -1 && queryPos == -1)
+ // No special char -> any mode could be used, choose semicolon for simplicity.
+ semicolonMode = true;
+ else {
+ if (semicolonPos != -1) {
+ // Use semicolon mode if it appears earlier than slash or query.
+ semicolonMode =
+ (slashPos == -1 || semicolonPos < slashPos) && (queryPos == -1 || semicolonPos < queryPos);
+ }
+ else
+ // Semicolon is not found.
+ semicolonMode = false;
+ }
+
+ if (semicolonMode)
+ parseUrlWithSemicolon(url, props);
+ else
+ parseUrlWithQuery(url, props);
+ }
+
+ /**
+ * Parse URL in semicolon mode.
+ *
+ * @param url Naked URL
+ * @param props Properties.
+ * @throws SQLException If failed.
+ */
+ private void parseUrlWithSemicolon(String url, Properties props) throws SQLException {
+ int pathPartEndPos = url.indexOf(';');
if (pathPartEndPos == -1)
- pathPartEndPos = nakedUrl.length();
+ pathPartEndPos = url.length();
- String pathPart = nakedUrl.substring(0, pathPartEndPos);
+ String pathPart = url.substring(0, pathPartEndPos);
String paramPart = null;
- if (pathPartEndPos > 0 && pathPartEndPos < nakedUrl.length())
- paramPart = nakedUrl.substring(pathPartEndPos + 1, nakedUrl.length());
+ if (pathPartEndPos > 0 && pathPartEndPos < url.length())
+ paramPart = url.substring(pathPartEndPos + 1, url.length());
+
+ parseEndpoints(pathPart);
+
+ if (!F.isEmpty(paramPart))
+ parseParameters(paramPart, props, ";");
+ }
+
+ /**
+ * Parse URL in query mode.
+ *
+ * @param url Naked URL
+ * @param props Properties.
+ * @throws SQLException If failed.
+ */
+ private void parseUrlWithQuery(String url, Properties props) throws SQLException {
+ int pathPartEndPos = url.indexOf('?');
+
+ if (pathPartEndPos == -1)
+ pathPartEndPos = url.length();
+
+ String pathPart = url.substring(0, pathPartEndPos);
+
+ String paramPart = null;
+
+ if (pathPartEndPos > 0 && pathPartEndPos < url.length())
+ paramPart = url.substring(pathPartEndPos + 1, url.length());
String[] pathParts = pathPart.split("/");
- String [] endpoints = pathParts[0].split(",");
+ parseEndpoints(pathParts[0]);
+
+ if (pathParts.length > 2) {
+ throw new SQLException("Invalid URL format (only schema name is allowed in URL path parameter " +
+ "'host:port[/schemaName]'): " + this.url, SqlStateCode.CLIENT_CONNECTION_FAILED);
+ }
+
+ setSchema(pathParts.length == 2 ? pathParts[1] : null);
+
+ if (!F.isEmpty(paramPart))
+ parseParameters(paramPart, props, "&");
+ }
+
+ /**
+ * Parse endpoints.
+ *
+ * @param endpointStr Endpoint string.
+ * @throws SQLException If failed.
+ */
+ private void parseEndpoints(String endpointStr) throws SQLException {
+ String [] endpoints = endpointStr.split(",");
if (endpoints.length > 0)
addrs = new HostAndPortRange[endpoints.length];
@@ -519,16 +612,6 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
if (F.isEmpty(addrs) || F.isEmpty(addrs[0].host()))
throw new SQLException("Host name is empty", SqlStateCode.CLIENT_CONNECTION_FAILED);
-
- if (pathParts.length > 2) {
- throw new SQLException("Invalid URL format (only schema name is allowed in URL path parameter " +
- "'host:port[/schemaName]'): " + url, SqlStateCode.CLIENT_CONNECTION_FAILED);
- }
-
- setSchema(pathParts.length == 2 ? pathParts[1] : null);
-
- if (!F.isEmpty(paramPart))
- parseParameters(paramPart, props);
}
/**
@@ -536,10 +619,11 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
*
* @param paramStr Parameters string.
* @param props Properties.
+ * @param delimChar Delimiter character.
* @throws SQLException If failed.
*/
- private void parseParameters(String paramStr, Properties props) throws SQLException {
- StringTokenizer st = new StringTokenizer(paramStr, "&");
+ private void parseParameters(String paramStr, Properties props, String delimChar) throws SQLException {
+ StringTokenizer st = new StringTokenizer(paramStr, delimChar);
boolean insideBrace = false;
@@ -553,8 +637,8 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
int eqSymPos = token.indexOf('=');
if (eqSymPos < 0) {
- throw new SQLException("Invalid parameter format " +
- "(URL properties format: key0=value0&key1=value1&... etc. pair: " + token);
+ throw new SQLException("Invalid parameter format (should be \"key1=val1" + delimChar +
+ "key2=val2" + delimChar + "...\"): " + token);
}
if (eqSymPos == token.length())
@@ -570,7 +654,7 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
}
}
else
- val += "&" + token;
+ val += delimChar + token;
if (val.endsWith("}")) {
insideBrace = false;
@@ -587,22 +671,24 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
if (key.isEmpty() || val.isEmpty())
throw new SQLException("Invalid parameter format (key and value cannot be empty): " + token);
- props.setProperty(PROP_PREFIX + key, val);
+ if (PROP_SCHEMA.equalsIgnoreCase(key))
+ setSchema(val);
+ else
+ props.setProperty(PROP_PREFIX + key, val);
}
}
}
-
/**
* @return Driver's properties info array.
*/
public DriverPropertyInfo[] getDriverPropertyInfo() {
- DriverPropertyInfo[] dpis = new DriverPropertyInfo[propsArray.length];
+ DriverPropertyInfo[] infos = new DriverPropertyInfo[propsArray.length];
for (int i = 0; i < propsArray.length; ++i)
- dpis[i] = propsArray[i].getDriverPropertyInfo();
+ infos[i] = propsArray[i].getDriverPropertyInfo();
- return dpis;
+ return infos;
}
/**
@@ -741,23 +827,6 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
}
/**
- * @param ref Reference object.
- * @throws SQLException On error.
- */
- void init(Reference ref) throws SQLException {
- RefAddr refAddr = ref.get(name);
-
- if (refAddr != null) {
- String str = (String) refAddr.getContent();
-
- if (validator != null)
- validator.validate(str);
-
- init(str);
- }
- }
-
- /**
* @param str String representation of the
* @throws SQLException on error.
*/
[09/54] [abbrv] ignite git commit: IGNITE-7222 Added ZooKeeper
discovery SPI
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
new file mode 100644
index 0000000..c42fa57
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
@@ -0,0 +1,519 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.internal;
+
+import java.net.InetSocketAddress;
+import java.nio.channels.SocketChannel;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
+import org.apache.ignite.internal.util.GridLeanMap;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.nio.GridNioServer;
+import org.apache.ignite.internal.util.nio.GridNioSession;
+import org.apache.ignite.internal.util.nio.GridNioSessionMetaKey;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+
+/**
+ * Tcp Communication Connection Check Future.
+ */
+public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<BitSet> implements IgniteSpiTimeoutObject, GridLocalEventListener {
+ /** Session future. */
+ public static final int SES_FUT_META = GridNioSessionMetaKey.nextUniqueKey();
+
+ /** */
+ private static final AtomicIntegerFieldUpdater<SingleAddressConnectFuture> connFutDoneUpdater =
+ AtomicIntegerFieldUpdater.newUpdater(SingleAddressConnectFuture.class, "done");
+
+ /** */
+ private static final AtomicIntegerFieldUpdater<MultipleAddressesConnectFuture> connResCntUpdater =
+ AtomicIntegerFieldUpdater.newUpdater(MultipleAddressesConnectFuture.class, "resCnt");
+
+ /** */
+ private final AtomicInteger resCntr = new AtomicInteger();
+
+ /** */
+ private final List<ClusterNode> nodes;
+
+ /** */
+ private volatile ConnectFuture[] futs;
+
+ /** */
+ private final GridNioServer nioSrvr;
+
+ /** */
+ private final TcpCommunicationSpi spi;
+
+ /** */
+ private final IgniteUuid timeoutObjId = IgniteUuid.randomUuid();
+
+ /** */
+ private final BitSet resBitSet;
+
+ /** */
+ private long endTime;
+
+ /** */
+ private final IgniteLogger log;
+
+ /**
+ * @param spi SPI instance.
+ * @param log Logger.
+ * @param nioSrvr NIO server.
+ * @param nodes Nodes to check.
+ */
+ public TcpCommunicationConnectionCheckFuture(TcpCommunicationSpi spi,
+ IgniteLogger log,
+ GridNioServer nioSrvr,
+ List<ClusterNode> nodes)
+ {
+ this.spi = spi;
+ this.log = log;
+ this.nioSrvr = nioSrvr;
+ this.nodes = nodes;
+
+ resBitSet = new BitSet(nodes.size());
+ }
+
+ /**
+ * @param timeout Connect timeout.
+ */
+ public void init(long timeout) {
+ ConnectFuture[] futs = new ConnectFuture[nodes.size()];
+
+ UUID locId = spi.getSpiContext().localNode().id();
+
+ for (int i = 0; i < nodes.size(); i++) {
+ ClusterNode node = nodes.get(i);
+
+ if (!node.id().equals(locId)) {
+ if (spi.getSpiContext().node(node.id()) == null) {
+ receivedConnectionStatus(i, false);
+
+ continue;
+ }
+
+ Collection<InetSocketAddress> addrs;
+
+ try {
+ addrs = spi.nodeAddresses(node, false);
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to get node addresses: " + node, e);
+
+ receivedConnectionStatus(i, false);
+
+ continue;
+ }
+
+ if (addrs.size() == 1) {
+ SingleAddressConnectFuture fut = new SingleAddressConnectFuture(i);
+
+ fut.init(addrs.iterator().next(), node.id());
+
+ futs[i] = fut;
+ }
+ else {
+ MultipleAddressesConnectFuture fut = new MultipleAddressesConnectFuture(i);
+
+ fut.init(addrs, node.id());
+
+ futs[i] = fut;
+ }
+ }
+ else
+ receivedConnectionStatus(i, true);
+ }
+
+ this.futs = futs;
+
+ spi.getSpiContext().addLocalEventListener(this, EVT_NODE_LEFT, EVT_NODE_FAILED);
+
+ if (!isDone()) {
+ endTime = System.currentTimeMillis() + timeout;
+
+ spi.getSpiContext().addTimeoutObject(this);
+ }
+ }
+
+ /**
+ * @param idx Node index.
+ * @param res Success flag.
+ */
+ private void receivedConnectionStatus(int idx, boolean res) {
+ assert resCntr.get() < nodes.size();
+
+ synchronized (resBitSet) {
+ resBitSet.set(idx, res);
+ }
+
+ if (resCntr.incrementAndGet() == nodes.size())
+ onDone(resBitSet);
+ }
+
+ /**
+ * @param nodeIdx Node index.
+ * @return Node ID.
+ */
+ private UUID nodeId(int nodeIdx) {
+ return nodes.get(nodeIdx).id();
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid id() {
+ return timeoutObjId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long endTime() {
+ return endTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onEvent(Event evt) {
+ if (isDone())
+ return;
+
+ assert evt instanceof DiscoveryEvent : evt;
+ assert evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED ;
+
+ UUID nodeId = ((DiscoveryEvent)evt).eventNode().id();
+
+ for (int i = 0; i < nodes.size(); i++) {
+ if (nodes.get(i).id().equals(nodeId)) {
+ ConnectFuture fut = futs[i];
+
+ if (fut != null)
+ fut.onNodeFailed();
+
+ return;
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onTimeout() {
+ if (isDone())
+ return;
+
+ ConnectFuture[] futs = this.futs;
+
+ for (int i = 0; i < futs.length; i++) {
+ ConnectFuture fut = futs[i];
+
+ if (fut != null)
+ fut.onTimeout();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean onDone(@Nullable BitSet res, @Nullable Throwable err) {
+ if (super.onDone(res, err)) {
+ spi.getSpiContext().removeTimeoutObject(this);
+
+ spi.getSpiContext().removeLocalEventListener(this);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ *
+ */
+ private interface ConnectFuture {
+ /**
+ *
+ */
+ void onTimeout();
+
+ /**
+ *
+ */
+ void onNodeFailed();
+ }
+
+ /**
+ *
+ */
+ private class SingleAddressConnectFuture implements TcpCommunicationNodeConnectionCheckFuture, ConnectFuture {
+ /** */
+ final int nodeIdx;
+
+ /** */
+ volatile int done;
+
+ /** */
+ Map<Integer, Object> sesMeta;
+
+ /** */
+ private SocketChannel ch;
+
+ /**
+ * @param nodeIdx Node index.
+ */
+ SingleAddressConnectFuture(int nodeIdx) {
+ this.nodeIdx = nodeIdx;
+ }
+
+ /**
+ * @param addr Node address.
+ * @param rmtNodeId Id of node to open connection check session with.
+ */
+ public void init(InetSocketAddress addr, UUID rmtNodeId) {
+ boolean connect;
+
+ try {
+ ch = SocketChannel.open();
+
+ ch.configureBlocking(false);
+
+ ch.socket().setTcpNoDelay(true);
+ ch.socket().setKeepAlive(false);
+
+ connect = ch.connect(addr);
+ }
+ catch (Exception e) {
+ finish(false);
+
+ return;
+ }
+
+ if (!connect) {
+ sesMeta = new GridLeanMap<>(3);
+
+ // Set dummy key to identify connection-check outgoing connection.
+ sesMeta.put(TcpCommunicationSpi.CONN_IDX_META, new ConnectionKey(rmtNodeId, -1, -1, true));
+ sesMeta.put(SES_FUT_META, this);
+
+ nioSrvr.createSession(ch, sesMeta, true, new IgniteInClosure<IgniteInternalFuture<GridNioSession>>() {
+ @Override public void apply(IgniteInternalFuture<GridNioSession> fut) {
+ if (fut.error() != null)
+ finish(false);
+ }
+ });
+ }
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("unchecked")
+ void cancel() {
+ if (finish(false))
+ nioSrvr.cancelConnect(ch, sesMeta);
+ }
+
+ /** {@inheritDoc} */
+ public void onTimeout() {
+ cancel();
+ }
+
+ /** {@inheritDoc} */
+ public void onConnected(UUID rmtNodeId) {
+ finish(nodeId(nodeIdx).equals(rmtNodeId));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onNodeFailed() {
+ cancel();
+ }
+
+ /**
+ * @param res Result.
+ * @return {@code True} if result was set by this call.
+ */
+ public boolean finish(boolean res) {
+ if (connFutDoneUpdater.compareAndSet(this, 0, 1)) {
+ onStatusReceived(res);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @param res Result.
+ */
+ void onStatusReceived(boolean res) {
+ receivedConnectionStatus(nodeIdx, res);
+ }
+ }
+
+ /**
+ *
+ */
+ private class MultipleAddressesConnectFuture implements ConnectFuture {
+ /** */
+ volatile int resCnt;
+
+ /** */
+ volatile SingleAddressConnectFuture[] futs;
+
+ /** */
+ final int nodeIdx;
+
+ /**
+ * @param nodeIdx Node index.
+ */
+ MultipleAddressesConnectFuture(int nodeIdx) {
+ this.nodeIdx = nodeIdx;
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onNodeFailed() {
+ SingleAddressConnectFuture[] futs = this.futs;
+
+ for (int i = 0; i < futs.length; i++) {
+ ConnectFuture fut = futs[i];
+
+ if (fut != null)
+ fut.onNodeFailed();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onTimeout() {
+ SingleAddressConnectFuture[] futs = this.futs;
+
+ for (int i = 0; i < futs.length; i++) {
+ ConnectFuture fut = futs[i];
+
+ if (fut != null)
+ fut.onTimeout();
+ }
+ }
+
+ /**
+ * @param addrs Node addresses.
+ * @param rmtNodeId Id of node to open connection check session with.
+ */
+ void init(Collection<InetSocketAddress> addrs, UUID rmtNodeId) {
+ SingleAddressConnectFuture[] futs = new SingleAddressConnectFuture[addrs.size()];
+
+ for (int i = 0; i < addrs.size(); i++) {
+ SingleAddressConnectFuture fut = new SingleAddressConnectFuture(nodeIdx) {
+ @Override void onStatusReceived(boolean res) {
+ receivedAddressStatus(res);
+ }
+ };
+
+ futs[i] = fut;
+ }
+
+ this.futs = futs;
+
+ int idx = 0;
+
+ for (InetSocketAddress addr : addrs) {
+ futs[idx++].init(addr, rmtNodeId);
+
+ if (resCnt == Integer.MAX_VALUE)
+ return;
+ }
+
+ // Close race.
+ if (done())
+ cancelFutures();
+ }
+
+ /**
+ * @return {@code True}
+ */
+ private boolean done() {
+ int resCnt0 = resCnt;
+
+ return resCnt0 == Integer.MAX_VALUE || resCnt0 == futs.length;
+ }
+
+ /**
+ *
+ */
+ private void cancelFutures() {
+ SingleAddressConnectFuture[] futs = this.futs;
+
+ if (futs != null) {
+ for (int i = 0; i < futs.length; i++) {
+ SingleAddressConnectFuture fut = futs[i];
+
+ fut.cancel();
+ }
+ }
+ }
+
+ /**
+ * @param res Result.
+ */
+ void receivedAddressStatus(boolean res) {
+ if (res) {
+ for (;;) {
+ int resCnt0 = resCnt;
+
+ if (resCnt0 == Integer.MAX_VALUE)
+ return;
+
+ if (connResCntUpdater.compareAndSet(this, resCnt0, Integer.MAX_VALUE)) {
+ receivedConnectionStatus(nodeIdx, true);
+
+ cancelFutures(); // Cancel others connects if they are still in progress.
+
+ return;
+ }
+ }
+ }
+ else {
+ for (;;) {
+ int resCnt0 = resCnt;
+
+ if (resCnt0 == Integer.MAX_VALUE)
+ return;
+
+ int resCnt1 = resCnt0 + 1;
+
+ if (connResCntUpdater.compareAndSet(this, resCnt0, resCnt1)) {
+ if (resCnt1 == futs.length)
+ receivedConnectionStatus(nodeIdx, false);
+
+ return;
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationNodeConnectionCheckFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationNodeConnectionCheckFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationNodeConnectionCheckFuture.java
new file mode 100644
index 0000000..cbf27b5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationNodeConnectionCheckFuture.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.internal;
+
+import java.util.UUID;
+
+/**
+ * Tcp Communication Node Connection Check Future.
+ */
+public interface TcpCommunicationNodeConnectionCheckFuture {
+ /**
+ * @param nodeId Remote node ID.
+ */
+ public void onConnected(UUID nodeId);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiCustomMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiCustomMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiCustomMessage.java
index a0f9b75..f26ad33 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiCustomMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiCustomMessage.java
@@ -30,12 +30,23 @@ import org.jetbrains.annotations.Nullable;
*/
public interface DiscoverySpiCustomMessage extends Serializable {
/**
- * Called when message passed the ring.
+ * Called when custom message has been handled by all nodes.
+ *
+ * @return Ack message or {@code null} if ack is not required.
*/
@Nullable public DiscoverySpiCustomMessage ackMessage();
/**
- * @return {@code true} if message can be modified during listener notification. Changes will be send to next nodes.
+ * @return {@code True} if message can be modified during listener notification. Changes will be send to next nodes.
*/
public boolean isMutable();
+
+ /**
+ * Called on discovery coordinator node after listener is notified. If returns {@code true}
+ * then message is not passed to others nodes, if after this method {@link #ackMessage()} returns non-null ack
+ * message, it is sent to all nodes.
+ *
+ * @return {@code True} if message should not be sent to all nodes.
+ */
+ public boolean stopProcess();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMutableCustomMessageSupport.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMutableCustomMessageSupport.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMutableCustomMessageSupport.java
new file mode 100644
index 0000000..37aa323
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMutableCustomMessageSupport.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * This annotation is for all implementations of {@link DiscoverySpi} that support
+ * topology mutable {@link DiscoverySpiCustomMessage}s.
+ */
+@Documented
+@Inherited
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE})
+public @interface DiscoverySpiMutableCustomMessageSupport {
+ /**
+ * @return Whether or not target SPI supports mutable {@link DiscoverySpiCustomMessage}s.
+ */
+ public boolean value();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 2d9a314..f0a5186 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -55,6 +55,8 @@ import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.AddressResolver;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.F;
@@ -88,6 +90,7 @@ import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange;
import org.apache.ignite.spi.discovery.DiscoverySpiHistorySupport;
import org.apache.ignite.spi.discovery.DiscoverySpiListener;
+import org.apache.ignite.spi.discovery.DiscoverySpiMutableCustomMessageSupport;
import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
import org.apache.ignite.spi.discovery.tcp.internal.DiscoveryDataPacket;
@@ -103,6 +106,7 @@ import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAuthFailedMessag
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryCheckFailedMessage;
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryDuplicateIdMessage;
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryEnsureDelivery;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryJoinRequestMessage;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_CONSISTENT_ID_BY_HOST_WITHOUT_PORT;
@@ -223,7 +227,8 @@ import static org.apache.ignite.IgniteSystemProperties.getBoolean;
@IgniteSpiMultipleInstancesSupport(true)
@DiscoverySpiOrderSupport(true)
@DiscoverySpiHistorySupport(true)
-public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
+@DiscoverySpiMutableCustomMessageSupport(true)
+public class TcpDiscoverySpi extends IgniteSpiAdapter implements IgniteDiscoverySpi {
/** Node attribute that is mapped to node's external addresses (value is <tt>disc.tcp.ext-addrs</tt>). */
public static final String ATTR_EXT_ADDRS = "disc.tcp.ext-addrs";
@@ -409,6 +414,9 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
/** */
protected IgniteSpiContext spiCtx;
+ /** */
+ private IgniteDiscoverySpiInternalListener internalLsnr;
+
/**
* Gets current SPI state.
*
@@ -473,6 +481,13 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
/** {@inheritDoc} */
@Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) throws IgniteException {
+ IgniteDiscoverySpiInternalListener internalLsnr = this.internalLsnr;
+
+ if (internalLsnr != null) {
+ if (!internalLsnr.beforeSendCustomEvent(this, log, msg))
+ return;
+ }
+
impl.sendCustomEvent(msg);
}
@@ -1559,6 +1574,9 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
OutputStream out,
TcpDiscoveryAbstractMessage msg,
long timeout) throws IOException, IgniteCheckedException {
+ if (internalLsnr != null && msg instanceof TcpDiscoveryJoinRequestMessage)
+ internalLsnr.beforeJoin(locNode, log);
+
assert sock != null;
assert msg != null;
assert out != null;
@@ -2118,15 +2136,31 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
return ignite().configuration().getSslContextFactory() != null;
}
- /**
- * Force reconnect to cluster.
- *
- * @throws IgniteSpiException If failed.
- */
- public void reconnect() throws IgniteSpiException {
+ /** {@inheritDoc} */
+ public void clientReconnect() throws IgniteSpiException {
impl.reconnect();
}
+ /** {@inheritDoc} */
+ @Override public boolean knownNode(UUID nodeId) {
+ return getNode0(nodeId) != null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean clientReconnectSupported() {
+ return !clientReconnectDisabled;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean supportsCommunicationFailureResolve() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resolveCommunicationFailure(ClusterNode node, Exception err) {
+ throw new UnsupportedOperationException();
+ }
+
/**
* <strong>FOR TEST ONLY!!!</strong>
*/
@@ -2148,6 +2182,11 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
sndMsgLsnrs.add(lsnr);
}
+ /** {@inheritDoc} */
+ @Override public void setInternalListener(IgniteDiscoverySpiInternalListener lsnr) {
+ this.internalLsnr = lsnr;
+ }
+
/**
* <strong>FOR TEST ONLY!!!</strong>
*/
@@ -2185,7 +2224,7 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
* <p>
* This method is intended for test purposes only.
*/
- protected void simulateNodeFailure() {
+ public void simulateNodeFailure() {
impl.simulateNodeFailure();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
index 01534f7..55fe4e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
@@ -33,9 +33,9 @@ import java.util.UUID;
import org.apache.ignite.cache.CacheMetrics;
import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.ClusterMetricsSnapshot;
import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -58,7 +58,7 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTE
* <strong>This class is not intended for public use</strong> and has been made
* <tt>public</tt> due to certain limitations of Java technology.
*/
-public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements ClusterNode,
+public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements IgniteClusterNode,
Comparable<TcpDiscoveryNode>, Externalizable {
/** */
private static final long serialVersionUID = 0L;
@@ -291,26 +291,14 @@ public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements Cluste
return metrics;
}
- /**
- * Sets node metrics.
- *
- * @param metrics Node metrics.
- */
+ /** {@inheritDoc} */
public void setMetrics(ClusterMetrics metrics) {
assert metrics != null;
this.metrics = metrics;
}
- /**
- * Gets collections of cache metrics for this node. Note that node cache metrics are constantly updated
- * and provide up to date information about caches.
- * <p>
- * Cache metrics are updated with some delay which is directly related to metrics update
- * frequency. For example, by default the update will happen every {@code 2} seconds.
- *
- * @return Runtime metrics snapshots for this node.
- */
+ /** {@inheritDoc} */
public Map<Integer, CacheMetrics> cacheMetrics() {
if (metricsProvider != null) {
Map<Integer, CacheMetrics> cacheMetrics0 = metricsProvider.cacheMetrics();
@@ -323,11 +311,7 @@ public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements Cluste
return cacheMetrics;
}
- /**
- * Sets node cache metrics.
- *
- * @param cacheMetrics Cache metrics.
- */
+ /** {@inheritDoc} */
public void setCacheMetrics(Map<Integer, CacheMetrics> cacheMetrics) {
this.cacheMetrics = cacheMetrics != null ? cacheMetrics : Collections.<Integer, CacheMetrics>emptyMap();
}
@@ -544,11 +528,7 @@ public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements Cluste
return node;
}
- /**
- * Whether this node is cache client (see {@link IgniteConfiguration#isClientMode()}).
- *
- * @return {@code True if client}.
- */
+ /** {@inheritDoc} */
public boolean isCacheClient() {
if (!cacheCliInit) {
cacheCli = CU.clientNodeDirect(this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/resources/META-INF/classnames.properties
----------------------------------------------------------------------
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index f0f143d..6dc3d85 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -832,6 +832,7 @@ org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPar
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$1
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$2
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloaderAssignments
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionCountersMap
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionCountersMap2
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionHistorySuppliersMap
@@ -1129,6 +1130,7 @@ org.apache.ignite.internal.processors.closure.GridClosureProcessor$T8
org.apache.ignite.internal.processors.closure.GridClosureProcessor$T9
org.apache.ignite.internal.processors.closure.GridClosureProcessor$TaskNoReduceAdapter
org.apache.ignite.internal.processors.closure.GridPeerDeployAwareTaskAdapter
+org.apache.ignite.internal.processors.cluster.ClusterNodeMetrics
org.apache.ignite.internal.processors.cluster.BaselineTopology
org.apache.ignite.internal.processors.cluster.BaselineTopologyHistory
org.apache.ignite.internal.processors.cluster.BaselineTopologyHistoryItem
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionExcludeNeighborsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionExcludeNeighborsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionExcludeNeighborsAbstractSelfTest.java
index 900d4f5..eee47c7 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionExcludeNeighborsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionExcludeNeighborsAbstractSelfTest.java
@@ -32,7 +32,6 @@ import org.apache.ignite.internal.IgniteNodeAttributes;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteProductVersion;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -124,12 +123,9 @@ public abstract class AffinityFunctionExcludeNeighborsAbstractSelfTest extends G
Affinity<Object> aff = g.affinity(DEFAULT_CACHE_NAME);
- List<TcpDiscoveryNode> top = new ArrayList<>();
+ List<ClusterNode> top = new ArrayList<>(g.cluster().nodes());
- for (ClusterNode node : g.cluster().nodes())
- top.add((TcpDiscoveryNode) node);
-
- Collections.sort(top);
+ Collections.sort((List)top);
assertEquals(grids, top.size());
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java b/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java
index 4e4d75a..5eca7d6 100644
--- a/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java
@@ -120,6 +120,10 @@ public class FailureHandlerTriggeredTest extends GridCommonAbstractTest {
@Override public boolean isMutable() {
return false;
}
+
+ @Override public boolean stopProcess() {
+ return false;
+ }
});
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java
index 2328c84..141f4af 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java
@@ -61,6 +61,9 @@ public class ClusterGroupHostsSelfTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
public void testForHosts() throws Exception {
+ if (!tcpDiscovery())
+ return;
+
Ignite ignite = grid();
assertEquals(1, ignite.cluster().forHost("h_1").nodes().size());
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupSelfTest.java
index 9df561a..99006d1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupSelfTest.java
@@ -68,6 +68,8 @@ public class ClusterGroupSelfTest extends ClusterGroupAbstractTest {
if (i == 0)
ignite = g;
}
+
+ waitForTopology(NODES_CNT);
}
finally {
Ignition.setClientMode(false);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsUpdateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsUpdateTest.java
new file mode 100644
index 0000000..6e6b4a4
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsUpdateTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import junit.framework.AssertionFailedError;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class ClusterNodeMetricsUpdateTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private boolean client;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+ cfg.setMetricsUpdateFrequency(500);
+
+ cfg.setClientMode(client);
+
+ return cfg;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMetrics() throws Exception {
+ int NODES = 6;
+
+ Ignite srv0 = startGridsMultiThreaded(NODES / 2);
+
+ client = true;
+
+ startGridsMultiThreaded(NODES / 2, NODES / 2);
+
+ Map<UUID, Integer> expJobs = new HashMap<>();
+
+ for (int i = 0; i < NODES; i++)
+ expJobs.put(nodeId(i), 0);
+
+ checkMetrics(NODES, expJobs);
+
+ for (int i = 0; i < NODES; i++) {
+ UUID nodeId = nodeId(i);
+
+ IgniteCompute c = srv0.compute(srv0.cluster().forNodeId(nodeId(i)));
+
+ c.call(new DummyCallable(null));
+
+ expJobs.put(nodeId, 1);
+ }
+ }
+
+ /**
+ * @param expNodes Expected nodes.
+ * @param expJobs Expected jobs number per node.
+ */
+ private void checkMetrics0(int expNodes, Map<UUID, Integer> expJobs) {
+ List<Ignite> nodes = Ignition.allGrids();
+
+ assertEquals(expNodes, nodes.size());
+ assertEquals(expNodes, expJobs.size());
+
+ int totalJobs = 0;
+
+ for (Integer c : expJobs.values())
+ totalJobs += c;
+
+ for (final Ignite ignite : nodes) {
+ ClusterMetrics m = ignite.cluster().metrics();
+
+ assertEquals(expNodes, m.getTotalNodes());
+ assertEquals(totalJobs, m.getTotalExecutedJobs());
+
+ for (Map.Entry<UUID, Integer> e : expJobs.entrySet()) {
+ UUID nodeId = e.getKey();
+
+ ClusterGroup g = ignite.cluster().forNodeId(nodeId);
+
+ ClusterMetrics nodeM = g.metrics();
+
+ assertEquals(e.getValue(), (Integer)nodeM.getTotalExecutedJobs());
+ }
+ }
+ }
+
+ /**
+ * @param expNodes Expected nodes.
+ * @param expJobs Expected jobs number per node.
+ * @throws Exception If failed.
+ */
+ private void checkMetrics(final int expNodes, final Map<UUID, Integer> expJobs) throws Exception {
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ try {
+ checkMetrics0(expNodes, expJobs);
+ }
+ catch (AssertionFailedError e) {
+ return false;
+ }
+
+ return true;
+ }
+ }, 5000);
+
+ checkMetrics0(expNodes, expJobs);
+ }
+
+ /**
+ *
+ */
+ private static class DummyCallable implements IgniteCallable<Object> {
+ /** */
+ private byte[] data;
+
+ /**
+ * @param data Data.
+ */
+ DummyCallable(byte[] data) {
+ this.data = data;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object call() throws Exception {
+ return data;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/DiscoverySpiTestListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/DiscoverySpiTestListener.java b/modules/core/src/test/java/org/apache/ignite/internal/DiscoverySpiTestListener.java
new file mode 100644
index 0000000..46d9edc
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/DiscoverySpiTestListener.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Test callback for discovery SPI.
+ * <p>
+ * Allows block/delay node join and custom event sending.
+ */
+public class DiscoverySpiTestListener implements IgniteDiscoverySpiInternalListener {
+ /** */
+ private volatile CountDownLatch joinLatch;
+
+ /** */
+ private Set<Class<?>> blockCustomEvtCls;
+
+ /** */
+ private final Object mux = new Object();
+
+ /** */
+ private List<DiscoverySpiCustomMessage> blockedMsgs = new ArrayList<>();
+
+ /** */
+ private volatile DiscoverySpi spi;
+
+ /** */
+ private volatile IgniteLogger log;
+
+ /**
+ *
+ */
+ public void startBlockJoin() {
+ joinLatch = new CountDownLatch(1);
+ }
+
+ /**
+ *
+ */
+ public void stopBlockJoin() {
+ joinLatch.countDown();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void beforeJoin(ClusterNode locNode, IgniteLogger log) {
+ try {
+ CountDownLatch writeLatch0 = joinLatch;
+
+ if (writeLatch0 != null) {
+ log.info("Block join");
+
+ U.await(writeLatch0);
+ }
+ }
+ catch (Exception e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean beforeSendCustomEvent(DiscoverySpi spi, IgniteLogger log, DiscoverySpiCustomMessage msg) {
+ this.spi = spi;
+ this.log = log;
+
+ synchronized (mux) {
+ if (blockCustomEvtCls != null) {
+ DiscoveryCustomMessage msg0 = GridTestUtils.getFieldValue(msg, "delegate");
+
+ if (blockCustomEvtCls.contains(msg0.getClass())) {
+ log.info("Block custom message: " + msg0);
+
+ blockedMsgs.add(msg);
+
+ mux.notifyAll();
+
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * @param blockCustomEvtCls Event class to block.
+ */
+ public void blockCustomEvent(Class<?> cls0, Class<?> ... blockCustomEvtCls) {
+ synchronized (mux) {
+ assert blockedMsgs.isEmpty() : blockedMsgs;
+
+ this.blockCustomEvtCls = new HashSet<>();
+
+ this.blockCustomEvtCls.add(cls0);
+
+ Collections.addAll(this.blockCustomEvtCls, blockCustomEvtCls);
+ }
+ }
+
+ /**
+ * @throws InterruptedException If interrupted.
+ */
+ public void waitCustomEvent() throws InterruptedException {
+ synchronized (mux) {
+ while (blockedMsgs.isEmpty())
+ mux.wait();
+ }
+ }
+
+ /**
+ *
+ */
+ public void stopBlockCustomEvents() {
+ if (spi == null)
+ return;
+
+ List<DiscoverySpiCustomMessage> msgs;
+
+ synchronized (this) {
+ msgs = new ArrayList<>(blockedMsgs);
+
+ blockCustomEvtCls = null;
+
+ blockedMsgs.clear();
+ }
+
+ for (DiscoverySpiCustomMessage msg : msgs) {
+ log.info("Resend blocked message: " + msg);
+
+ spi.sendCustomEvent(msg);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/GridDiscoverySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridDiscoverySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridDiscoverySelfTest.java
index e6b678b..883d677 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridDiscoverySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridDiscoverySelfTest.java
@@ -49,6 +49,7 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.jetbrains.annotations.Nullable;
import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.lang.IgniteProductVersion.fromString;
@@ -158,10 +159,10 @@ public class GridDiscoverySelfTest extends GridCommonAbstractTest {
final AtomicInteger cnt = new AtomicInteger();
- /** Joined nodes counter. */
+ // Joined nodes counter.
final CountDownLatch joinedCnt = new CountDownLatch(NODES_CNT);
- /** Left nodes counter. */
+ // Left nodes counter.
final CountDownLatch leftCnt = new CountDownLatch(NODES_CNT);
IgnitePredicate<Event> lsnr = new IgnitePredicate<Event>() {
@@ -171,7 +172,7 @@ public class GridDiscoverySelfTest extends GridCommonAbstractTest {
joinedCnt.countDown();
}
- else if (EVT_NODE_LEFT == evt.type()) {
+ else if (EVT_NODE_LEFT == evt.type() || EVT_NODE_FAILED == evt.type()) {
int i = cnt.decrementAndGet();
assert i >= 0;
@@ -185,7 +186,10 @@ public class GridDiscoverySelfTest extends GridCommonAbstractTest {
}
};
- ignite.events().localListen(lsnr, EVT_NODE_LEFT, EVT_NODE_JOINED);
+ int[] evts = tcpDiscovery() ? new int[]{EVT_NODE_LEFT, EVT_NODE_JOINED} :
+ new int[]{EVT_NODE_LEFT, EVT_NODE_FAILED, EVT_NODE_JOINED};
+
+ ignite.events().localListen(lsnr, evts);
try {
for (int i = 0; i < NODES_CNT; i++)
@@ -242,6 +246,8 @@ public class GridDiscoverySelfTest extends GridCommonAbstractTest {
for (int i = 0; i < NODES_CNT; i++)
stopGrid(i);
+ waitForTopology(1);
+
final long topVer = discoMgr.topologyVersion();
assert topVer == topVer0 + NODES_CNT * 2 : "Unexpected topology version: " + topVer;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/GridJobMasterLeaveAwareSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridJobMasterLeaveAwareSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridJobMasterLeaveAwareSelfTest.java
index cd6b2c0..a8be541 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridJobMasterLeaveAwareSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridJobMasterLeaveAwareSelfTest.java
@@ -259,6 +259,8 @@ public class GridJobMasterLeaveAwareSelfTest extends GridCommonAbstractTest {
// Now we stop master grid.
stopGrid(lastGridIdx, true);
+ waitForTopology(GRID_CNT - 1);
+
// Release communication SPI wait latches. As master node is stopped, job worker will receive and exception.
for (int i = 0; i < lastGridIdx; i++)
((CommunicationSpi)grid(i).configuration().getCommunicationSpi()).releaseWaitLatch();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java
index f3a19aa..6824d51 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java
@@ -187,6 +187,8 @@ public class GridJobStealingSelfTest extends GridCommonAbstractTest {
public void testProjectionPredicateInternalStealing() throws Exception {
final Ignite ignite3 = startGrid(3);
+ waitForTopology(3);
+
final UUID node1 = ignite1.cluster().localNode().id();
final UUID node3 = ignite3.cluster().localNode().id();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/GridSameVmStartupSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridSameVmStartupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridSameVmStartupSelfTest.java
index 66e9cf4..a04c38e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridSameVmStartupSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridSameVmStartupSelfTest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal;
import java.util.Collection;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import org.apache.ignite.Ignite;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.events.DiscoveryEvent;
@@ -75,8 +76,10 @@ public class GridSameVmStartupSelfTest extends GridCommonAbstractTest {
ignite2.events().localListen(new IgnitePredicate<Event>() {
@Override public boolean apply(Event evt) {
- assert evt.type() != EVT_NODE_FAILED :
- "Node1 did not exit gracefully.";
+ boolean tcpDiscovery = tcpDiscovery();
+
+ if (tcpDiscovery)
+ assert evt.type() != EVT_NODE_FAILED : "Node1 did not exit gracefully.";
if (evt instanceof DiscoveryEvent) {
// Local node can send METRICS_UPDATED event.
@@ -86,8 +89,14 @@ public class GridSameVmStartupSelfTest extends GridCommonAbstractTest {
((DiscoveryEvent) evt).eventNode().id() + ", expected=" + grid1LocNodeId +
", type=" + evt.type() + ']';
- if (evt.type() == EVT_NODE_LEFT)
- latch.countDown();
+ if (tcpDiscovery) {
+ if (evt.type() == EVT_NODE_LEFT)
+ latch.countDown();
+ }
+ else {
+ if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED)
+ latch.countDown();
+ }
}
return true;
@@ -96,7 +105,7 @@ public class GridSameVmStartupSelfTest extends GridCommonAbstractTest {
stopGrid(1);
- latch.await();
+ assertTrue(latch.await(10, TimeUnit.SECONDS));
Collection<ClusterNode> top2 = ignite2.cluster().forRemotes().nodes();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/GridSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridSelfTest.java
index 7e368cb..f71ffb0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridSelfTest.java
@@ -45,6 +45,8 @@ public class GridSelfTest extends ClusterGroupAbstractTest {
for (int i = 0; i < NODES_CNT; i++)
startGrid(i);
+
+ waitForTopology(NODES_CNT);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
index fa9cc35..e68ea13 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal;
import java.io.IOException;
import java.io.OutputStream;
import java.net.Socket;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -38,6 +39,7 @@ import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteFuture;
@@ -143,6 +145,14 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
/**
* @param ignite Node.
+ * @return Discovery SPI.
+ */
+ protected static IgniteDiscoverySpi spi0(Ignite ignite) {
+ return ((IgniteDiscoverySpi)ignite.configuration().getDiscoverySpi());
+ }
+
+ /**
+ * @param ignite Node.
* @return Communication SPI.
*/
protected BlockTcpCommunicationSpi commSpi(Ignite ignite) {
@@ -185,16 +195,28 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
* @return Server node client connected to.
*/
protected Ignite clientRouter(Ignite client) {
- TcpDiscoveryNode node = (TcpDiscoveryNode)client.cluster().localNode();
+ if (tcpDiscovery()) {
+ TcpDiscoveryNode node = (TcpDiscoveryNode)client.cluster().localNode();
+
+ assertTrue(node.isClient());
+ assertNotNull(node.clientRouterNodeId());
- assertTrue(node.isClient());
- assertNotNull(node.clientRouterNodeId());
+ Ignite srv = G.ignite(node.clientRouterNodeId());
- Ignite srv = G.ignite(node.clientRouterNodeId());
+ assertNotNull(srv);
+
+ return srv;
+ }
+ else {
+ for (Ignite node : G.allGrids()) {
+ if (!node.cluster().localNode().isClient())
+ return node;
+ }
- assertNotNull(srv);
+ fail();
- return srv;
+ return null;
+ }
}
/**
@@ -251,15 +273,24 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
List<Ignite> clients, Ignite srv,
@Nullable Runnable disconnectedC)
throws Exception {
- final TestTcpDiscoverySpi srvSpi = spi(srv);
+ final IgniteDiscoverySpi srvSpi = spi0(srv);
final CountDownLatch disconnectLatch = new CountDownLatch(clients.size());
final CountDownLatch reconnectLatch = new CountDownLatch(clients.size());
log.info("Block reconnect.");
- for (Ignite client : clients)
- spi(client).writeLatch = new CountDownLatch(1);
+ List<DiscoverySpiTestListener> blockLsnrs = new ArrayList<>();
+
+ for (Ignite client : clients) {
+ DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+ lsnr.startBlockJoin();
+
+ blockLsnrs.add(lsnr);
+
+ spi0(client).setInternalListener(lsnr);
+ }
IgnitePredicate<Event> p = new IgnitePredicate<Event>() {
@Override public boolean apply(Event evt) {
@@ -291,8 +322,8 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
log.info("Allow reconnect.");
- for (Ignite client : clients)
- spi(client).writeLatch.countDown();
+ for (DiscoverySpiTestListener blockLsnr : blockLsnrs)
+ blockLsnr.stopBlockJoin();
waitReconnectEvent(log, reconnectLatch);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectApiExceptionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectApiExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectApiExceptionTest.java
index 06bde99..43da2d1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectApiExceptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectApiExceptionTest.java
@@ -44,6 +44,7 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.CollectionConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.util.typedef.C1;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -51,6 +52,7 @@ import org.apache.ignite.lang.IgniteBiPredicate;
import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
import org.apache.ignite.testframework.GridTestUtils;
import static java.util.concurrent.TimeUnit.SECONDS;
@@ -99,7 +101,7 @@ public class IgniteClientReconnectApiExceptionTest extends IgniteClientReconnect
* @throws Exception If failed.
*/
@SuppressWarnings("unchecked")
- public void dataStructureOperationsTest() throws Exception {
+ private void dataStructureOperationsTest() throws Exception {
clientMode = true;
final Ignite client = startGrid(serverCount());
@@ -219,7 +221,7 @@ public class IgniteClientReconnectApiExceptionTest extends IgniteClientReconnect
* @throws Exception If failed.
*/
@SuppressWarnings("unchecked")
- public void cacheOperationsTest() throws Exception {
+ private void cacheOperationsTest() throws Exception {
clientMode = true;
final Ignite client = startGrid(serverCount());
@@ -537,7 +539,7 @@ public class IgniteClientReconnectApiExceptionTest extends IgniteClientReconnect
* @throws Exception If failed.
*/
@SuppressWarnings("unchecked")
- public void igniteOperationsTest() throws Exception {
+ private void igniteOperationsTest() throws Exception {
clientMode = true;
final Ignite client = startGrid(serverCount());
@@ -775,11 +777,11 @@ public class IgniteClientReconnectApiExceptionTest extends IgniteClientReconnect
throws Exception {
assertNotNull(client.cache(DEFAULT_CACHE_NAME));
- final TestTcpDiscoverySpi clientSpi = spi(client);
+ final IgniteDiscoverySpi clientSpi = spi0(client);
Ignite srv = clientRouter(client);
- TestTcpDiscoverySpi srvSpi = spi(srv);
+ DiscoverySpi srvSpi = spi0(srv);
final CountDownLatch disconnectLatch = new CountDownLatch(1);
@@ -787,7 +789,10 @@ public class IgniteClientReconnectApiExceptionTest extends IgniteClientReconnect
log.info("Block reconnect.");
- clientSpi.writeLatch = new CountDownLatch(1);
+ DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+ clientSpi.setInternalListener(lsnr);
+ lsnr.startBlockJoin();
final List<IgniteInternalFuture> futs = new ArrayList<>();
@@ -832,7 +837,7 @@ public class IgniteClientReconnectApiExceptionTest extends IgniteClientReconnect
log.info("Allow reconnect.");
- clientSpi.writeLatch.countDown();
+ lsnr.stopBlockJoin();
waitReconnectEvent(reconnectLatch);
@@ -857,7 +862,7 @@ public class IgniteClientReconnectApiExceptionTest extends IgniteClientReconnect
}
}
finally {
- clientSpi.writeLatch.countDown();
+ lsnr.stopBlockJoin();
for (IgniteInternalFuture fut : futs)
fut.cancel();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAtomicsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAtomicsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAtomicsTest.java
index 00daf5f..d1e3ade 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAtomicsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAtomicsTest.java
@@ -111,7 +111,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
IgniteAtomicSequence clientAtomicSeq = client.atomicSequence("atomicSeq", 0, true);
@@ -144,7 +144,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final IgniteAtomicSequence clientAtomicSeq = client.atomicSequence("atomicSeqRmv", 0, true);
@@ -192,7 +192,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
BlockTcpCommunicationSpi commSpi = commSpi(srv);
@@ -253,7 +253,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
IgniteAtomicReference<String> clientAtomicRef = client.atomicReference("atomicRef", "1st value", true);
@@ -294,7 +294,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final IgniteAtomicReference<String> clientAtomicRef =
client.atomicReference("atomicRefRemoved", "1st value", true);
@@ -347,7 +347,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final IgniteAtomicReference<String> clientAtomicRef =
client.atomicReference("atomicRefInProg", "1st value", true);
@@ -414,7 +414,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
IgniteAtomicStamped clientAtomicStamped = client.atomicStamped("atomicStamped", 0, 0, true);
@@ -455,7 +455,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final IgniteAtomicStamped clientAtomicStamped = client.atomicStamped("atomicStampedRemoved", 0, 0, true);
@@ -506,7 +506,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final IgniteAtomicStamped clientAtomicStamped = client.atomicStamped("atomicStampedInProgress", 0, 0, true);
@@ -574,7 +574,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
IgniteAtomicLong clientAtomicLong = client.atomicLong("atomicLong", 0, true);
@@ -605,7 +605,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final IgniteAtomicLong clientAtomicLong = client.atomicLong("atomicLongRmv", 0, true);
@@ -646,7 +646,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
BlockTcpCommunicationSpi commSpi = commSpi(srv);
@@ -701,7 +701,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
IgniteCountDownLatch clientLatch = client.countDownLatch("latch1", 3, false, true);
@@ -742,7 +742,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
IgniteSemaphore clientSemaphore = client.semaphore("semaphore1", 3, false, true);
@@ -789,7 +789,7 @@ public class IgniteClientReconnectAtomicsTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
IgniteLock clientLock = client.reentrantLock("lock1", true, fair, true);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
index 518e674..3cb82e0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
@@ -49,6 +49,7 @@ import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.managers.communication.GridIoMessage;
import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateResponse;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetResponse;
@@ -67,6 +68,7 @@ import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.resources.LoggerResource;
import org.apache.ignite.spi.IgniteSpiException;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.transactions.Transaction;
@@ -155,11 +157,11 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
IgniteEx client = startGrid(SRV_CNT);
- final TestTcpDiscoverySpi clientSpi = spi(client);
+ final IgniteDiscoverySpi clientSpi = spi0(client);
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
- TestTcpDiscoverySpi srvSpi = spi(srv);
+ DiscoverySpi srvSpi = ignite(0).configuration().getDiscoverySpi();
final IgniteCache<Object, Object> cache = client.getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME));
@@ -188,7 +190,11 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
log.info("Block reconnect.");
- clientSpi.writeLatch = new CountDownLatch(1);
+ DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+ clientSpi.setInternalListener(lsnr);
+
+ lsnr.startBlockJoin();
final AtomicReference<IgniteInternalFuture> blockPutRef = new AtomicReference<>();
@@ -254,7 +260,7 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
log.info("Allow reconnect.");
- clientSpi.writeLatch.countDown();
+ lsnr.stopBlockJoin();
assertTrue(reconnectLatch.await(5000, MILLISECONDS));
@@ -319,7 +325,7 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
IgniteEx client = startGrid(SRV_CNT);
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
@@ -412,17 +418,21 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
final TransactionConcurrency txConcurrency,
final IgniteCache<Object, Object> cache)
throws Exception {
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
- final TestTcpDiscoverySpi clientSpi = spi(client);
- final TestTcpDiscoverySpi srvSpi = spi(srv);
+ final IgniteDiscoverySpi clientSpi = spi0(client);
+ final DiscoverySpi srvSpi = spi0(srv);
final CountDownLatch disconnectLatch = new CountDownLatch(1);
final CountDownLatch reconnectLatch = new CountDownLatch(1);
log.info("Block reconnect.");
- clientSpi.writeLatch = new CountDownLatch(1);
+ DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+ clientSpi.setInternalListener(lsnr);
+
+ lsnr.startBlockJoin();
client.events().localListen(new IgnitePredicate<Event>() {
@Override public boolean apply(Event evt) {
@@ -530,7 +540,7 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
assertTrue(putFailed.await(5000, MILLISECONDS));
- clientSpi.writeLatch.countDown();
+ lsnr.stopBlockJoin();
waitReconnectEvent(reconnectLatch);
@@ -604,9 +614,9 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
IgniteEx client = startGrid(SRV_CNT);
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
- TestTcpDiscoverySpi srvSpi = spi(srv);
+ DiscoverySpi srvSpi = spi0(srv);
TestCommunicationSpi coordCommSpi = (TestCommunicationSpi)grid(0).configuration().getCommunicationSpi();
@@ -691,7 +701,7 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
IgniteInternalFuture<Boolean> fut = GridTestUtils.runAsync(new Callable<Boolean>() {
@Override public Boolean call() throws Exception {
try {
- Ignition.start(optimize(getConfiguration(getTestIgniteInstanceName(SRV_CNT))));
+ startGrid(optimize(getConfiguration(getTestIgniteInstanceName(SRV_CNT))));
// Commented due to IGNITE-4473, because
// IgniteClientDisconnectedException won't
@@ -722,7 +732,7 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
}
});
- TestTcpDiscoverySpi srvSpi = spi(srv);
+ DiscoverySpi srvSpi = spi0(srv);
try {
if (!joinLatch.await(10_000, MILLISECONDS)) {
@@ -1256,30 +1266,35 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
*
*/
static class TestClass1 implements Serializable {
+ // No-op.
}
/**
*
*/
static class TestClass2 implements Serializable {
+ // No-op.
}
/**
*
*/
static class TestClass3 implements Serializable {
+ // No-op.
}
/**
*
*/
static class TestClass4 implements Serializable {
+ // No-op.
}
/**
*
*/
static class TestClass5 implements Serializable {
+ // No-op.
}
/**
@@ -1294,11 +1309,11 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
Class<?> msgToBlock,
final IgniteInClosure<IgniteCache<Object, Object>> c)
throws Exception {
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final UUID id = client.localNode().id();
- TestTcpDiscoverySpi srvSpi = spi(srv);
+ DiscoverySpi srvSpi = spi0(srv);
final IgniteCache<Object, Object> cache = client.getOrCreateCache(ccfg);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCollectionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCollectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCollectionsTest.java
index 3f0e33d..5be59b0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCollectionsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCollectionsTest.java
@@ -180,7 +180,7 @@ public class IgniteClientReconnectCollectionsTest extends IgniteClientReconnectA
private void serverNodeReconnect(CollectionConfiguration colCfg) throws Exception {
final Ignite client = grid(serverCount());
- final Ignite srv = clientRouter(client);
+ final Ignite srv = ignite(0);
assertNotNull(srv.queue("q", 0, colCfg));
assertNotNull(srv.set("s", colCfg));
@@ -201,7 +201,7 @@ public class IgniteClientReconnectCollectionsTest extends IgniteClientReconnectA
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final String setName = "set-" + colCfg.getAtomicityMode();
@@ -235,7 +235,7 @@ public class IgniteClientReconnectCollectionsTest extends IgniteClientReconnectA
assertTrue(client.cluster().localNode().isClient());
- final Ignite srv = clientRouter(client);
+ final Ignite srv = ignite(0);
final String setName = "set-rm-" + colCfg.getAtomicityMode();
@@ -281,7 +281,7 @@ public class IgniteClientReconnectCollectionsTest extends IgniteClientReconnectA
assertTrue(client.cluster().localNode().isClient());
- final Ignite srv = clientRouter(client);
+ final Ignite srv = ignite(0);
final String setName = "set-in-progress-" + colCfg.getAtomicityMode();
@@ -347,7 +347,7 @@ public class IgniteClientReconnectCollectionsTest extends IgniteClientReconnectA
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final String setName = "queue-" + colCfg.getAtomicityMode();
@@ -379,7 +379,7 @@ public class IgniteClientReconnectCollectionsTest extends IgniteClientReconnectA
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final String setName = "queue-rmv" + colCfg.getAtomicityMode();
@@ -423,7 +423,7 @@ public class IgniteClientReconnectCollectionsTest extends IgniteClientReconnectA
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
final String setName = "queue-rmv" + colCfg.getAtomicityMode();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectComputeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectComputeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectComputeTest.java
index cce0c7e..57d3188 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectComputeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectComputeTest.java
@@ -49,7 +49,7 @@ public class IgniteClientReconnectComputeTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
IgniteCache<Integer, Integer> cache = client.getOrCreateCache("test-cache");
@@ -103,7 +103,7 @@ public class IgniteClientReconnectComputeTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
BlockTcpCommunicationSpi commSpi = commSpi(srv);
@@ -152,7 +152,7 @@ public class IgniteClientReconnectComputeTest extends IgniteClientReconnectAbstr
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
BlockTcpCommunicationSpi commSpi = commSpi(srv);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
index ca0d889..d68fc1c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
@@ -28,6 +28,7 @@ import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.events.Event;
import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.util.typedef.P2;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.lang.IgniteRunnable;
@@ -61,9 +62,9 @@ public class IgniteClientReconnectContinuousProcessorTest extends IgniteClientRe
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
- TestTcpDiscoverySpi srvSpi = spi(srv);
+ IgniteDiscoverySpi srvSpi = spi0(srv);
EventListener lsnr = new EventListener();
@@ -133,9 +134,9 @@ public class IgniteClientReconnectContinuousProcessorTest extends IgniteClientRe
assertTrue(client.cluster().localNode().isClient());
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
- TestTcpDiscoverySpi srvSpi = spi(srv);
+ IgniteDiscoverySpi srvSpi = spi0(srv);
final String topic = "testTopic";
@@ -309,9 +310,9 @@ public class IgniteClientReconnectContinuousProcessorTest extends IgniteClientRe
CacheEventListener lsnr)
throws Exception
{
- Ignite srv = clientRouter(client);
+ Ignite srv = ignite(0);
- TestTcpDiscoverySpi srvSpi = spi(srv);
+ IgniteDiscoverySpi srvSpi = spi0(srv);
final CountDownLatch reconnectLatch = new CountDownLatch(1);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDiscoveryStateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDiscoveryStateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDiscoveryStateTest.java
index c071ee2..6e77742 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDiscoveryStateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDiscoveryStateTest.java
@@ -27,6 +27,7 @@ import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.events.Event;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED;
@@ -64,20 +65,23 @@ public class IgniteClientReconnectDiscoveryStateTest extends IgniteClientReconne
nodeCnt.put(1, 1);
nodeCnt.put(2, 2);
nodeCnt.put(3, 3);
- nodeCnt.put(4, 4);
- for (Map.Entry<Integer, Integer> e : nodeCnt.entrySet()) {
- Collection<ClusterNode> nodes = cluster.topology(e.getKey());
+ if (tcpDiscovery()) {
+ nodeCnt.put(4, 4);
- assertNotNull("No nodes for topology: " + e.getKey(), nodes);
- assertEquals((int)e.getValue(), nodes.size());
+ for (Map.Entry<Integer, Integer> e : nodeCnt.entrySet()) {
+ Collection<ClusterNode> nodes = cluster.topology(e.getKey());
+
+ assertNotNull("No nodes for topology: " + e.getKey(), nodes);
+ assertEquals((int)e.getValue(), nodes.size());
+ }
}
ClusterNode locNode = cluster.localNode();
assertEquals(topVer, locNode.order());
- TestTcpDiscoverySpi srvSpi = spi(clientRouter(client));
+ DiscoverySpi srvSpi = ignite(0).configuration().getDiscoverySpi();
final CountDownLatch reconnectLatch = new CountDownLatch(1);
@@ -112,7 +116,11 @@ public class IgniteClientReconnectDiscoveryStateTest extends IgniteClientReconne
assertEquals(topVer, locNode.order());
assertEquals(topVer, cluster.topologyVersion());
- nodeCnt.put(5, 3);
+ if (tcpDiscovery())
+ nodeCnt.put(5, 3);
+ else
+ nodeCnt.clear();
+
nodeCnt.put(6, 4);
for (Map.Entry<Integer, Integer> e : nodeCnt.entrySet()) {
[21/54] [abbrv] ignite git commit: ignite-7772 System workers
critical failures handling
Posted by ag...@apache.org.
ignite-7772 System workers critical failures handling
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c807ae95
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c807ae95
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c807ae95
Branch: refs/heads/ignite-6083
Commit: c807ae952c233cf1a8c0a63d543fafe19c40c6aa
Parents: 05d7092
Author: Andrey Kuznetsov <st...@gmail.com>
Authored: Tue Apr 10 17:30:12 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Apr 10 17:30:12 2018 +0300
----------------------------------------------------------------------
.../org/apache/ignite/internal/IgnitionEx.java | 15 +-
.../GridClientConnectionManagerAdapter.java | 6 +
.../impl/GridTcpRouterNioListenerAdapter.java | 6 +
.../discovery/GridDiscoveryManager.java | 16 +-
.../GridCachePartitionExchangeManager.java | 12 +-
.../cache/GridCacheSharedTtlCleanupManager.java | 41 +++--
.../GridCacheDatabaseSharedManager.java | 60 +++++--
.../wal/FileWriteAheadLogManager.java | 157 ++++++++++++-------
.../wal/FsyncModeFileWriteAheadLogManager.java | 34 +++-
.../timeout/GridTimeoutProcessor.java | 102 +++++++-----
.../ignite/internal/util/StripedExecutor.java | 69 +++++---
.../ignite/internal/util/nio/GridNioServer.java | 43 ++++-
.../util/nio/GridNioServerListener.java | 6 +
.../util/nio/GridNioServerListenerAdapter.java | 6 +
.../communication/tcp/TcpCommunicationSpi.java | 41 ++++-
.../ignite/spi/discovery/tcp/ServerImpl.java | 51 +++++-
.../internal/util/StripedExecutorTest.java | 2 +-
17 files changed, 501 insertions(+), 166 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 417ba1e..10a0752 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -137,6 +137,7 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.configuration.IgniteConfiguration.DFLT_THREAD_KEEP_ALIVE_TIME;
import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEMORY_POLICY_MAX_SIZE;
import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEM_PLC_DEFAULT_NAME;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.IgniteComponentType.SPRING;
import static org.apache.ignite.plugin.segmentation.SegmentationPolicy.RESTART_JVM;
@@ -1806,7 +1807,13 @@ public class IgnitionEx {
cfg.getStripedPoolSize(),
cfg.getIgniteInstanceName(),
"sys",
- log);
+ log,
+ new Thread.UncaughtExceptionHandler() {
+ @Override public void uncaughtException(Thread thread, Throwable t) {
+ if (grid != null)
+ grid.context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, t));
+ }
+ });
// Note that since we use 'LinkedBlockingQueue', number of
// maximum threads has no effect.
@@ -1846,6 +1853,12 @@ public class IgnitionEx {
cfg.getIgniteInstanceName(),
"data-streamer",
log,
+ new Thread.UncaughtExceptionHandler() {
+ @Override public void uncaughtException(Thread thread, Throwable t) {
+ if (grid != null)
+ grid.context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, t));
+ }
+ },
true);
// Note that we do not pre-start threads here as igfs pool may not be needed.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
index 829b188..fe0453f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
@@ -38,6 +38,7 @@ import java.util.logging.Logger;
import javax.net.ssl.SSLContext;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.client.GridClientClosedException;
import org.apache.ignite.internal.client.GridClientConfiguration;
import org.apache.ignite.internal.client.GridClientException;
@@ -656,6 +657,11 @@ public abstract class GridClientConnectionManagerAdapter implements GridClientCo
}
}
+ /** {@inheritDoc} */
+ @Override public void onFailure(FailureType failureType, Throwable failure) {
+ // No-op.
+ }
+
/**
* Handles client handshake response.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
index 22f5152..75aa6f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.client.GridClientException;
import org.apache.ignite.internal.client.GridClientFuture;
import org.apache.ignite.internal.client.GridClientFutureListener;
@@ -191,6 +192,11 @@ public abstract class GridTcpRouterNioListenerAdapter implements GridNioServerLi
}
/** {@inheritDoc} */
+ @Override public void onFailure(FailureType failureType, Throwable failure) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Override public void onSessionWriteTimeout(GridNioSession ses) {
U.warn(log, "Closing NIO session because of write timeout.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 4c5690e..b0d3256 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -147,6 +147,8 @@ import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DATA_REGIONS_OFFHEAP_SIZE;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
@@ -2669,13 +2671,21 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
body0();
}
catch (InterruptedException e) {
+ if (!isCancelled)
+ ctx.failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, e));
+
throw e;
}
catch (Throwable t) {
- U.error(log, "Unexpected exception in discovery worker thread (ignored).", t);
+ U.error(log, "Exception in discovery worker thread.", t);
+
+ if (t instanceof Error) {
+ FailureType type = t instanceof OutOfMemoryError ? CRITICAL_ERROR : SYSTEM_WORKER_TERMINATION;
- if (t instanceof Error)
- throw (Error)t;
+ ctx.failure().process(new FailureContext(type, t));
+
+ throw t;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 77ffce3..e40493f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -123,6 +123,7 @@ import static org.apache.ignite.IgniteSystemProperties.getLong;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE;
import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
@@ -2274,11 +2275,20 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
try {
body0();
}
+ catch (InterruptedException | IgniteInterruptedCheckedException e) {
+ if (!stop)
+ err = e;
+ }
catch (Throwable e) {
err = e;
}
finally {
- if (!stop)
+ if (err == null && !stop)
+ err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
+
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
index 8f3d738..613e93b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
@@ -20,11 +20,15 @@ package org.apache.ignite.internal.processors.cache;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.util.worker.GridWorker;
import org.apache.ignite.thread.IgniteThread;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
+
/**
* Periodically removes expired entities from caches with {@link CacheConfiguration#isEagerTtl()} flag set.
*/
@@ -122,19 +126,38 @@ public class GridCacheSharedTtlCleanupManager extends GridCacheSharedManagerAdap
/** {@inheritDoc} */
@Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
- while (!isCancelled()) {
- boolean expiredRemains = false;
+ Throwable err = null;
+
+ try {
+ while (!isCancelled()) {
+ boolean expiredRemains = false;
+
+ for (GridCacheTtlManager mgr : mgrs) {
+ if (mgr.expire(CLEANUP_WORKER_ENTRIES_PROCESS_LIMIT))
+ expiredRemains = true;
- for (GridCacheTtlManager mgr : mgrs) {
- if (mgr.expire(CLEANUP_WORKER_ENTRIES_PROCESS_LIMIT))
- expiredRemains = true;
+ if (isCancelled())
+ return;
+ }
- if (isCancelled())
- return;
+ if (!expiredRemains)
+ U.sleep(CLEANUP_WORKER_SLEEP_INTERVAL);
}
+ }
+ catch (Throwable t) {
+ if (!(t instanceof IgniteInterruptedCheckedException))
+ err = t;
- if (!expiredRemains)
- U.sleep(CLEANUP_WORKER_SLEEP_INTERVAL);
+ throw t;
+ }
+ finally {
+ if (err == null && !isCancelled)
+ err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
+
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 70fc688..caf27b7 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -165,6 +165,8 @@ import static java.nio.file.StandardOpenOption.READ;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID;
/**
@@ -2787,32 +2789,58 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/** {@inheritDoc} */
- @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
- while (!isCancelled()) {
- waitCheckpointEvent();
+ @Override protected void body() {
+ Throwable err = null;
- GridFutureAdapter<Void> enableChangeApplied = GridCacheDatabaseSharedManager.this.enableChangeApplied;
+ try {
+ while (!isCancelled()) {
+ waitCheckpointEvent();
- if (enableChangeApplied != null) {
- enableChangeApplied.onDone();
+ GridFutureAdapter<Void> enableChangeApplied = GridCacheDatabaseSharedManager.this.enableChangeApplied;
- GridCacheDatabaseSharedManager.this.enableChangeApplied = null;
- }
+ if (enableChangeApplied != null) {
+ enableChangeApplied.onDone();
- if (checkpointsEnabled)
- doCheckpoint();
- else {
- synchronized (this) {
- scheduledCp.nextCpTs = U.currentTimeMillis() + checkpointFreq;
+ GridCacheDatabaseSharedManager.this.enableChangeApplied = null;
+ }
+
+ if (checkpointsEnabled)
+ doCheckpoint();
+ else {
+ synchronized (this) {
+ scheduledCp.nextCpTs = U.currentTimeMillis() + checkpointFreq;
+ }
}
}
}
+ catch (Throwable t) {
+ err = t;
+
+ scheduledCp.cpFinishFut.onDone(t);
+
+ throw t;
+ }
+ finally {
+ if (err == null && !(stopping && isCancelled))
+ err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
+
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
// Final run after the cancellation.
- if (checkpointsEnabled && !shutdownNow)
- doCheckpoint();
+ if (checkpointsEnabled && !shutdownNow) {
+ try {
+ doCheckpoint();
- scheduledCp.cpFinishFut.onDone(new NodeStoppingException("Node is stopping."));
+ scheduledCp.cpFinishFut.onDone(new NodeStoppingException("Node is stopping."));
+ }
+ catch (Throwable e) {
+ scheduledCp.cpFinishFut.onDone(e);
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 2fff481..a40811b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -117,6 +117,8 @@ import static java.nio.file.StandardOpenOption.WRITE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.SWITCH_SEGMENT_RECORD;
import static org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRingByteBuffer.BufferMode.DIRECT;
import static org.apache.ignite.internal.util.IgniteUtils.findField;
@@ -682,7 +684,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (IgniteCheckedException e) {
U.error(log, "Unable to perform segment rollover: " + e.getMessage(), e);
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
}
}
@@ -1234,7 +1236,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (IOException e) {
StorageException se = new StorageException("Unable to initialize WAL segment", e);
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, se));
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
throw se;
}
@@ -1499,6 +1501,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
}
+ Throwable err = null;
+
try {
synchronized (this) {
while (curAbsWalIdx == -1 && !stopped)
@@ -1560,6 +1564,18 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
+ catch (Throwable t) {
+ err = t;
+ }
+ finally {
+ if (err == null && !stopped)
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
+
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
}
/**
@@ -1884,8 +1900,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
catch (IgniteCheckedException | IOException e) {
U.error(log, "Unexpected error during WAL compression", e);
-
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
}
catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
@@ -2005,6 +2019,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/** {@inheritDoc} */
@Override public void run() {
+ Throwable err = null;
+
while (!Thread.currentThread().isInterrupted() && !stopped) {
try {
long segmentToDecompress = segmentsQueue.take();
@@ -2034,10 +2050,17 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
- catch (IOException e) {
- U.error(log, "Unexpected error during WAL decompression", e);
+ catch (Throwable t) {
+ err = t;
+ }
+ finally {
+ if (err == null && !stopped)
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
}
}
}
@@ -3146,78 +3169,94 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/** {@inheritDoc} */
@Override public void run() {
- while (!shutdown && !Thread.currentThread().isInterrupted()) {
- while (waiters.isEmpty()) {
- if (!shutdown)
- LockSupport.park();
- else {
- unparkWaiters(Long.MAX_VALUE);
-
- return;
- }
- }
+ Throwable err = null;
- Long pos = null;
+ try {
+ while (!shutdown && !Thread.currentThread().isInterrupted()) {
+ while (waiters.isEmpty()) {
+ if (!shutdown)
+ LockSupport.park();
+ else {
+ unparkWaiters(Long.MAX_VALUE);
- for (Long val : waiters.values()) {
- if (val > Long.MIN_VALUE)
- pos = val;
- }
+ return;
+ }
+ }
- if (pos == null)
- continue;
- else if (pos < UNCONDITIONAL_FLUSH) {
- try {
- assert pos == FILE_CLOSE || pos == FILE_FORCE : pos;
+ Long pos = null;
- if (pos == FILE_CLOSE)
- currHnd.fileIO.close();
- else if (pos == FILE_FORCE)
- currHnd.fileIO.force();
+ for (Long val : waiters.values()) {
+ if (val > Long.MIN_VALUE)
+ pos = val;
}
- catch (IOException e) {
- log.error("Exception in WAL writer thread: ", e);
- err = e;
+ if (pos == null)
+ continue;
+ else if (pos < UNCONDITIONAL_FLUSH) {
+ try {
+ assert pos == FILE_CLOSE || pos == FILE_FORCE : pos;
- unparkWaiters(Long.MAX_VALUE);
+ if (pos == FILE_CLOSE)
+ currHnd.fileIO.close();
+ else if (pos == FILE_FORCE)
+ currHnd.fileIO.force();
+ }
+ catch (IOException e) {
+ log.error("Exception in WAL writer thread: ", e);
- return;
- }
+ err = e;
- unparkWaiters(pos);
- }
+ unparkWaiters(Long.MAX_VALUE);
- List<SegmentedRingByteBuffer.ReadSegment> segs = currentHandle().buf.poll(pos);
+ return;
+ }
- if (segs == null) {
- unparkWaiters(pos);
+ unparkWaiters(pos);
+ }
- continue;
- }
+ List<SegmentedRingByteBuffer.ReadSegment> segs = currentHandle().buf.poll(pos);
- for (int i = 0; i < segs.size(); i++) {
- SegmentedRingByteBuffer.ReadSegment seg = segs.get(i);
+ if (segs == null) {
+ unparkWaiters(pos);
- try {
- writeBuffer(seg.position(), seg.buffer());
+ continue;
}
- catch (Throwable e) {
- log.error("Exception in WAL writer thread: ", e);
- err = e;
- }
- finally {
- seg.release();
+ for (int i = 0; i < segs.size(); i++) {
+ SegmentedRingByteBuffer.ReadSegment seg = segs.get(i);
+
+ try {
+ writeBuffer(seg.position(), seg.buffer());
+ }
+ catch (Throwable e) {
+ log.error("Exception in WAL writer thread: ", e);
+
+ err = e;
+ }
+ finally {
+ seg.release();
- long p = pos <= UNCONDITIONAL_FLUSH || err != null ? Long.MAX_VALUE : currentHandle().written;
+ long p = pos <= UNCONDITIONAL_FLUSH || err != null ? Long.MAX_VALUE : currentHandle().written;
- unparkWaiters(p);
+ unparkWaiters(p);
+ }
}
}
+
+ unparkWaiters(Long.MAX_VALUE);
}
+ catch (Throwable t) {
+ err = t;
+ }
+ finally {
+ if (err == null && !shutdown)
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
- unparkWaiters(Long.MAX_VALUE);
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
}
/**
@@ -3283,7 +3322,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
Throwable err = walWriter.err;
if (err != null)
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, err));
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
if (expPos == UNCONDITIONAL_FLUSH)
expPos = (currentHandle().buf.tail());
@@ -3372,7 +3411,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (IOException e) {
StorageException se = new StorageException("Unable to write", e);
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, se));
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
throw se;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 59196bb..c7d2c11 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -110,6 +110,7 @@ import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
/**
* File WAL manager.
@@ -1338,6 +1339,8 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
}
}
+ Throwable err = null;
+
try {
synchronized (this) {
while (curAbsWalIdx == -1 && !stopped)
@@ -1399,6 +1402,18 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
+ catch (Throwable t) {
+ err = t;
+ }
+ finally {
+ if (err == null && !stopped)
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
+
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
}
/**
@@ -1721,8 +1736,6 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
}
catch (IgniteCheckedException | IOException e) {
U.error(log, "Unexpected error during WAL compression", e);
-
- cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
@@ -1814,6 +1827,8 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
/** {@inheritDoc} */
@Override public void run() {
+ Throwable err = null;
+
while (!Thread.currentThread().isInterrupted() && !stopped) {
try {
long segmentToDecompress = segmentsQueue.take();
@@ -1840,13 +1855,20 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
decompressionFutures.remove(segmentToDecompress).onDone();
}
}
- catch (InterruptedException e){
+ catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
- catch (IOException e) {
- U.error(log, "Unexpected error during WAL decompression", e);
+ catch (Throwable t) {
+ err = t;
+ }
+ finally {
+ if (err == null && !stopped)
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
- cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
+ if (err instanceof OutOfMemoryError)
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
index ff6beb4..a09d6fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
@@ -21,6 +21,7 @@ import java.io.Closeable;
import java.util.Comparator;
import java.util.Iterator;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.processors.GridProcessorAdapter;
import org.apache.ignite.internal.util.GridConcurrentSkipListSet;
@@ -32,6 +33,9 @@ import org.apache.ignite.internal.util.worker.GridWorker;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.thread.IgniteThread;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
+
/**
* Detects timeout events and processes them.
*/
@@ -146,61 +150,81 @@ public class GridTimeoutProcessor extends GridProcessorAdapter {
/** {@inheritDoc} */
@Override protected void body() throws InterruptedException {
- while (!isCancelled()) {
- long now = U.currentTimeMillis();
+ Throwable err = null;
- for (Iterator<GridTimeoutObject> iter = timeoutObjs.iterator(); iter.hasNext();) {
- GridTimeoutObject timeoutObj = iter.next();
+ try {
+ while (!isCancelled()) {
+ long now = U.currentTimeMillis();
- if (timeoutObj.endTime() <= now) {
- try {
- boolean rmvd = timeoutObjs.remove(timeoutObj);
+ for (Iterator<GridTimeoutObject> iter = timeoutObjs.iterator(); iter.hasNext(); ) {
+ GridTimeoutObject timeoutObj = iter.next();
- if (log.isDebugEnabled())
- log.debug("Timeout has occurred [obj=" + timeoutObj + ", process=" + rmvd + ']');
+ if (timeoutObj.endTime() <= now) {
+ try {
+ boolean rmvd = timeoutObjs.remove(timeoutObj);
- if (rmvd)
- timeoutObj.onTimeout();
- }
- catch (Throwable e) {
- if (isCancelled() && !(e instanceof Error)){
if (log.isDebugEnabled())
- log.debug("Error when executing timeout callback: " + timeoutObj);
+ log.debug("Timeout has occurred [obj=" + timeoutObj + ", process=" + rmvd + ']');
- return;
+ if (rmvd)
+ timeoutObj.onTimeout();
}
+ catch (Throwable e) {
+ if (isCancelled() && !(e instanceof Error)) {
+ if (log.isDebugEnabled())
+ log.debug("Error when executing timeout callback: " + timeoutObj);
- U.error(log, "Error when executing timeout callback: " + timeoutObj, e);
+ return;
+ }
- if (e instanceof Error)
- throw e;
+ U.error(log, "Error when executing timeout callback: " + timeoutObj, e);
+
+ if (e instanceof Error)
+ throw e;
+ }
}
+ else
+ break;
}
- else
- break;
- }
-
- synchronized (mux) {
- while (!isCancelled()) {
- // Access of the first element must be inside of
- // synchronization block, so we don't miss out
- // on thread notification events sent from
- // 'addTimeoutObject(..)' method.
- GridTimeoutObject first = timeoutObjs.firstx();
-
- if (first != null) {
- long waitTime = first.endTime() - U.currentTimeMillis();
- if (waitTime > 0)
- mux.wait(waitTime);
+ synchronized (mux) {
+ while (!isCancelled()) {
+ // Access of the first element must be inside of
+ // synchronization block, so we don't miss out
+ // on thread notification events sent from
+ // 'addTimeoutObject(..)' method.
+ GridTimeoutObject first = timeoutObjs.firstx();
+
+ if (first != null) {
+ long waitTime = first.endTime() - U.currentTimeMillis();
+
+ if (waitTime > 0)
+ mux.wait(waitTime);
+ else
+ break;
+ }
else
- break;
+ mux.wait(5000);
}
- else
- mux.wait(5000);
}
}
}
+ catch (Throwable t) {
+ if (!(t instanceof InterruptedException))
+ err = t;
+
+ throw t;
+ }
+ finally {
+ if (err == null && !isCancelled)
+ err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly.");
+
+ if (err instanceof OutOfMemoryError)
+ ctx.failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ ctx.failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
+
}
}
@@ -284,4 +308,4 @@ public class GridTimeoutProcessor extends GridProcessorAdapter {
return S.toString(CancelableTask.class, this);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
index 630d34c..c6383ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
@@ -64,9 +64,11 @@ public class StripedExecutor implements ExecutorService {
* @param igniteInstanceName Node name.
* @param poolName Pool name.
* @param log Logger.
+ * @param errHnd Exception handler.
*/
- public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log) {
- this(cnt, igniteInstanceName, poolName, log, false);
+ public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log,
+ Thread.UncaughtExceptionHandler errHnd) {
+ this(cnt, igniteInstanceName, poolName, log, errHnd, false);
}
/**
@@ -74,9 +76,11 @@ public class StripedExecutor implements ExecutorService {
* @param igniteInstanceName Node name.
* @param poolName Pool name.
* @param log Logger.
+ * @param errHnd Exception handler.
* @param stealTasks {@code True} to steal tasks.
*/
- public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log, boolean stealTasks) {
+ public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log,
+ Thread.UncaughtExceptionHandler errHnd, boolean stealTasks) {
A.ensure(cnt > 0, "cnt > 0");
boolean success = false;
@@ -91,15 +95,9 @@ public class StripedExecutor implements ExecutorService {
try {
for (int i = 0; i < cnt; i++) {
- stripes[i] = stealTasks ? new StripeConcurrentQueue(
- igniteInstanceName,
- poolName,
- i,
- log, stripes) : new StripeConcurrentQueue(
- igniteInstanceName,
- poolName,
- i,
- log);
+ stripes[i] = stealTasks
+ ? new StripeConcurrentQueue(igniteInstanceName, poolName, i, log, stripes, errHnd)
+ : new StripeConcurrentQueue(igniteInstanceName, poolName, i, log, errHnd);
}
for (int i = 0; i < cnt; i++)
@@ -434,22 +432,28 @@ public class StripedExecutor implements ExecutorService {
/** Thread executing the loop. */
protected Thread thread;
+ /** Exception handler. */
+ private Thread.UncaughtExceptionHandler errHnd;
+
/**
* @param igniteInstanceName Ignite instance name.
* @param poolName Pool name.
* @param idx Stripe index.
* @param log Logger.
+ * @param errHnd Exception handler.
*/
public Stripe(
String igniteInstanceName,
String poolName,
int idx,
- IgniteLogger log
+ IgniteLogger log,
+ Thread.UncaughtExceptionHandler errHnd
) {
this.igniteInstanceName = igniteInstanceName;
this.poolName = poolName;
this.idx = idx;
this.log = log;
+ this.errHnd = errHnd;
}
/**
@@ -463,6 +467,8 @@ public class StripedExecutor implements ExecutorService {
idx,
GridIoPolicy.UNDEFINED);
+ thread.setUncaughtExceptionHandler(errHnd);
+
thread.start();
}
@@ -518,9 +524,19 @@ public class StripedExecutor implements ExecutorService {
return;
}
catch (Throwable e) {
+ if (e instanceof OutOfMemoryError) {
+ // Re-throwing to exploit uncaught exception handler.
+ throw e;
+ }
+
U.error(log, "Failed to execute runnable.", e);
}
}
+
+ if (!stopping) {
+ throw new IllegalStateException("Thread " + Thread.currentThread().getName() +
+ " is terminated unexpectedly");
+ }
}
/**
@@ -576,14 +592,16 @@ public class StripedExecutor implements ExecutorService {
* @param poolName Pool name.
* @param idx Stripe index.
* @param log Logger.
+ * @param errHnd Exception handler.
*/
StripeConcurrentQueue(
String igniteInstanceName,
String poolName,
int idx,
- IgniteLogger log
+ IgniteLogger log,
+ Thread.UncaughtExceptionHandler errHnd
) {
- this(igniteInstanceName, poolName, idx, log, null);
+ this(igniteInstanceName, poolName, idx, log, null, errHnd);
}
/**
@@ -591,19 +609,22 @@ public class StripedExecutor implements ExecutorService {
* @param poolName Pool name.
* @param idx Stripe index.
* @param log Logger.
+ * @param errHnd Exception handler.
*/
StripeConcurrentQueue(
String igniteInstanceName,
String poolName,
int idx,
IgniteLogger log,
- Stripe[] others
+ Stripe[] others,
+ Thread.UncaughtExceptionHandler errHnd
) {
super(
igniteInstanceName,
poolName,
idx,
- log);
+ log,
+ errHnd);
this.others = others;
@@ -702,17 +723,20 @@ public class StripedExecutor implements ExecutorService {
* @param poolName Pool name.
* @param idx Stripe index.
* @param log Logger.
+ * @param errHnd Exception handler.
*/
public StripeConcurrentQueueNoPark(
String igniteInstanceName,
String poolName,
int idx,
- IgniteLogger log
+ IgniteLogger log,
+ Thread.UncaughtExceptionHandler errHnd
) {
super(igniteInstanceName,
poolName,
idx,
- log);
+ log,
+ errHnd);
}
/** {@inheritDoc} */
@@ -758,17 +782,20 @@ public class StripedExecutor implements ExecutorService {
* @param poolName Pool name.
* @param idx Stripe index.
* @param log Logger.
+ * @param errHnd Exception handler.
*/
public StripeConcurrentBlockingQueue(
String igniteInstanceName,
String poolName,
int idx,
- IgniteLogger log
+ IgniteLogger log,
+ Thread.UncaughtExceptionHandler errHnd
) {
super(igniteInstanceName,
poolName,
idx,
- log);
+ log,
+ errHnd);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index 0fcde0e..3597a05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -77,6 +77,8 @@ import org.apache.ignite.plugin.extensions.communication.MessageWriter;
import org.apache.ignite.thread.IgniteThread;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.MSG_WRITER;
import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.NIO_OPERATION;
@@ -1749,6 +1751,8 @@ public class GridNioServer<T> {
/** {@inheritDoc} */
@Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+ Throwable err = null;
+
try {
boolean reset = false;
@@ -1774,9 +1778,24 @@ public class GridNioServer<T> {
catch (Throwable e) {
U.error(log, "Caught unhandled exception in NIO worker thread (restart the node).", e);
+ err = e;
+
if (e instanceof Error)
throw e;
}
+ finally {
+ if (err instanceof OutOfMemoryError)
+ lsnr.onFailure(CRITICAL_ERROR, err);
+ else if (!closed) {
+ if (err == null)
+ lsnr.onFailure(SYSTEM_WORKER_TERMINATION,
+ new IllegalStateException("Thread " + name() + " is terminated unexpectedly"));
+ else if (err instanceof InterruptedException)
+ lsnr.onFailure(SYSTEM_WORKER_TERMINATION, err);
+ }
+ else if (err != null)
+ lsnr.onFailure(SYSTEM_WORKER_TERMINATION, err);
+ }
}
/**
@@ -2790,6 +2809,8 @@ public class GridNioServer<T> {
/** {@inheritDoc} */
@Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+ Throwable err = null;
+
try {
boolean reset = false;
@@ -2812,8 +2833,28 @@ public class GridNioServer<T> {
}
}
}
+ catch (Throwable t) {
+ if (!(t instanceof IgniteInterruptedCheckedException))
+ err = t;
+
+ throw t;
+ }
finally {
- closeSelector(); // Safety.
+ try {
+ closeSelector(); // Safety.
+ }
+ catch (RuntimeException ignore) {
+ // No-op.
+ }
+
+ if (err == null && !closed)
+ err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
+
+ if (err instanceof OutOfMemoryError)
+ lsnr.onFailure(CRITICAL_ERROR, err);
+ else if (err != null)
+ lsnr.onFailure(SYSTEM_WORKER_TERMINATION, err);
+
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java
index db28792..14c5a74 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.util.nio;
+import org.apache.ignite.failure.FailureType;
import org.jetbrains.annotations.Nullable;
/**
@@ -69,4 +70,9 @@ public interface GridNioServerListener<T> {
* @param ses Session that is idle.
*/
public void onSessionIdleTimeout(GridNioSession ses);
+
+ /**
+ * Called when critical failure occurs in server implementation.
+ */
+ public void onFailure(FailureType failureType, Throwable failure);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java
index 5d222c1..b6b20b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java
@@ -17,6 +17,8 @@
package org.apache.ignite.internal.util.nio;
+import org.apache.ignite.failure.FailureType;
+
/**
* Server listener adapter providing empty methods implementation for rarely used methods.
*/
@@ -35,4 +37,8 @@ public abstract class GridNioServerListenerAdapter<T> implements GridNioServerLi
@Override public void onMessageSent(GridNioSession ses, T msg) {
// No-op.
}
+
+ @Override public void onFailure(FailureType failureType, Throwable failure) {
+ // No-op.
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 4a0710e..9e7b592 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -62,7 +62,10 @@ import org.apache.ignite.configuration.AddressResolver;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.IgniteKernal;
@@ -151,6 +154,8 @@ import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META;
import static org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture.SES_FUT_META;
import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.ALREADY_CONNECTED;
@@ -798,6 +803,11 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
}
+ /** {@inheritDoc} */
+ @Override public void onFailure(FailureType failureType, Throwable failure) {
+ ((IgniteEx)ignite).context().failure().process(new FailureContext(failureType, failure));
+ }
+
/**
* @param recovery Recovery descriptor.
* @param ses Session.
@@ -4190,13 +4200,32 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
if (log.isDebugEnabled())
log.debug("Tcp communication worker has been started.");
- while (!isInterrupted()) {
- DisconnectedSessionInfo disconnectData = q.poll(idleConnTimeout, TimeUnit.MILLISECONDS);
+ Throwable err = null;
- if (disconnectData != null)
- processDisconnect(disconnectData);
- else
- processIdle();
+ try {
+ while (!isInterrupted()) {
+ DisconnectedSessionInfo disconnectData = q.poll(idleConnTimeout, TimeUnit.MILLISECONDS);
+
+ if (disconnectData != null)
+ processDisconnect(disconnectData);
+ else
+ processIdle();
+ }
+ }
+ catch (Throwable t) {
+ if (!(t instanceof InterruptedException))
+ err = t;
+
+ throw t;
+ }
+ finally {
+ if (err == null && !stopping)
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly.");
+
+ if (err instanceof OutOfMemoryError)
+ ((IgniteEx)ignite).context().failure().process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ ((IgniteEx)ignite).context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 4aa1316..7bf37e1 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -50,6 +50,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.LinkedBlockingDeque;
@@ -66,6 +67,8 @@ import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.CacheMetrics;
import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.IgniteNodeAttributes;
@@ -73,6 +76,7 @@ import org.apache.ignite.internal.IgnitionEx;
import org.apache.ignite.internal.events.DiscoveryCustomEvent;
import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
import org.apache.ignite.internal.managers.discovery.DiscoveryServerOnlyCustomMessage;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
import org.apache.ignite.internal.processors.security.SecurityContext;
import org.apache.ignite.internal.processors.security.SecurityUtils;
import org.apache.ignite.internal.util.GridBoundedLinkedHashSet;
@@ -137,7 +141,6 @@ import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryServerOnlyCustom
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryStatusCheckMessage;
import org.apache.ignite.thread.IgniteThreadPoolExecutor;
import org.jetbrains.annotations.Nullable;
-import java.util.concurrent.ConcurrentHashMap;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCOVERY_CLIENT_RECONNECT_HISTORY_SIZE;
@@ -149,6 +152,8 @@ import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER;
@@ -2609,12 +2614,20 @@ class ServerImpl extends TcpDiscoveryImpl {
/** {@inheritDoc} */
@Override protected void body() throws InterruptedException {
+ Throwable err = null;
+
try {
super.body();
}
+ catch (InterruptedException e) {
+ if (!spi.isNodeStopping0())
+ err = e;
+
+ throw e;
+ }
catch (Throwable e) {
if (!spi.isNodeStopping0() && spiStateCopy() != DISCONNECTING) {
- final Ignite ignite = spi.ignite();
+ final Ignite ignite = spi.ignite();
if (ignite != null) {
U.error(log, "TcpDiscoverSpi's message worker thread failed abnormally. " +
@@ -2637,9 +2650,22 @@ class ServerImpl extends TcpDiscoveryImpl {
}
}
+ err = e;
+
// Must be processed by IgniteSpiThread as well.
throw e;
}
+ finally {
+ if (err == null && !spi.isNodeStopping0())
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly.");
+
+ FailureProcessor failure = ((IgniteEx)spi.ignite()).context().failure();
+
+ if (err instanceof OutOfMemoryError)
+ failure.process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ failure.process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
}
/**
@@ -5597,7 +5623,9 @@ class ServerImpl extends TcpDiscoveryImpl {
}
/** {@inheritDoc} */
- @Override protected void body() throws InterruptedException {
+ @Override protected void body() {
+ Throwable err = null;
+
try {
while (!isInterrupted()) {
Socket sock = srvrSock.accept();
@@ -5630,13 +5658,30 @@ class ServerImpl extends TcpDiscoveryImpl {
onException("Failed to accept TCP connection.", e);
if (!isInterrupted()) {
+ err = e;
+
if (U.isMacInvalidArgumentError(e))
U.error(log, "Failed to accept TCP connection\n\t" + U.MAC_INVALID_ARG_MSG, e);
else
U.error(log, "Failed to accept TCP connection.", e);
}
}
+ catch (Throwable t) {
+ err = t;
+
+ throw t;
+ }
finally {
+ if (err == null && !spi.isNodeStopping0())
+ err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly.");
+
+ FailureProcessor failure = ((IgniteEx)spi.ignite()).context().failure();
+
+ if (err instanceof OutOfMemoryError)
+ failure.process(new FailureContext(CRITICAL_ERROR, err));
+ else if (err != null)
+ failure.process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+
U.closeQuiet(srvrSock);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c807ae95/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
index 543907f..3fca7af 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
@@ -29,7 +29,7 @@ public class StripedExecutorTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override public void beforeTest() {
- stripedExecSvc = new StripedExecutor(3, "foo name", "pool name", new JavaLogger());
+ stripedExecSvc = new StripedExecutor(3, "foo name", "pool name", new JavaLogger(), (thread, t) -> {});
}
/** {@inheritDoc} */
[29/54] [abbrv] ignite git commit: IGNITE-8216 Fixed javadoc for
release build
Posted by ag...@apache.org.
IGNITE-8216 Fixed javadoc for release build
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6557fe62
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6557fe62
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6557fe62
Branch: refs/heads/ignite-6083
Commit: 6557fe62696ac24c740e445b53482da298b59b27
Parents: 780fc07
Author: Sergey Chugunov <se...@gmail.com>
Authored: Wed Apr 11 12:28:40 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Apr 11 12:28:40 2018 +0300
----------------------------------------------------------------------
parent/pom.xml | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6557fe62/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index 16a9395..3decc16 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -404,6 +404,10 @@
<packages>org.apache.ignite.spi.eventstorage*</packages>
</group>
<group>
+ <title>Communication Failure Detection</title>
+ <packages>org.apache.ignite.failure</packages>
+ </group>
+ <group>
<title>Segmentation Detection</title>
<packages>org.apache.ignite.plugin.segmentation</packages>
</group>
[41/54] [abbrv] ignite git commit: IGNITE-7996 Merge with master.
Posted by ag...@apache.org.
IGNITE-7996 Merge with master.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e333f306
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e333f306
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e333f306
Branch: refs/heads/ignite-6083
Commit: e333f306d0f32d4c02057fff4238081f25775cf1
Parents: d1be9b8 d02e87b
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Thu Apr 12 11:07:02 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Apr 12 11:10:27 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/app/app.js | 2 -
.../components/cache-edit-form/template.tpl.pug | 22 +-
.../cache-edit-form/templates/affinity.pug | 86 +++++
.../cache-edit-form/templates/concurrency.pug | 64 ++++
.../cache-edit-form/templates/general.pug | 113 +++++++
.../cache-edit-form/templates/memory.pug | 158 ++++++++++
.../templates/near-cache-client.pug | 50 +++
.../templates/near-cache-server.pug | 51 +++
.../cache-edit-form/templates/node-filter.pug | 53 ++++
.../cache-edit-form/templates/query.pug | 114 +++++++
.../cache-edit-form/templates/rebalance.pug | 66 ++++
.../cache-edit-form/templates/statistics.pug | 34 ++
.../cache-edit-form/templates/store.pug | 310 +++++++++++++++++++
.../cluster-edit-form/template.tpl.pug | 62 ++--
.../cluster-edit-form/templates/atomic.pug | 75 +++++
.../cluster-edit-form/templates/attributes.pug | 40 +++
.../cluster-edit-form/templates/binary.pug | 80 +++++
.../templates/cache-key-cfg.pug | 63 ++++
.../cluster-edit-form/templates/checkpoint.pug | 82 +++++
.../templates/checkpoint/fs.pug | 36 +++
.../templates/checkpoint/jdbc.pug | 47 +++
.../templates/checkpoint/s3.pug | 204 ++++++++++++
.../templates/client-connector.pug | 76 +++++
.../cluster-edit-form/templates/collision.pug | 58 ++++
.../templates/collision/custom.pug | 23 ++
.../templates/collision/fifo-queue.pug | 26 ++
.../templates/collision/job-stealing.pug | 51 +++
.../templates/collision/priority-queue.pug | 41 +++
.../templates/communication.pug | 134 ++++++++
.../cluster-edit-form/templates/connector.pug | 100 ++++++
.../templates/data-storage.pug | 301 ++++++++++++++++++
.../cluster-edit-form/templates/deployment.pug | 192 ++++++++++++
.../cluster-edit-form/templates/discovery.pug | 97 ++++++
.../cluster-edit-form/templates/events.pug | 66 ++++
.../cluster-edit-form/templates/failover.pug | 89 ++++++
.../cluster-edit-form/templates/general.pug | 89 ++++++
.../templates/general/discovery/cloud.pug | 78 +++++
.../templates/general/discovery/google.pug | 38 +++
.../templates/general/discovery/jdbc.pug | 35 +++
.../templates/general/discovery/kubernetes.pug | 38 +++
.../templates/general/discovery/multicast.pug | 63 ++++
.../templates/general/discovery/s3.pug | 38 +++
.../templates/general/discovery/shared.pug | 24 ++
.../templates/general/discovery/vm.pug | 55 ++++
.../templates/general/discovery/zookeeper.pug | 84 +++++
.../retrypolicy/bounded-exponential-backoff.pug | 26 ++
.../discovery/zookeeper/retrypolicy/custom.pug | 25 ++
.../retrypolicy/exponential-backoff.pug | 26 ++
.../discovery/zookeeper/retrypolicy/forever.pug | 23 ++
.../discovery/zookeeper/retrypolicy/n-times.pug | 24 ++
.../zookeeper/retrypolicy/one-time.pug | 23 ++
.../zookeeper/retrypolicy/until-elapsed.pug | 24 ++
.../cluster-edit-form/templates/hadoop.pug | 87 ++++++
.../cluster-edit-form/templates/igfs.pug | 34 ++
.../templates/load-balancing.pug | 115 +++++++
.../cluster-edit-form/templates/logger.pug | 60 ++++
.../templates/logger/custom.pug | 24 ++
.../templates/logger/log4j.pug | 49 +++
.../templates/logger/log4j2.pug | 38 +++
.../cluster-edit-form/templates/marshaller.pug | 75 +++++
.../cluster-edit-form/templates/memory.pug | 195 ++++++++++++
.../cluster-edit-form/templates/metrics.pug | 46 +++
.../cluster-edit-form/templates/misc.pug | 58 ++++
.../cluster-edit-form/templates/odbc.pug | 70 +++++
.../cluster-edit-form/templates/persistence.pug | 82 +++++
.../cluster-edit-form/templates/service.pug | 89 ++++++
.../templates/sql-connector.pug | 58 ++++
.../cluster-edit-form/templates/ssl.pug | 89 ++++++
.../cluster-edit-form/templates/swap.pug | 74 +++++
.../cluster-edit-form/templates/thread.pug | 144 +++++++++
.../cluster-edit-form/templates/time.pug | 44 +++
.../templates/transactions.pug | 65 ++++
.../components/igfs-edit-form/template.tpl.pug | 12 +-
.../igfs-edit-form/templates/dual.pug | 42 +++
.../igfs-edit-form/templates/fragmentizer.pug | 37 +++
.../igfs-edit-form/templates/general.pug | 72 +++++
.../components/igfs-edit-form/templates/ipc.pug | 55 ++++
.../igfs-edit-form/templates/misc.pug | 110 +++++++
.../igfs-edit-form/templates/secondary.pug | 55 ++++
.../components/model-edit-form/template.tpl.pug | 6 +-
.../model-edit-form/templates/general.pug | 57 ++++
.../model-edit-form/templates/query.pug | 255 +++++++++++++++
.../model-edit-form/templates/store.pug | 123 ++++++++
.../page-configure-basic/template.pug | 18 +-
.../components/preview-panel/directive.js | 246 +++++++++++++++
.../components/preview-panel/index.js | 23 ++
.../app/components/page-configure/index.js | 25 +-
.../services/ConfigurationResource.js | 49 +++
.../page-configure/services/SummaryZipper.js | 44 +++
.../page-configure/services/summary.worker.js | 147 +++++++++
.../app/components/page-configure/states.js | 270 ++++++++++++++++
.../app/modules/states/configuration.state.js | 297 ------------------
.../configuration/Configuration.resource.js | 42 ---
.../states/configuration/caches/affinity.pug | 86 -----
.../states/configuration/caches/concurrency.pug | 64 ----
.../states/configuration/caches/general.pug | 113 -------
.../states/configuration/caches/memory.pug | 158 ----------
.../configuration/caches/near-cache-client.pug | 50 ---
.../configuration/caches/near-cache-server.pug | 51 ---
.../states/configuration/caches/node-filter.pug | 53 ----
.../states/configuration/caches/query.pug | 114 -------
.../states/configuration/caches/rebalance.pug | 66 ----
.../states/configuration/caches/statistics.pug | 34 --
.../states/configuration/caches/store.pug | 310 -------------------
.../states/configuration/clusters/atomic.pug | 75 -----
.../configuration/clusters/attributes.pug | 40 ---
.../states/configuration/clusters/binary.pug | 80 -----
.../configuration/clusters/cache-key-cfg.pug | 63 ----
.../configuration/clusters/checkpoint.pug | 82 -----
.../configuration/clusters/checkpoint/fs.pug | 36 ---
.../configuration/clusters/checkpoint/jdbc.pug | 47 ---
.../configuration/clusters/checkpoint/s3.pug | 204 ------------
.../configuration/clusters/client-connector.pug | 76 -----
.../states/configuration/clusters/collision.pug | 58 ----
.../configuration/clusters/collision/custom.pug | 23 --
.../clusters/collision/fifo-queue.pug | 26 --
.../clusters/collision/job-stealing.pug | 51 ---
.../clusters/collision/priority-queue.pug | 41 ---
.../configuration/clusters/communication.pug | 134 --------
.../states/configuration/clusters/connector.pug | 100 ------
.../configuration/clusters/data-storage.pug | 301 ------------------
.../configuration/clusters/deployment.pug | 192 ------------
.../states/configuration/clusters/discovery.pug | 97 ------
.../states/configuration/clusters/events.pug | 66 ----
.../states/configuration/clusters/failover.pug | 89 ------
.../states/configuration/clusters/general.pug | 89 ------
.../clusters/general/discovery/cloud.pug | 78 -----
.../clusters/general/discovery/google.pug | 38 ---
.../clusters/general/discovery/jdbc.pug | 35 ---
.../clusters/general/discovery/kubernetes.pug | 38 ---
.../clusters/general/discovery/multicast.pug | 63 ----
.../clusters/general/discovery/s3.pug | 38 ---
.../clusters/general/discovery/shared.pug | 24 --
.../clusters/general/discovery/vm.pug | 55 ----
.../clusters/general/discovery/zookeeper.pug | 84 -----
.../retrypolicy/bounded-exponential-backoff.pug | 26 --
.../discovery/zookeeper/retrypolicy/custom.pug | 25 --
.../retrypolicy/exponential-backoff.pug | 26 --
.../discovery/zookeeper/retrypolicy/forever.pug | 23 --
.../discovery/zookeeper/retrypolicy/n-times.pug | 24 --
.../zookeeper/retrypolicy/one-time.pug | 23 --
.../zookeeper/retrypolicy/until-elapsed.pug | 24 --
.../states/configuration/clusters/hadoop.pug | 87 ------
.../states/configuration/clusters/igfs.pug | 34 --
.../configuration/clusters/load-balancing.pug | 115 -------
.../states/configuration/clusters/logger.pug | 60 ----
.../configuration/clusters/logger/custom.pug | 24 --
.../configuration/clusters/logger/log4j.pug | 49 ---
.../configuration/clusters/logger/log4j2.pug | 38 ---
.../configuration/clusters/marshaller.pug | 75 -----
.../states/configuration/clusters/memory.pug | 195 ------------
.../states/configuration/clusters/metrics.pug | 46 ---
.../states/configuration/clusters/misc.pug | 58 ----
.../states/configuration/clusters/odbc.pug | 70 -----
.../configuration/clusters/persistence.pug | 82 -----
.../states/configuration/clusters/service.pug | 89 ------
.../configuration/clusters/sql-connector.pug | 58 ----
.../states/configuration/clusters/ssl.pug | 89 ------
.../states/configuration/clusters/swap.pug | 74 -----
.../states/configuration/clusters/thread.pug | 144 ---------
.../states/configuration/clusters/time.pug | 44 ---
.../configuration/clusters/transactions.pug | 65 ----
.../states/configuration/domains/general.pug | 57 ----
.../states/configuration/domains/query.pug | 255 ---------------
.../states/configuration/domains/store.pug | 123 --------
.../modules/states/configuration/igfs/dual.pug | 42 ---
.../states/configuration/igfs/fragmentizer.pug | 37 ---
.../states/configuration/igfs/general.pug | 72 -----
.../modules/states/configuration/igfs/ipc.pug | 55 ----
.../modules/states/configuration/igfs/misc.pug | 110 -------
.../states/configuration/igfs/secondary.pug | 55 ----
.../configuration/preview-panel.directive.js | 239 --------------
.../summary/summary-zipper.service.js | 39 ---
.../configuration/summary/summary.worker.js | 147 ---------
174 files changed, 6858 insertions(+), 6822 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e333f306/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/communication.pug
----------------------------------------------------------------------
diff --cc modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/communication.pug
index 0000000,bd8971a..8b43521
mode 000000,100644..100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/communication.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/communication.pug
@@@ -1,0 -1,134 +1,134 @@@
+ //-
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ include /app/helpers/jade/mixins
+
+ -var form = 'communication'
+ -var model = '$ctrl.clonedCluster'
+ -var communication = model + '.communication'
+
+ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
+ panel-title Communication
+ panel-description
+ | Configuration of communication with other nodes by TCP/IP.
+ | Provide basic plumbing to send and receive grid messages and is utilized for all distributed grid operations.
+ | #[a.link-success(href="https://apacheignite.readme.io/docs/network-config" target="_blank") More info]
+ panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
+ .pca-form-column-6.pc-form-grid-row
+ .pc-form-grid-col-30
+ +number('Timeout:', `${model}.networkTimeout`, '"commNetworkTimeout"', 'true', '5000', '1', 'Maximum timeout in milliseconds for network requests')
+ .pc-form-grid-col-30
+ +number('Send retry delay:', `${model}.networkSendRetryDelay`, '"networkSendRetryDelay"', 'true', '1000', '1', 'Interval in milliseconds between message send retries')
+ .pc-form-grid-col-30
+ +number('Send retry count:', `${model}.networkSendRetryCount`, '"networkSendRetryCount"', 'true', '3', '1', 'Message send retries count')
+ .pc-form-grid-col-30(ng-if='$ctrl.available(["1.0.0", "2.3.0"])')
+ +number('Discovery startup delay:', `${model}.discoveryStartupDelay`, '"discoveryStartupDelay"', 'true', '60000', '1', 'This value is used to expire messages from waiting list whenever node discovery discrepancies happen')
+ .pc-form-grid-col-60
+ +java-class('Communication listener:', `${communication}.listener`, '"comListener"', 'true', 'false', 'Listener of communication events')
+ .pc-form-grid-col-30
+ +text-ip-address('Local IP address:', `${communication}.localAddress`, '"comLocalAddress"', 'true', '0.0.0.0',
+ 'Local host address for socket binding<br/>\
+ If not specified use all available addres on local host')
+ .pc-form-grid-col-30
+ +number-min-max('Local port:', `${communication}.localPort`, '"comLocalPort"', 'true', '47100', '1024', '65535', 'Local port for socket binding')
+ .pc-form-grid-col-30
+ +number('Local port range:', `${communication}.localPortRange`, '"comLocalPortRange"', 'true', '100', '1', 'Local port range for local host ports')
+ .pc-form-grid-col-30
+ +sane-ignite-form-field-number({
+ label: 'Shared memory port:',
+ model: `${communication}.sharedMemoryPort`,
+ name: '"sharedMemoryPort"',
+ placeholder: '{{ ::$ctrl.Clusters.sharedMemoryPort.default }}',
+ min: '{{ ::$ctrl.Clusters.sharedMemoryPort.min }}',
+ max: '{{ ::$ctrl.Clusters.sharedMemoryPort.max }}',
+ tip: `Local port to accept shared memory connections<br/>If set to <b>-1</b> shared memory communication will be disabled`
+ })(
+ pc-not-in-collection='::$ctrl.Clusters.sharedMemoryPort.invalidValues'
+ )
+ +form-field-feedback('"sharedMemoryPort"', 'notInCollection', 'Shared memory port should be more than "{{ ::$ctrl.Clusters.sharedMemoryPort.invalidValues[0] }}" or equal to "{{ ::$ctrl.Clusters.sharedMemoryPort.min }}"')
+ .pc-form-grid-col-30
+ +number('Idle connection timeout:', `${communication}.idleConnectionTimeout`, '"idleConnectionTimeout"', 'true', '30000', '1',
+ 'Maximum idle connection timeout upon which a connection to client will be closed')
+ .pc-form-grid-col-30
+ +number('Connect timeout:', `${communication}.connectTimeout`, '"connectTimeout"', 'true', '5000', '0', 'Connect timeout used when establishing connection with remote nodes')
+ .pc-form-grid-col-30
+ +number('Max. connect timeout:', `${communication}.maxConnectTimeout`, '"maxConnectTimeout"', 'true', '600000', '0', 'Maximum connect timeout')
+ .pc-form-grid-col-30
+ +number('Reconnect count:', `${communication}.reconnectCount`, '"comReconnectCount"', 'true', '10', '1',
+ 'Maximum number of reconnect attempts used when establishing connection with remote nodes')
+ .pc-form-grid-col-30
+ +number('Socket send buffer:', `${communication}.socketSendBuffer`, '"socketSendBuffer"', 'true', '32768', '0', 'Send buffer size for sockets created or accepted by this SPI')
+ .pc-form-grid-col-30
+ +number('Socket receive buffer:', `${communication}.socketReceiveBuffer`, '"socketReceiveBuffer"', 'true', '32768', '0', 'Receive buffer size for sockets created or accepted by this SPI')
+ .pc-form-grid-col-30
+ +number('Slow client queue limit:', `${communication}.slowClientQueueLimit`, '"slowClientQueueLimit"', 'true', '0', '0', 'Slow client queue limit')
+ .pc-form-grid-col-30
+ +sane-ignite-form-field-number({
+ label: 'Ack send threshold:',
+ model: `${communication}.ackSendThreshold`,
+ name: '"ackSendThreshold"',
+ placeholder: '{{ ::$ctrl.Clusters.ackSendThreshold.default }}',
+ min: '{{ ::$ctrl.Clusters.ackSendThreshold.min }}',
+ tip: 'Number of received messages per connection to node after which acknowledgment message is sent'
+ })
+ .pc-form-grid-col-30
+ +sane-ignite-form-field-number({
+ label: 'Message queue limit:',
+ model: `${communication}.messageQueueLimit`,
+ name: '"messageQueueLimit"',
+ placeholder: '{{ ::$ctrl.Clusters.messageQueueLimit.default }}',
+ min: '{{ ::$ctrl.Clusters.messageQueueLimit.min }}',
+ tip: 'Message queue limit for incoming and outgoing messages'
+ })
+ .pc-form-grid-col-30
++ //- allowInvalid: true prevents from infinite digest loop when old value was 0 and becomes less than allowed minimum
+ +sane-ignite-form-field-number({
+ label: 'Unacknowledged messages:',
+ model: `${communication}.unacknowledgedMessagesBufferSize`,
+ name: '"unacknowledgedMessagesBufferSize"',
+ placeholder: '{{ ::$ctrl.Clusters.unacknowledgedMessagesBufferSize.default }}',
+ min: `{{ $ctrl.Clusters.unacknowledgedMessagesBufferSize.min(
+ ${communication}.unacknowledgedMessagesBufferSize,
+ ${communication}.messageQueueLimit,
+ ${communication}.ackSendThreshold
+ ) }}`,
+ tip: `Maximum number of stored unacknowledged messages per connection to node<br/>
+ If specified non zero value it should be
+ <ul>
+ <li>At least ack send threshold * {{ ::$ctrl.Clusters.unacknowledgedMessagesBufferSize.validRatio }}</li>
+ <li>At least message queue limit * {{ ::$ctrl.Clusters.unacknowledgedMessagesBufferSize.validRatio }}</li>
+ </ul>`
+ })(
- //- allowInvalid: true prevents from infinite digest loop when old value was 0 and becomes less than allowed minimum
+ ng-model-options=`{
+ allowInvalid: true
+ }`
+ )
+ .pc-form-grid-col-30
+ +number('Socket write timeout:', `${communication}.socketWriteTimeout`, '"socketWriteTimeout"', 'true', '2000', '0', 'Socket write timeout')
+ .pc-form-grid-col-30
+ +number('Selectors count:', `${communication}.selectorsCount`, '"selectorsCount"', 'true', 'min(4, availableProcessors)', '1', 'Count of selectors te be used in TCP server')
+ .pc-form-grid-col-60
+ +java-class('Address resolver:', `${communication}.addressResolver`, '"comAddressResolver"', 'true', 'false', 'Provides resolution between external and internal addresses')
+ .pc-form-grid-col-60
+ +checkbox('Direct buffer', `${communication}.directBuffer`, '"directBuffer"',
+ 'If value is true, then SPI will use ByteBuffer.allocateDirect(int) call<br/>\
+ Otherwise, SPI will use ByteBuffer.allocate(int) call')
+ .pc-form-grid-col-60
+ +checkbox('Direct send buffer', `${communication}.directSendBuffer`, '"directSendBuffer"', 'Flag defining whether direct send buffer should be used')
+ .pc-form-grid-col-60
+ +checkbox('TCP_NODELAY option', `${communication}.tcpNoDelay`, '"tcpNoDelay"', 'Value for TCP_NODELAY socket option')
+ .pca-form-column-6
+ +preview-xml-java(model, 'clusterCommunication')
http://git-wip-us.apache.org/repos/asf/ignite/blob/e333f306/modules/web-console/frontend/app/components/page-configure/services/summary.worker.js
----------------------------------------------------------------------
diff --cc modules/web-console/frontend/app/components/page-configure/services/summary.worker.js
index 0000000,c80d698..b3b0bce
mode 000000,100644..100644
--- a/modules/web-console/frontend/app/components/page-configure/services/summary.worker.js
+++ b/modules/web-console/frontend/app/components/page-configure/services/summary.worker.js
@@@ -1,0 -1,147 +1,147 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ import JSZip from 'jszip';
+
+ import IgniteMavenGenerator from 'app/modules/configuration/generator/Maven.service';
+ import IgniteDockerGenerator from 'app/modules/configuration/generator/Docker.service';
+ import IgniteReadmeGenerator from 'app/modules/configuration/generator/Readme.service';
+ import IgnitePropertiesGenerator from 'app/modules/configuration/generator/Properties.service';
+ import IgniteConfigurationGenerator from 'app/modules/configuration/generator/ConfigurationGenerator';
+
+ import IgniteJavaTransformer from 'app/modules/configuration/generator/JavaTransformer.service';
+ import IgniteSpringTransformer from 'app/modules/configuration/generator/SpringTransformer.service';
+
+ import {nonEmpty, nonNil} from 'app/utils/lodashMixins';
+ import get from 'lodash/get';
+ import filter from 'lodash/filter';
+ import isEmpty from 'lodash/isEmpty';
+
+ const maven = new IgniteMavenGenerator();
+ const docker = new IgniteDockerGenerator();
+ const readme = new IgniteReadmeGenerator();
+ const properties = new IgnitePropertiesGenerator();
+
+ const java = IgniteJavaTransformer;
+ const spring = IgniteSpringTransformer;
+
+ const generator = IgniteConfigurationGenerator;
+
+ const escapeFileName = (name) => name.replace(/[\\\/*\"\[\],\.:;|=<>?]/g, '-').replace(/ /g, '_');
+
+ const kubernetesConfig = (cluster) => {
+ if (!cluster.discovery.Kubernetes)
+ cluster.discovery.Kubernetes = { serviceName: 'ignite' };
+
+ return `apiVersion: v1\n\
+ kind: Service\n\
+ metadata:\n\
+ # Name of Ignite Service used by Kubernetes IP finder for IP addresses lookup.\n\
+ name: ${ cluster.discovery.Kubernetes.serviceName || 'ignite' }\n\
+ spec:\n\
+ clusterIP: None # custom value.\n\
+ ports:\n\
+ - port: 9042 # custom value.\n\
+ selector:\n\
+ # Must be equal to one of the labels set in Ignite pods'\n\
+ # deployement configuration.\n\
+ app: ${ cluster.discovery.Kubernetes.serviceName || 'ignite' }`;
+ };
+
+ // eslint-disable-next-line no-undef
+ onmessage = function(e) {
+ const {cluster, data, demo, targetVer} = e.data;
+
+ const zip = new JSZip();
+
+ if (!data.docker)
+ data.docker = docker.generate(cluster, targetVer);
+
+ zip.file('Dockerfile', data.docker);
+ zip.file('.dockerignore', docker.ignoreFile());
+
+ const cfg = generator.igniteConfiguration(cluster, targetVer, false);
+ const clientCfg = generator.igniteConfiguration(cluster, targetVer, true);
+ const clientNearCaches = filter(cluster.caches, (cache) =>
+ cache.cacheMode === 'PARTITIONED' && get(cache, 'clientNearConfiguration.enabled'));
+
+ const secProps = properties.generate(cfg);
+
+ if (secProps)
+ zip.file('src/main/resources/secret.properties', secProps);
+
+ const srcPath = 'src/main/java';
+ const resourcesPath = 'src/main/resources';
+
+ const serverXml = `${escapeFileName(cluster.name)}-server.xml`;
+ const clientXml = `${escapeFileName(cluster.name)}-client.xml`;
+
+ const metaPath = `${resourcesPath}/META-INF`;
+
+ if (cluster.discovery.kind === 'Kubernetes')
+ zip.file(`${metaPath}/ignite-service.yaml`, kubernetesConfig(cluster));
+
+ zip.file(`${metaPath}/${serverXml}`, spring.igniteConfiguration(cfg, targetVer).asString());
+ zip.file(`${metaPath}/${clientXml}`, spring.igniteConfiguration(clientCfg, targetVer, clientNearCaches).asString());
+
+ const cfgPath = `${srcPath}/config`;
+
+ zip.file(`${cfgPath}/ServerConfigurationFactory.java`, java.igniteConfiguration(cfg, targetVer, 'config', 'ServerConfigurationFactory').asString());
+ zip.file(`${cfgPath}/ClientConfigurationFactory.java`, java.igniteConfiguration(clientCfg, targetVer, 'config', 'ClientConfigurationFactory', clientNearCaches).asString());
+
+ if (java.isDemoConfigured(cluster, demo)) {
+ zip.file(`${srcPath}/demo/DemoStartup.java`, java.nodeStartup(cluster, 'demo.DemoStartup',
+ 'ServerConfigurationFactory.createConfiguration()', 'config.ServerConfigurationFactory'));
+ }
+
+ // Generate loader for caches with configured store.
- const cachesToLoad = filter(cluster.caches, (cache) => nonNil(cache.cacheStoreFactory));
++ const cachesToLoad = filter(cluster.caches, (cache) => nonNil(_.get(cache, 'cacheStoreFactory.kind')));
+
+ if (nonEmpty(cachesToLoad))
+ zip.file(`${srcPath}/load/LoadCaches.java`, java.loadCaches(cachesToLoad, 'load', 'LoadCaches', `"${clientXml}"`));
+
+ const startupPath = `${srcPath}/startup`;
+
+ zip.file(`${startupPath}/ServerNodeSpringStartup.java`, java.nodeStartup(cluster, 'startup.ServerNodeSpringStartup', `"${serverXml}"`));
+ zip.file(`${startupPath}/ClientNodeSpringStartup.java`, java.nodeStartup(cluster, 'startup.ClientNodeSpringStartup', `"${clientXml}"`));
+
+ zip.file(`${startupPath}/ServerNodeCodeStartup.java`, java.nodeStartup(cluster, 'startup.ServerNodeCodeStartup',
+ 'ServerConfigurationFactory.createConfiguration()', 'config.ServerConfigurationFactory'));
+ zip.file(`${startupPath}/ClientNodeCodeStartup.java`, java.nodeStartup(cluster, 'startup.ClientNodeCodeStartup',
+ 'ClientConfigurationFactory.createConfiguration()', 'config.ClientConfigurationFactory', clientNearCaches));
+
+ zip.file('pom.xml', maven.generate(cluster, targetVer));
+
+ zip.file('README.txt', readme.generate());
+ zip.file('jdbc-drivers/README.txt', readme.generateJDBC());
+
+ if (isEmpty(data.pojos))
+ data.pojos = java.pojos(cluster.caches, true);
+
+ for (const pojo of data.pojos) {
+ if (pojo.keyClass)
+ zip.file(`${srcPath}/${pojo.keyType.replace(/\./g, '/')}.java`, pojo.keyClass);
+
+ zip.file(`${srcPath}/${pojo.valueType.replace(/\./g, '/')}.java`, pojo.valueClass);
+ }
+
+ zip.generateAsync({
+ type: 'blob',
+ compression: 'DEFLATE',
+ mimeType: 'application/octet-stream'
+ }).then((blob) => postMessage(blob));
+ };
http://git-wip-us.apache.org/repos/asf/ignite/blob/e333f306/modules/web-console/frontend/app/components/page-configure/states.js
----------------------------------------------------------------------
diff --cc modules/web-console/frontend/app/components/page-configure/states.js
index 0000000,f8bb4dc..a75e851
mode 000000,100644..100644
--- a/modules/web-console/frontend/app/components/page-configure/states.js
+++ b/modules/web-console/frontend/app/components/page-configure/states.js
@@@ -1,0 -1,273 +1,270 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
-import angular from 'angular';
-
+ import base2 from 'views/base2.pug';
+ import pageConfigureAdvancedClusterComponent from '../page-configure-advanced/components/page-configure-advanced-cluster/component';
+ import pageConfigureAdvancedModelsComponent from '../page-configure-advanced/components/page-configure-advanced-models/component';
+ import pageConfigureAdvancedCachesComponent from '../page-configure-advanced/components/page-configure-advanced-caches/component';
+ import pageConfigureAdvancedIGFSComponent from '../page-configure-advanced/components/page-configure-advanced-igfs/component';
+
-import get from 'lodash/get';
+ import {Observable} from 'rxjs/Observable';
+
+ const idRegex = `new|[a-z0-9]+`;
+
+ const shortCachesResolve = ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
+ if ($transition$.params().clusterID === 'new') return Promise.resolve();
+ return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
+ .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+ .switchMap((cluster) => {
+ return etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id});
+ })
+ .toPromise();
+ }];
+
+ function registerStates($stateProvider) {
+ // Setup the states.
+ $stateProvider
+ .state('base.configuration', {
+ abstract: true,
+ permission: 'configuration',
+ url: '/configuration',
+ onEnter: ['ConfigureState', (ConfigureState) => ConfigureState.dispatchAction({type: 'PRELOAD_STATE', state: {}})],
+ views: {
+ '@': {
+ template: base2
+ }
+ },
+ resolve: {
+ _shortClusters: ['ConfigEffects', ({etp}) => {
+ return etp('LOAD_USER_CLUSTERS');
+ }]
+ },
+ resolvePolicy: {
+ async: 'NOWAIT'
+ }
+ })
+ .state('base.configuration.overview', {
+ url: '/overview',
+ component: 'pageConfigureOverview',
+ permission: 'configuration',
+ tfMetaTags: {
+ title: 'Configuration'
+ }
+ })
+ .state('base.configuration.edit', {
+ url: `/{clusterID:${idRegex}}`,
+ permission: 'configuration',
+ component: 'pageConfigure',
+ resolve: {
+ _cluster: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
+ return $transition$.injector().getAsync('_shortClusters').then(() => {
+ return etp('LOAD_AND_EDIT_CLUSTER', {clusterID: $transition$.params().clusterID});
+ });
+ }]
+ },
+ data: {
+ errorState: 'base.configuration.overview'
+ },
+ redirectTo: ($transition$) => {
+ const [ConfigureState, ConfigSelectors] = ['ConfigureState', 'ConfigSelectors'].map((t) => $transition$.injector().get(t));
+ const waitFor = ['_cluster', '_shortClusters'].map((t) => $transition$.injector().getAsync(t));
+ return Observable.fromPromise(Promise.all(waitFor)).switchMap(() => {
+ return Observable.combineLatest(
+ ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1),
+ ConfigureState.state$.let(ConfigSelectors.selectShortClusters()).take(1)
+ );
+ })
+ .map(([cluster = {caches: []}, clusters]) => {
+ return (clusters.value.size > 10 || cluster.caches.length > 5)
+ ? 'base.configuration.edit.advanced'
+ : 'base.configuration.edit.basic';
+ })
+ .toPromise();
+ },
+ failState: 'signin',
+ tfMetaTags: {
+ title: 'Configuration'
+ }
+ })
+ .state('base.configuration.edit.basic', {
+ url: '/basic',
+ component: 'pageConfigureBasic',
+ permission: 'configuration',
+ resolve: {
+ _shortCaches: shortCachesResolve
+ },
+ resolvePolicy: {
+ async: 'NOWAIT'
+ },
+ tfMetaTags: {
+ title: 'Basic Configuration'
+ }
+ })
+ .state('base.configuration.edit.advanced', {
+ url: '/advanced',
+ component: 'pageConfigureAdvanced',
+ permission: 'configuration',
+ redirectTo: 'base.configuration.edit.advanced.cluster'
+ })
+ .state('base.configuration.edit.advanced.cluster', {
+ url: '/cluster',
+ component: pageConfigureAdvancedClusterComponent.name,
+ permission: 'configuration',
+ resolve: {
+ _shortCaches: shortCachesResolve
+ },
+ resolvePolicy: {
+ async: 'NOWAIT'
+ },
+ tfMetaTags: {
+ title: 'Configure Cluster'
+ }
+ })
+ .state('base.configuration.edit.advanced.caches', {
+ url: '/caches',
+ permission: 'configuration',
+ component: pageConfigureAdvancedCachesComponent.name,
+ resolve: {
+ _shortCachesAndModels: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
+ if ($transition$.params().clusterID === 'new') return Promise.resolve();
+ return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
+ .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+ .map((cluster) => {
+ return Promise.all([
+ etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
+ etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id}),
+ etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
+ ]);
+ })
+ .toPromise();
+ }]
+ },
+ resolvePolicy: {
+ async: 'NOWAIT'
+ },
+ tfMetaTags: {
+ title: 'Configure Caches'
+ }
+ })
+ .state('base.configuration.edit.advanced.caches.cache', {
+ url: `/{cacheID:${idRegex}}`,
+ permission: 'configuration',
+ resolve: {
+ _cache: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
+ const {clusterID, cacheID} = $transition$.params();
+ if (cacheID === 'new') return Promise.resolve();
+ return etp('LOAD_CACHE', {cacheID});
+ }]
+ },
+ data: {
+ errorState: 'base.configuration.edit.advanced.caches'
+ },
+ resolvePolicy: {
+ async: 'NOWAIT'
+ },
+ tfMetaTags: {
+ title: 'Configure Caches'
+ }
+ })
+ .state('base.configuration.edit.advanced.models', {
+ url: '/models',
+ component: pageConfigureAdvancedModelsComponent.name,
+ permission: 'configuration',
+ resolve: {
+ _shortCachesAndModels: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
+ if ($transition$.params().clusterID === 'new') return Promise.resolve();
+ return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
+ .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+ .map((cluster) => {
+ return Promise.all([
+ etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
+ etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id})
+ ]);
+ })
+ .toPromise();
+ }]
+ },
+ resolvePolicy: {
+ async: 'NOWAIT'
+ },
+ tfMetaTags: {
+ title: 'Configure SQL Schemes'
+ }
+ })
+ .state('base.configuration.edit.advanced.models.model', {
+ url: `/{modelID:${idRegex}}`,
+ resolve: {
+ _cache: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
+ const {clusterID, modelID} = $transition$.params();
+ if (modelID === 'new') return Promise.resolve();
+ return etp('LOAD_MODEL', {modelID});
+ }]
+ },
+ data: {
+ errorState: 'base.configuration.edit.advanced.models'
+ },
+ permission: 'configuration',
+ resolvePolicy: {
+ async: 'NOWAIT'
+ }
+ })
+ .state('base.configuration.edit.advanced.igfs', {
+ url: '/igfs',
+ component: pageConfigureAdvancedIGFSComponent.name,
+ permission: 'configuration',
+ resolve: {
+ _shortIGFSs: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
+ if ($transition$.params().clusterID === 'new') return Promise.resolve();
+ return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
+ .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+ .map((cluster) => {
+ return Promise.all([
+ etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
+ ]);
+ })
+ .toPromise();
+ }]
+ },
+ resolvePolicy: {
+ async: 'NOWAIT'
+ },
+ tfMetaTags: {
+ title: 'Configure IGFS'
+ }
+ })
+ .state('base.configuration.edit.advanced.igfs.igfs', {
+ url: `/{igfsID:${idRegex}}`,
+ permission: 'configuration',
+ resolve: {
+ _igfs: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
+ const {clusterID, igfsID} = $transition$.params();
+ if (igfsID === 'new') return Promise.resolve();
+ return etp('LOAD_IGFS', {igfsID});
+ }]
+ },
+ data: {
+ errorState: 'base.configuration.edit.advanced.igfs'
+ },
+ resolvePolicy: {
+ async: 'NOWAIT'
+ }
+ });
+ }
+
+ registerStates.$inject = ['$stateProvider'];
+
+ export {registerStates};
[44/54] [abbrv] ignite git commit: IGNITE-8176: Integrate gradient
descent linear regression with partition based dataset
Posted by ag...@apache.org.
IGNITE-8176: Integrate gradient descent linear regression with partition based dataset
this closes #3787
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/df6356d5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/df6356d5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/df6356d5
Branch: refs/heads/ignite-6083
Commit: df6356d5d1470337a6ea705a332cf07f1dce2222
Parents: 67023a8
Author: dmitrievanthony <dm...@gmail.com>
Authored: Thu Apr 12 11:16:22 2018 +0300
Committer: YuriBabak <y....@gmail.com>
Committed: Thu Apr 12 11:16:22 2018 +0300
----------------------------------------------------------------------
.../ml/knn/KNNClassificationExample.java | 11 +-
.../examples/ml/nn/MLPTrainerExample.java | 4 +-
.../ml/preprocessing/NormalizationExample.java | 17 +--
...nWithLSQRTrainerAndNormalizationExample.java | 23 ++--
...dLinearRegressionWithLSQRTrainerExample.java | 14 +--
...tedLinearRegressionWithQRTrainerExample.java | 9 +-
...edLinearRegressionWithSGDTrainerExample.java | 78 +++++++++---
.../binary/SVMBinaryClassificationExample.java | 11 +-
.../SVMMultiClassClassificationExample.java | 24 ++--
...ecisionTreeClassificationTrainerExample.java | 7 +-
.../DecisionTreeRegressionTrainerExample.java | 4 +-
.../org/apache/ignite/ml/nn/Activators.java | 20 ++++
.../org/apache/ignite/ml/nn/MLPTrainer.java | 46 ++++++--
.../ml/preprocessing/PreprocessingTrainer.java | 41 ++++++-
.../normalization/NormalizationTrainer.java | 35 ++++--
.../linear/FeatureExtractorWrapper.java | 55 +++++++++
.../linear/LinearRegressionLSQRTrainer.java | 38 +-----
.../linear/LinearRegressionSGDTrainer.java | 118 +++++++++++++------
.../ignite/ml/trainers/DatasetTrainer.java | 46 ++++++++
.../ignite/ml/knn/KNNClassificationTest.java | 20 ++--
.../ignite/ml/nn/MLPTrainerIntegrationTest.java | 14 +--
.../org/apache/ignite/ml/nn/MLPTrainerTest.java | 22 ++--
.../MLPTrainerMnistIntegrationTest.java | 7 +-
.../ml/nn/performance/MLPTrainerMnistTest.java | 11 +-
.../normalization/NormalizationTrainerTest.java | 10 +-
.../ml/regressions/RegressionsTestSuite.java | 15 +--
...stributedLinearRegressionSGDTrainerTest.java | 35 ------
...stributedLinearRegressionSGDTrainerTest.java | 35 ------
...wareAbstractLinearRegressionTrainerTest.java | 3 +
.../linear/LinearRegressionLSQRTrainerTest.java | 14 ++-
.../linear/LinearRegressionSGDTrainerTest.java | 94 +++++++++++++++
.../LocalLinearRegressionSGDTrainerTest.java | 35 ------
.../ignite/ml/svm/SVMBinaryTrainerTest.java | 11 +-
.../ignite/ml/svm/SVMMultiClassTrainerTest.java | 11 +-
...reeClassificationTrainerIntegrationTest.java | 9 +-
.../DecisionTreeClassificationTrainerTest.java | 12 +-
...ionTreeRegressionTrainerIntegrationTest.java | 9 +-
.../tree/DecisionTreeRegressionTrainerTest.java | 12 +-
.../DecisionTreeMNISTIntegrationTest.java | 7 +-
.../tree/performance/DecisionTreeMNISTTest.java | 11 +-
40 files changed, 612 insertions(+), 386 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
index f3cdbbe..39a8431 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
@@ -17,9 +17,6 @@
package org.apache.ignite.examples.ml.knn;
-import java.util.Arrays;
-import java.util.UUID;
-import javax.cache.Cache;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
@@ -27,7 +24,6 @@ import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
import org.apache.ignite.ml.knn.classification.KNNClassificationTrainer;
import org.apache.ignite.ml.knn.classification.KNNStrategy;
@@ -35,6 +31,10 @@ import org.apache.ignite.ml.math.distances.EuclideanDistance;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
import org.apache.ignite.thread.IgniteThread;
+import javax.cache.Cache;
+import java.util.Arrays;
+import java.util.UUID;
+
/**
* Run kNN multi-class classification trainer over distributed dataset.
*
@@ -56,7 +56,8 @@ public class KNNClassificationExample {
KNNClassificationTrainer trainer = new KNNClassificationTrainer();
KNNClassificationModel knnMdl = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, dataCache),
+ ignite,
+ dataCache,
(k, v) -> Arrays.copyOfRange(v, 1, v.length),
(k, v) -> v[0]
).withK(3)
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
index efa1ba7..ce44cc6 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
@@ -23,7 +23,6 @@ import org.apache.ignite.Ignition;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.examples.ExampleNodeStartup;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
import org.apache.ignite.ml.nn.Activators;
@@ -99,7 +98,8 @@ public class MLPTrainerExample {
// Train neural network and get multilayer perceptron model.
MultilayerPerceptron mlp = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+ ignite,
+ trainingSet,
(k, v) -> new double[] {v.x, v.y},
(k, v) -> new double[] {v.lb}
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/NormalizationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/NormalizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/NormalizationExample.java
index e0bcd08..b2c4e12 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/NormalizationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/NormalizationExample.java
@@ -17,21 +17,19 @@
package org.apache.ignite.examples.ml.preprocessing;
-import java.util.Arrays;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.examples.ml.dataset.model.Person;
-import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.DatasetFactory;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.dataset.primitive.SimpleDataset;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.preprocessing.normalization.NormalizationPreprocessor;
import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
+import java.util.Arrays;
+
/**
* Example that shows how to use normalization preprocessor to normalize data.
*
@@ -47,8 +45,6 @@ public class NormalizationExample {
IgniteCache<Integer, Person> persons = createCache(ignite);
- DatasetBuilder<Integer, Person> builder = new CacheBasedDatasetBuilder<>(ignite, persons);
-
// Defines first preprocessor that extracts features from an upstream data.
IgniteBiFunction<Integer, Person, double[]> featureExtractor = (k, v) -> new double[] {
v.getAge(),
@@ -56,14 +52,11 @@ public class NormalizationExample {
};
// Defines second preprocessor that normalizes features.
- NormalizationPreprocessor<Integer, Person> preprocessor = new NormalizationTrainer<Integer, Person>()
- .fit(builder, featureExtractor, 2);
+ IgniteBiFunction<Integer, Person, double[]> preprocessor = new NormalizationTrainer<Integer, Person>()
+ .fit(ignite, persons, featureExtractor);
// Creates a cache based simple dataset containing features and providing standard dataset API.
- try (SimpleDataset<?> dataset = DatasetFactory.createSimpleDataset(
- builder,
- preprocessor
- )) {
+ try (SimpleDataset<?> dataset = DatasetFactory.createSimpleDataset(ignite, persons, preprocessor)) {
// Calculation of the mean value. This calculation will be performed in map-reduce manner.
double[] mean = dataset.mean();
System.out.println("Mean \n\t" + Arrays.toString(mean));
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
index 567a599..99e6577 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
@@ -17,9 +17,6 @@
package org.apache.ignite.examples.ml.regression.linear;
-import java.util.Arrays;
-import java.util.UUID;
-import javax.cache.Cache;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
@@ -28,7 +25,7 @@ import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.examples.ml.math.matrix.SparseDistributedMatrixExample;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
import org.apache.ignite.ml.preprocessing.normalization.NormalizationPreprocessor;
import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
@@ -36,6 +33,10 @@ import org.apache.ignite.ml.regressions.linear.LinearRegressionLSQRTrainer;
import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
import org.apache.ignite.thread.IgniteThread;
+import javax.cache.Cache;
+import java.util.Arrays;
+import java.util.UUID;
+
/**
* Run linear regression model over distributed matrix.
*
@@ -119,21 +120,17 @@ public class DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample {
NormalizationTrainer<Integer, double[]> normalizationTrainer = new NormalizationTrainer<>();
System.out.println(">>> Perform the training to get the normalization preprocessor.");
- NormalizationPreprocessor<Integer, double[]> preprocessor = normalizationTrainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, dataCache),
- (k, v) -> Arrays.copyOfRange(v, 1, v.length),
- 4
+ IgniteBiFunction<Integer, double[], double[]> preprocessor = normalizationTrainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> Arrays.copyOfRange(v, 1, v.length)
);
System.out.println(">>> Create new linear regression trainer object.");
LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
System.out.println(">>> Perform the training to get the model.");
- LinearRegressionModel mdl = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, dataCache),
- preprocessor,
- (k, v) -> v[0]
- );
+ LinearRegressionModel mdl = trainer.fit(ignite, dataCache, preprocessor, (k, v) -> v[0]);
System.out.println(">>> Linear regression model: " + mdl);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerExample.java
index a853092..25aec0c 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerExample.java
@@ -17,9 +17,6 @@
package org.apache.ignite.examples.ml.regression.linear;
-import java.util.Arrays;
-import java.util.UUID;
-import javax.cache.Cache;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
@@ -27,13 +24,15 @@ import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.examples.ml.math.matrix.SparseDistributedMatrixExample;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
import org.apache.ignite.ml.regressions.linear.LinearRegressionLSQRTrainer;
import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
import org.apache.ignite.thread.IgniteThread;
+import javax.cache.Cache;
+import java.util.Arrays;
+import java.util.UUID;
+
/**
* Run linear regression model over distributed matrix.
*
@@ -108,7 +107,7 @@ public class DistributedLinearRegressionWithLSQRTrainerExample {
// Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
// because we create ignite cache internally.
IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- SparseDistributedMatrixExample.class.getSimpleName(), () -> {
+ DistributedLinearRegressionWithLSQRTrainerExample.class.getSimpleName(), () -> {
IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
System.out.println(">>> Create new linear regression trainer object.");
@@ -116,7 +115,8 @@ public class DistributedLinearRegressionWithLSQRTrainerExample {
System.out.println(">>> Perform the training to get the model.");
LinearRegressionModel mdl = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, dataCache),
+ ignite,
+ dataCache,
(k, v) -> Arrays.copyOfRange(v, 1, v.length),
(k, v) -> v[0]
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java
index 2b45aa2..98d5e4e 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java
@@ -17,7 +17,6 @@
package org.apache.ignite.examples.ml.regression.linear;
-import java.util.Arrays;
import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.examples.ml.math.matrix.SparseDistributedMatrixExample;
@@ -30,6 +29,8 @@ import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
import org.apache.ignite.ml.regressions.linear.LinearRegressionQRTrainer;
import org.apache.ignite.thread.IgniteThread;
+import java.util.Arrays;
+
/**
* Run linear regression model over distributed matrix.
*
@@ -113,15 +114,15 @@ public class DistributedLinearRegressionWithQRTrainerExample {
Trainer<LinearRegressionModel, Matrix> trainer = new LinearRegressionQRTrainer();
System.out.println(">>> Perform the training to get the model.");
- LinearRegressionModel model = trainer.train(distributedMatrix);
- System.out.println(">>> Linear regression model: " + model);
+ LinearRegressionModel mdl = trainer.train(distributedMatrix);
+ System.out.println(">>> Linear regression model: " + mdl);
System.out.println(">>> ---------------------------------");
System.out.println(">>> | Prediction\t| Ground Truth\t|");
System.out.println(">>> ---------------------------------");
for (double[] observation : data) {
Vector inputs = new SparseDistributedVector(Arrays.copyOfRange(observation, 1, observation.length));
- double prediction = model.apply(inputs);
+ double prediction = mdl.apply(inputs);
double groundTruth = observation[0];
System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithSGDTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithSGDTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithSGDTrainerExample.java
index f3b2655..44366e1 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithSGDTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithSGDTrainerExample.java
@@ -17,20 +17,26 @@
package org.apache.ignite.examples.ml.regression.linear;
-import java.util.Arrays;
import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
-import org.apache.ignite.examples.ml.math.matrix.SparseDistributedMatrixExample;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
import org.apache.ignite.ml.regressions.linear.LinearRegressionQRTrainer;
import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
+import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
import org.apache.ignite.thread.IgniteThread;
+import javax.cache.Cache;
+import java.util.Arrays;
+import java.util.UUID;
+
/**
* Run linear regression model over distributed matrix.
*
@@ -104,28 +110,43 @@ public class DistributedLinearRegressionWithSGDTrainerExample {
// Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
// because we create ignite cache internally.
IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- SparseDistributedMatrixExample.class.getSimpleName(), () -> {
+ DistributedLinearRegressionWithSGDTrainerExample.class.getSimpleName(), () -> {
- // Create SparseDistributedMatrix, new cache will be created automagically.
- System.out.println(">>> Create new SparseDistributedMatrix inside IgniteThread.");
- SparseDistributedMatrix distributedMatrix = new SparseDistributedMatrix(data);
+ IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
System.out.println(">>> Create new linear regression trainer object.");
- Trainer<LinearRegressionModel, Matrix> trainer = new LinearRegressionSGDTrainer(100_000, 1e-12);
+ LinearRegressionSGDTrainer<?> trainer = new LinearRegressionSGDTrainer<>(new UpdatesStrategy<>(
+ new RPropUpdateCalculator(),
+ RPropParameterUpdate::sumLocal,
+ RPropParameterUpdate::avg
+ ), 100000, 10, 100, 123L);
System.out.println(">>> Perform the training to get the model.");
- LinearRegressionModel model = trainer.train(distributedMatrix);
- System.out.println(">>> Linear regression model: " + model);
+ LinearRegressionModel mdl = trainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+ (k, v) -> v[0]
+ );
+
+ System.out.println(">>> Linear regression model: " + mdl);
System.out.println(">>> ---------------------------------");
System.out.println(">>> | Prediction\t| Ground Truth\t|");
System.out.println(">>> ---------------------------------");
- for (double[] observation : data) {
- Vector inputs = new SparseDistributedVector(Arrays.copyOfRange(observation, 1, observation.length));
- double prediction = model.apply(inputs);
- double groundTruth = observation[0];
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
+
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
+
+ double prediction = mdl.apply(new DenseLocalOnHeapVector(inputs));
+
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
+ }
}
+
System.out.println(">>> ---------------------------------");
});
@@ -134,4 +155,23 @@ public class DistributedLinearRegressionWithSGDTrainerExample {
igniteThread.join();
}
}
+
+ /**
+ * Fills cache with data and returns it.
+ *
+ * @param ignite Ignite instance.
+ * @return Filled Ignite Cache.
+ */
+ private static IgniteCache<Integer, double[]> getTestCache(Ignite ignite) {
+ CacheConfiguration<Integer, double[]> cacheConfiguration = new CacheConfiguration<>();
+ cacheConfiguration.setName("TEST_" + UUID.randomUUID());
+ cacheConfiguration.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+ IgniteCache<Integer, double[]> cache = ignite.createCache(cacheConfiguration);
+
+ for (int i = 0; i < data.length; i++)
+ cache.put(i, data[i]);
+
+ return cache;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
index f8bf521..ce37112 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
@@ -17,9 +17,6 @@
package org.apache.ignite.examples.ml.svm.binary;
-import java.util.Arrays;
-import java.util.UUID;
-import javax.cache.Cache;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
@@ -27,12 +24,15 @@ import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationTrainer;
import org.apache.ignite.thread.IgniteThread;
+import javax.cache.Cache;
+import java.util.Arrays;
+import java.util.UUID;
+
/**
* Run SVM binary-class classification model over distributed dataset.
*
@@ -54,7 +54,8 @@ public class SVMBinaryClassificationExample {
SVMLinearBinaryClassificationTrainer trainer = new SVMLinearBinaryClassificationTrainer();
SVMLinearBinaryClassificationModel mdl = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, dataCache),
+ ignite,
+ dataCache,
(k, v) -> Arrays.copyOfRange(v, 1, v.length),
(k, v) -> v[0]
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java
index f8281e4..4054201 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java
@@ -17,9 +17,6 @@
package org.apache.ignite.examples.ml.svm.multiclass;
-import java.util.Arrays;
-import java.util.UUID;
-import javax.cache.Cache;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
@@ -27,14 +24,17 @@ import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.preprocessing.normalization.NormalizationPreprocessor;
import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationModel;
import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationTrainer;
import org.apache.ignite.thread.IgniteThread;
+import javax.cache.Cache;
+import java.util.Arrays;
+import java.util.UUID;
+
/**
* Run SVM multi-class classification trainer over distributed dataset to build two models:
* one with normalization and one without normalization.
@@ -57,7 +57,8 @@ public class SVMMultiClassClassificationExample {
SVMLinearMultiClassClassificationTrainer trainer = new SVMLinearMultiClassClassificationTrainer();
SVMLinearMultiClassClassificationModel mdl = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, dataCache),
+ ignite,
+ dataCache,
(k, v) -> Arrays.copyOfRange(v, 1, v.length),
(k, v) -> v[0]
);
@@ -67,14 +68,15 @@ public class SVMMultiClassClassificationExample {
NormalizationTrainer<Integer, double[]> normalizationTrainer = new NormalizationTrainer<>();
- NormalizationPreprocessor<Integer, double[]> preprocessor = normalizationTrainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, dataCache),
- (k, v) -> Arrays.copyOfRange(v, 1, v.length),
- 5
+ IgniteBiFunction<Integer, double[], double[]> preprocessor = normalizationTrainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> Arrays.copyOfRange(v, 1, v.length)
);
SVMLinearMultiClassClassificationModel mdlWithNormalization = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, dataCache),
+ ignite,
+ dataCache,
preprocessor,
(k, v) -> v[0]
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
index cef6368..1ecf460 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
@@ -17,17 +17,17 @@
package org.apache.ignite.examples.ml.tree;
-import java.util.Random;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
import org.apache.ignite.thread.IgniteThread;
+import java.util.Random;
+
/**
* Example of using distributed {@link DecisionTreeClassificationTrainer}.
*/
@@ -65,7 +65,8 @@ public class DecisionTreeClassificationTrainerExample {
// Train decision tree model.
DecisionTreeNode mdl = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+ ignite,
+ trainingSet,
(k, v) -> new double[]{v.x, v.y},
(k, v) -> v.lb
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
index 61ba5f9..19b15f3 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
@@ -22,7 +22,6 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.tree.DecisionTreeNode;
import org.apache.ignite.ml.tree.DecisionTreeRegressionTrainer;
import org.apache.ignite.thread.IgniteThread;
@@ -61,7 +60,8 @@ public class DecisionTreeRegressionTrainerExample {
// Train decision tree model.
DecisionTreeNode mdl = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+ ignite,
+ trainingSet,
(k, v) -> new double[] {v.x},
(k, v) -> v.y
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java
index f05bde8..4c34cd2 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java
@@ -58,4 +58,24 @@ public class Activators {
return Math.max(val, 0);
}
};
+
+ /**
+ * Linear unit activation function.
+ */
+ public static IgniteDifferentiableDoubleToDoubleFunction LINEAR = new IgniteDifferentiableDoubleToDoubleFunction() {
+ /** {@inheritDoc} */
+ @Override public double differential(double pnt) {
+ return 1.0;
+ }
+
+ /**
+ * Differential of linear at pnt.
+ *
+ * @param pnt Point to differentiate at.
+ * @return Differential at pnt.
+ */
+ @Override public Double apply(double pnt) {
+ return pnt;
+ }
+ };
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
index 47d2022..fe955cb 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
@@ -17,11 +17,6 @@
package org.apache.ignite.ml.nn;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import org.apache.ignite.ml.trainers.MultiLabelDatasetTrainer;
import org.apache.ignite.ml.dataset.Dataset;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.primitive.builder.context.EmptyContextBuilder;
@@ -37,17 +32,23 @@ import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
import org.apache.ignite.ml.nn.initializers.RandomInitializer;
import org.apache.ignite.ml.optimization.updatecalculators.ParameterUpdateCalculator;
+import org.apache.ignite.ml.trainers.MultiLabelDatasetTrainer;
import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
import org.apache.ignite.ml.util.Utils;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
/**
* Multilayer perceptron trainer based on partition based {@link Dataset}.
*
* @param <P> Type of model update used in this trainer.
*/
public class MLPTrainer<P extends Serializable> implements MultiLabelDatasetTrainer<MultilayerPerceptron> {
- /** Multilayer perceptron architecture that defines layers and activators. */
- private final MLPArchitecture arch;
+ /** Multilayer perceptron architecture supplier that defines layers and activators. */
+ private final IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier;
/** Loss function to be minimized during the training. */
private final IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
@@ -81,7 +82,25 @@ public class MLPTrainer<P extends Serializable> implements MultiLabelDatasetTrai
public MLPTrainer(MLPArchitecture arch, IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss,
UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy, int maxIterations, int batchSize,
int locIterations, long seed) {
- this.arch = arch;
+ this(dataset -> arch, loss, updatesStgy, maxIterations, batchSize, locIterations, seed);
+ }
+
+ /**
+ * Constructs a new instance of multilayer perceptron trainer.
+ *
+ * @param archSupplier Multilayer perceptron architecture supplier that defines layers and activators.
+ * @param loss Loss function to be minimized during the training.
+ * @param updatesStgy Update strategy that defines how to update model parameters during the training.
+ * @param maxIterations Maximal number of iterations before the training will be stopped.
+ * @param batchSize Batch size (per every partition).
+ * @param locIterations Maximal number of local iterations before synchronization.
+ * @param seed Random initializer seed.
+ */
+ public MLPTrainer(IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier,
+ IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss,
+ UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy, int maxIterations, int batchSize,
+ int locIterations, long seed) {
+ this.archSupplier = archSupplier;
this.loss = loss;
this.updatesStgy = updatesStgy;
this.maxIterations = maxIterations;
@@ -94,13 +113,14 @@ public class MLPTrainer<P extends Serializable> implements MultiLabelDatasetTrai
public <K, V> MultilayerPerceptron fit(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, double[]> lbExtractor) {
- MultilayerPerceptron mdl = new MultilayerPerceptron(arch, new RandomInitializer(seed));
- ParameterUpdateCalculator<? super MultilayerPerceptron, P> updater = updatesStgy.getUpdatesCalculator();
-
try (Dataset<EmptyContext, SimpleLabeledDatasetData> dataset = datasetBuilder.build(
new EmptyContextBuilder<>(),
new SimpleLabeledDatasetDataBuilder<>(featureExtractor, lbExtractor)
)) {
+ MLPArchitecture arch = archSupplier.apply(dataset);
+ MultilayerPerceptron mdl = new MultilayerPerceptron(arch, new RandomInitializer(seed));
+ ParameterUpdateCalculator<? super MultilayerPerceptron, P> updater = updatesStgy.getUpdatesCalculator();
+
for (int i = 0; i < maxIterations; i += locIterations) {
MultilayerPerceptron finalMdl = mdl;
@@ -163,12 +183,12 @@ public class MLPTrainer<P extends Serializable> implements MultiLabelDatasetTrai
P update = updatesStgy.allUpdatesReducer().apply(totUp);
mdl = updater.update(mdl, update);
}
+
+ return mdl;
}
catch (Exception e) {
throw new RuntimeException(e);
}
-
- return mdl;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
index f5a6bb0..1886ee5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
@@ -17,9 +17,15 @@
package org.apache.ignite.ml.preprocessing;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import java.util.Map;
+
/**
* Trainer for preprocessor.
*
@@ -34,9 +40,40 @@ public interface PreprocessingTrainer<K, V, T, R> {
*
* @param datasetBuilder Dataset builder.
* @param basePreprocessor Base preprocessor.
- * @param cols Number of columns.
* @return Preprocessor.
*/
public IgniteBiFunction<K, V, R> fit(DatasetBuilder<K, V> datasetBuilder,
- IgniteBiFunction<K, V, T> basePreprocessor, int cols);
+ IgniteBiFunction<K, V, T> basePreprocessor);
+
+ /**
+ * Fits preprocessor.
+ *
+ * @param ignite Ignite instance.
+ * @param cache Ignite cache.
+ * @param basePreprocessor Base preprocessor.
+ * @return Preprocessor.
+ */
+ public default IgniteBiFunction<K, V, R> fit(Ignite ignite, IgniteCache<K, V> cache,
+ IgniteBiFunction<K, V, T> basePreprocessor) {
+ return fit(
+ new CacheBasedDatasetBuilder<>(ignite, cache),
+ basePreprocessor
+ );
+ }
+
+ /**
+ * Fits preprocessor.
+ *
+ * @param data Data.
+ * @param parts Number of partitions.
+ * @param basePreprocessor Base preprocessor.
+ * @return Preprocessor.
+ */
+ public default IgniteBiFunction<K, V, R> fit(Map<K, V> data, int parts,
+ IgniteBiFunction<K, V, T> basePreprocessor) {
+ return fit(
+ new LocalDatasetBuilder<>(data, parts),
+ basePreprocessor
+ );
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainer.java
index 16623ba..57acbad 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainer.java
@@ -33,33 +33,48 @@ import org.apache.ignite.ml.preprocessing.PreprocessingTrainer;
public class NormalizationTrainer<K, V> implements PreprocessingTrainer<K, V, double[], double[]> {
/** {@inheritDoc} */
@Override public NormalizationPreprocessor<K, V> fit(DatasetBuilder<K, V> datasetBuilder,
- IgniteBiFunction<K, V, double[]> basePreprocessor, int cols) {
+ IgniteBiFunction<K, V, double[]> basePreprocessor) {
try (Dataset<EmptyContext, NormalizationPartitionData> dataset = datasetBuilder.build(
(upstream, upstreamSize) -> new EmptyContext(),
(upstream, upstreamSize, ctx) -> {
- double[] min = new double[cols];
- double[] max = new double[cols];
-
- for (int i = 0; i < cols; i++) {
- min[i] = Double.MAX_VALUE;
- max[i] = -Double.MAX_VALUE;
- }
+ double[] min = null;
+ double[] max = null;
while (upstream.hasNext()) {
UpstreamEntry<K, V> entity = upstream.next();
double[] row = basePreprocessor.apply(entity.getKey(), entity.getValue());
- for (int i = 0; i < cols; i++) {
+
+ if (min == null) {
+ min = new double[row.length];
+ for (int i = 0; i < min.length; i++)
+ min[i] = Double.MAX_VALUE;
+ }
+ else
+ assert min.length == row.length : "Base preprocessor must return exactly " + min.length
+ + " features";
+
+ if (max == null) {
+ max = new double[row.length];
+ for (int i = 0; i < max.length; i++)
+ max[i] = -Double.MAX_VALUE;
+ }
+ else
+ assert max.length == row.length : "Base preprocessor must return exactly " + min.length
+ + " features";
+
+ for (int i = 0; i < row.length; i++) {
if (row[i] < min[i])
min[i] = row[i];
if (row[i] > max[i])
max[i] = row[i];
}
}
+
return new NormalizationPartitionData(min, max);
}
)) {
double[][] minMax = dataset.compute(
- data -> new double[][]{ data.getMin(), data.getMax() },
+ data -> data.getMin() != null ? new double[][]{ data.getMin(), data.getMax() } : null,
(a, b) -> {
if (a == null)
return b;
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/FeatureExtractorWrapper.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/FeatureExtractorWrapper.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/FeatureExtractorWrapper.java
new file mode 100644
index 0000000..8e8f467
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/FeatureExtractorWrapper.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.regressions.linear;
+
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+
+import java.util.Arrays;
+
+/**
+ * Feature extractor wrapper that adds additional column filled by 1.
+ *
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ */
+public class FeatureExtractorWrapper<K, V> implements IgniteBiFunction<K, V, double[]> {
+ /** */
+ private static final long serialVersionUID = -2686524650955735635L;
+
+ /** Underlying feature extractor. */
+ private final IgniteBiFunction<K, V, double[]> featureExtractor;
+
+ /**
+ * Constructs a new instance of feature extractor wrapper.
+ *
+ * @param featureExtractor Underlying feature extractor.
+ */
+ FeatureExtractorWrapper(IgniteBiFunction<K, V, double[]> featureExtractor) {
+ this.featureExtractor = featureExtractor;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double[] apply(K k, V v) {
+ double[] featureRow = featureExtractor.apply(k, v);
+ double[] row = Arrays.copyOf(featureRow, featureRow.length + 1);
+
+ row[featureRow.length] = 1.0;
+
+ return row;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
index ae15f2f..9526db1 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
@@ -17,8 +17,6 @@
package org.apache.ignite.ml.regressions.linear;
-import java.util.Arrays;
-import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.math.Vector;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
@@ -27,6 +25,9 @@ import org.apache.ignite.ml.math.isolve.LinSysPartitionDataBuilderOnHeap;
import org.apache.ignite.ml.math.isolve.lsqr.AbstractLSQR;
import org.apache.ignite.ml.math.isolve.lsqr.LSQROnHeap;
import org.apache.ignite.ml.math.isolve.lsqr.LSQRResult;
+import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
+
+import java.util.Arrays;
/**
* Trainer of the linear regression model based on LSQR algorithm.
@@ -55,37 +56,4 @@ public class LinearRegressionLSQRTrainer implements SingleLabelDatasetTrainer<Li
return new LinearRegressionModel(weights, x[x.length - 1]);
}
-
- /**
- * Feature extractor wrapper that adds additional column filled by 1.
- *
- * @param <K> Type of a key in {@code upstream} data.
- * @param <V> Type of a value in {@code upstream} data.
- */
- private static class FeatureExtractorWrapper<K, V> implements IgniteBiFunction<K, V, double[]> {
- /** */
- private static final long serialVersionUID = -2686524650955735635L;
-
- /** Underlying feature extractor. */
- private final IgniteBiFunction<K, V, double[]> featureExtractor;
-
- /**
- * Constructs a new instance of feature extractor wrapper.
- *
- * @param featureExtractor Underlying feature extractor.
- */
- FeatureExtractorWrapper(IgniteBiFunction<K, V, double[]> featureExtractor) {
- this.featureExtractor = featureExtractor;
- }
-
- /** {@inheritDoc} */
- @Override public double[] apply(K k, V v) {
- double[] featureRow = featureExtractor.apply(k, v);
- double[] row = Arrays.copyOf(featureRow, featureRow.length + 1);
-
- row[featureRow.length] = 1.0;
-
- return row;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
index aad4c7a..9be3fdd 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
@@ -17,51 +17,99 @@
package org.apache.ignite.ml.regressions.linear;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.optimization.BarzilaiBorweinUpdater;
-import org.apache.ignite.ml.optimization.GradientDescent;
-import org.apache.ignite.ml.optimization.LeastSquaresGradientFunction;
-import org.apache.ignite.ml.optimization.SimpleUpdater;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.dataset.primitive.data.SimpleLabeledDatasetData;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.nn.Activators;
+import org.apache.ignite.ml.nn.MLPTrainer;
+import org.apache.ignite.ml.nn.MultilayerPerceptron;
+import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
+import org.apache.ignite.ml.optimization.LossFunctions;
+import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
+import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
+
+import java.io.Serializable;
+import java.util.Arrays;
/**
- * Linear regression trainer based on least squares loss function and gradient descent optimization algorithm.
+ * Trainer of the linear regression model based on stochastic gradient descent algorithm.
*/
-public class LinearRegressionSGDTrainer implements Trainer<LinearRegressionModel, Matrix> {
- /**
- * Gradient descent optimizer.
- */
- private final GradientDescent gradientDescent;
+public class LinearRegressionSGDTrainer<P extends Serializable> implements SingleLabelDatasetTrainer<LinearRegressionModel> {
+ /** Update strategy. */
+ private final UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy;
- /** */
- public LinearRegressionSGDTrainer(GradientDescent gradientDescent) {
- this.gradientDescent = gradientDescent;
- }
+ /** Max number of iteration. */
+ private final int maxIterations;
- /** */
- public LinearRegressionSGDTrainer(int maxIterations, double convergenceTol) {
- this.gradientDescent = new GradientDescent(new LeastSquaresGradientFunction(), new BarzilaiBorweinUpdater())
- .withMaxIterations(maxIterations)
- .withConvergenceTol(convergenceTol);
- }
+ /** Batch size. */
+ private final int batchSize;
- /** */
- public LinearRegressionSGDTrainer(int maxIterations, double convergenceTol, double learningRate) {
- this.gradientDescent = new GradientDescent(new LeastSquaresGradientFunction(), new SimpleUpdater(learningRate))
- .withMaxIterations(maxIterations)
- .withConvergenceTol(convergenceTol);
- }
+ /** Number of local iterations. */
+ private final int locIterations;
+
+ /** Seed for random generator. */
+ private final long seed;
/**
- * {@inheritDoc}
+ * Constructs a new instance of linear regression SGD trainer.
+ *
+ * @param updatesStgy Update strategy.
+ * @param maxIterations Max number of iteration.
+ * @param batchSize Batch size.
+ * @param locIterations Number of local iterations.
+ * @param seed Seed for random generator.
*/
- @Override public LinearRegressionModel train(Matrix data) {
- Vector variables = gradientDescent.optimize(data, data.likeVector(data.columnSize()));
- Vector weights = variables.viewPart(1, variables.size() - 1);
+ public LinearRegressionSGDTrainer(UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy, int maxIterations,
+ int batchSize, int locIterations, long seed) {
+ this.updatesStgy = updatesStgy;
+ this.maxIterations = maxIterations;
+ this.batchSize = batchSize;
+ this.locIterations = locIterations;
+ this.seed = seed;
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> LinearRegressionModel fit(DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier = dataset -> {
+
+ int cols = dataset.compute(data -> {
+ if (data.getFeatures() == null)
+ return null;
+ return data.getFeatures().length / data.getRows();
+ }, (a, b) -> a == null ? b : a);
+
+ MLPArchitecture architecture = new MLPArchitecture(cols);
+ architecture = architecture.withAddedLayer(1, true, Activators.LINEAR);
+
+ return architecture;
+ };
+
+ MLPTrainer<?> trainer = new MLPTrainer<>(
+ archSupplier,
+ LossFunctions.MSE,
+ updatesStgy,
+ maxIterations,
+ batchSize,
+ locIterations,
+ seed
+ );
+
+ IgniteBiFunction<K, V, double[]> lbE = new IgniteBiFunction<K, V, double[]>() {
+ @Override public double[] apply(K k, V v) {
+ return new double[]{lbExtractor.apply(k, v)};
+ }
+ };
+
+ MultilayerPerceptron mlp = trainer.fit(datasetBuilder, featureExtractor, lbE);
- double intercept = variables.get(0);
+ double[] p = mlp.parameters().getStorage().data();
- return new LinearRegressionModel(weights, intercept);
+ return new LinearRegressionModel(new DenseLocalOnHeapVector(Arrays.copyOf(p, p.length - 1)), p[p.length - 1]);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
index 8119a29..fcde3f5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
@@ -17,10 +17,16 @@
package org.apache.ignite.ml.trainers;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
import org.apache.ignite.ml.Model;
import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import java.util.Map;
+
/**
* Interface for trainers. Trainer is just a function which produces model from the data.
*
@@ -40,4 +46,44 @@ public interface DatasetTrainer<M extends Model, L> {
*/
public <K, V> M fit(DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, double[]> featureExtractor,
IgniteBiFunction<K, V, L> lbExtractor);
+
+ /**
+ * Trains model based on the specified data.
+ *
+ * @param ignite Ignite instance.
+ * @param cache Ignite cache.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return Model.
+ */
+ public default <K, V> M fit(Ignite ignite, IgniteCache<K, V> cache, IgniteBiFunction<K, V, double[]> featureExtractor,
+ IgniteBiFunction<K, V, L> lbExtractor) {
+ return fit(
+ new CacheBasedDatasetBuilder<>(ignite, cache),
+ featureExtractor,
+ lbExtractor
+ );
+ }
+
+ /**
+ * Trains model based on the specified data.
+ *
+ * @param data Data.
+ * @param parts Number of partitions.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return Model.
+ */
+ public default <K, V> M fit(Map<K, V> data, int parts, IgniteBiFunction<K, V, double[]> featureExtractor,
+ IgniteBiFunction<K, V, L> lbExtractor) {
+ return fit(
+ new LocalDatasetBuilder<>(data, parts),
+ featureExtractor,
+ lbExtractor
+ );
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
index b5a4b54..b27fcba 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
@@ -17,11 +17,7 @@
package org.apache.ignite.ml.knn;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
import org.apache.ignite.ml.knn.classification.KNNClassificationTrainer;
import org.apache.ignite.ml.knn.classification.KNNStrategy;
@@ -29,6 +25,10 @@ import org.apache.ignite.ml.math.Vector;
import org.apache.ignite.ml.math.distances.EuclideanDistance;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
/** Tests behaviour of KNNClassificationTest. */
public class KNNClassificationTest extends BaseKNNTest {
/** */
@@ -46,7 +46,8 @@ public class KNNClassificationTest extends BaseKNNTest {
KNNClassificationTrainer trainer = new KNNClassificationTrainer();
KNNClassificationModel knnMdl = trainer.fit(
- new LocalDatasetBuilder<>(data, 2),
+ data,
+ 2,
(k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
(k, v) -> v[2]
).withK(3)
@@ -74,7 +75,8 @@ public class KNNClassificationTest extends BaseKNNTest {
KNNClassificationTrainer trainer = new KNNClassificationTrainer();
KNNClassificationModel knnMdl = trainer.fit(
- new LocalDatasetBuilder<>(data, 2),
+ data,
+ 2,
(k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
(k, v) -> v[2]
).withK(1)
@@ -102,7 +104,8 @@ public class KNNClassificationTest extends BaseKNNTest {
KNNClassificationTrainer trainer = new KNNClassificationTrainer();
KNNClassificationModel knnMdl = trainer.fit(
- new LocalDatasetBuilder<>(data, 2),
+ data,
+ 2,
(k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
(k, v) -> v[2]
).withK(3)
@@ -128,7 +131,8 @@ public class KNNClassificationTest extends BaseKNNTest {
KNNClassificationTrainer trainer = new KNNClassificationTrainer();
KNNClassificationModel knnMdl = trainer.fit(
- new LocalDatasetBuilder<>(data, 2),
+ data,
+ 2,
(k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
(k, v) -> v[2]
).withK(3)
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java
index 5ca661f..038b880 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java
@@ -17,7 +17,6 @@
package org.apache.ignite.ml.nn;
-import java.io.Serializable;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
@@ -25,22 +24,18 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.Tracer;
import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
import org.apache.ignite.ml.optimization.LossFunctions;
-import org.apache.ignite.ml.optimization.updatecalculators.NesterovParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.NesterovUpdateCalculator;
-import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
-import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalculator;
+import org.apache.ignite.ml.optimization.updatecalculators.*;
import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import java.io.Serializable;
+
/**
* Tests for {@link MLPTrainer} that require to start the whole Ignite infrastructure.
*/
@@ -137,7 +132,8 @@ public class MLPTrainerIntegrationTest extends GridCommonAbstractTest {
);
MultilayerPerceptron mlp = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, xorCache),
+ ignite,
+ xorCache,
(k, v) -> new double[]{ v.x, v.y },
(k, v) -> new double[]{ v.lb}
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
index 6906424..c53f6f1 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
@@ -17,24 +17,13 @@
package org.apache.ignite.ml.nn;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
import org.apache.ignite.ml.optimization.LossFunctions;
-import org.apache.ignite.ml.optimization.updatecalculators.NesterovParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.NesterovUpdateCalculator;
-import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
-import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalculator;
+import org.apache.ignite.ml.optimization.updatecalculators.*;
import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
import org.junit.Before;
import org.junit.Test;
@@ -42,6 +31,12 @@ import org.junit.experimental.runners.Enclosed;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
/**
* Tests for {@link MLPTrainer} that don't require to start the whole Ignite infrastructure.
*/
@@ -140,7 +135,8 @@ public class MLPTrainerTest {
);
MultilayerPerceptron mlp = trainer.fit(
- new LocalDatasetBuilder<>(xorData, parts),
+ xorData,
+ parts,
(k, v) -> v[0],
(k, v) -> v[1]
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java
index c787a47..a64af9b 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java
@@ -17,13 +17,11 @@
package org.apache.ignite.ml.nn.performance;
-import java.io.IOException;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.VectorUtils;
import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
@@ -38,6 +36,8 @@ import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
import org.apache.ignite.ml.util.MnistUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import java.io.IOException;
+
/**
* Tests {@link MLPTrainer} on the MNIST dataset that require to start the whole Ignite infrastructure.
*/
@@ -104,7 +104,8 @@ public class MLPTrainerMnistIntegrationTest extends GridCommonAbstractTest {
System.out.println("Start training...");
long start = System.currentTimeMillis();
MultilayerPerceptron mdl = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+ ignite,
+ trainingSet,
(k, v) -> v.getPixels(),
(k, v) -> VectorUtils.num2Vec(v.getLabel(), 10).getStorage().data()
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java
index 354af2c..d966484 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java
@@ -17,10 +17,6 @@
package org.apache.ignite.ml.nn.performance;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.VectorUtils;
import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
@@ -35,6 +31,10 @@ import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
import org.apache.ignite.ml.util.MnistUtils;
import org.junit.Test;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
import static org.junit.Assert.assertTrue;
/**
@@ -74,7 +74,8 @@ public class MLPTrainerMnistTest {
System.out.println("Start training...");
long start = System.currentTimeMillis();
MultilayerPerceptron mdl = trainer.fit(
- new LocalDatasetBuilder<>(trainingSet, 1),
+ trainingSet,
+ 1,
(k, v) -> v.getPixels(),
(k, v) -> VectorUtils.num2Vec(v.getLabel(), 10).getStorage().data()
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainerTest.java
index 1548253..e7a0d47 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainerTest.java
@@ -17,15 +17,16 @@
package org.apache.ignite.ml.preprocessing.normalization;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
import static org.junit.Assert.assertArrayEquals;
/**
@@ -66,8 +67,7 @@ public class NormalizationTrainerTest {
NormalizationPreprocessor<Integer, double[]> preprocessor = standardizationTrainer.fit(
datasetBuilder,
- (k, v) -> v,
- 3
+ (k, v) -> v
);
assertArrayEquals(new double[] {0, 4, 1}, preprocessor.getMin(), 1e-8);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java
index 82b3a1b..b3c9368 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java
@@ -17,14 +17,7 @@
package org.apache.ignite.ml.regressions;
-import org.apache.ignite.ml.regressions.linear.BlockDistributedLinearRegressionQRTrainerTest;
-import org.apache.ignite.ml.regressions.linear.BlockDistributedLinearRegressionSGDTrainerTest;
-import org.apache.ignite.ml.regressions.linear.DistributedLinearRegressionQRTrainerTest;
-import org.apache.ignite.ml.regressions.linear.DistributedLinearRegressionSGDTrainerTest;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionLSQRTrainerTest;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionModelTest;
-import org.apache.ignite.ml.regressions.linear.LocalLinearRegressionQRTrainerTest;
-import org.apache.ignite.ml.regressions.linear.LocalLinearRegressionSGDTrainerTest;
+import org.apache.ignite.ml.regressions.linear.*;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -35,12 +28,10 @@ import org.junit.runners.Suite;
@Suite.SuiteClasses({
LinearRegressionModelTest.class,
LocalLinearRegressionQRTrainerTest.class,
- LocalLinearRegressionSGDTrainerTest.class,
DistributedLinearRegressionQRTrainerTest.class,
- DistributedLinearRegressionSGDTrainerTest.class,
BlockDistributedLinearRegressionQRTrainerTest.class,
- BlockDistributedLinearRegressionSGDTrainerTest.class,
- LinearRegressionLSQRTrainerTest.class
+ LinearRegressionLSQRTrainerTest.class,
+ LinearRegressionSGDTrainerTest.class
})
public class RegressionsTestSuite {
// No-op.
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/BlockDistributedLinearRegressionSGDTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/BlockDistributedLinearRegressionSGDTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/BlockDistributedLinearRegressionSGDTrainerTest.java
deleted file mode 100644
index 58037e2..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/BlockDistributedLinearRegressionSGDTrainerTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.SparseBlockDistributedVector;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link SparseBlockDistributedMatrix}.
- */
-public class BlockDistributedLinearRegressionSGDTrainerTest extends GridAwareAbstractLinearRegressionTrainerTest {
- /** */
- public BlockDistributedLinearRegressionSGDTrainerTest() {
- super(
- new LinearRegressionSGDTrainer(100_000, 1e-12),
- SparseBlockDistributedMatrix::new,
- SparseBlockDistributedVector::new,
- 1e-2);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/DistributedLinearRegressionSGDTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/DistributedLinearRegressionSGDTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/DistributedLinearRegressionSGDTrainerTest.java
deleted file mode 100644
index 71d3b3b..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/DistributedLinearRegressionSGDTrainerTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link SparseDistributedMatrix}.
- */
-public class DistributedLinearRegressionSGDTrainerTest extends GridAwareAbstractLinearRegressionTrainerTest {
- /** */
- public DistributedLinearRegressionSGDTrainerTest() {
- super(
- new LinearRegressionSGDTrainer(100_000, 1e-12),
- SparseDistributedMatrix::new,
- SparseDistributedVector::new,
- 1e-2);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java
index 1a60b80..9b75bd4 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java
@@ -26,6 +26,9 @@ import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Test;
+/**
+ * Grid aware abstract linear regression trainer test.
+ */
public abstract class GridAwareAbstractLinearRegressionTrainerTest extends GridCommonAbstractTest {
/** Number of nodes in grid */
private static final int NODE_COUNT = 3;
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java
index e3f60ec..2414236 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java
@@ -17,14 +17,14 @@
package org.apache.ignite.ml.regressions.linear;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -72,7 +72,8 @@ public class LinearRegressionLSQRTrainerTest {
LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
LinearRegressionModel mdl = trainer.fit(
- new LocalDatasetBuilder<>(data, parts),
+ data,
+ parts,
(k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
(k, v) -> v[4]
);
@@ -110,7 +111,8 @@ public class LinearRegressionLSQRTrainerTest {
LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
LinearRegressionModel mdl = trainer.fit(
- new LocalDatasetBuilder<>(data, parts),
+ data,
+ parts,
(k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
(k, v) -> v[coef.length]
);
[20/54] [abbrv] ignite git commit: IGNITE-8025 Future must fail if
assertion error has been thrown in the worker thread
Posted by ag...@apache.org.
IGNITE-8025 Future must fail if assertion error has been thrown in the worker thread
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/05d7092e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/05d7092e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/05d7092e
Branch: refs/heads/ignite-6083
Commit: 05d7092e255295cf6b78b9d9082ad9eab5e8a246
Parents: 647620b
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Tue Apr 10 16:22:28 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Apr 10 16:23:06 2018 +0300
----------------------------------------------------------------------
.../test/java/org/apache/ignite/testframework/GridTestUtils.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d7092e/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index 4e9a7c2..e6c6657 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -814,7 +814,7 @@ public final class GridTestUtils {
catch (IgniteFutureCancelledCheckedException e) {
resFut.onCancelled();
}
- catch (IgniteCheckedException e) {
+ catch (Throwable e) {
resFut.onDone(e);
}
});
[05/54] [abbrv] ignite git commit: IGNITE-7222 Added ZooKeeper
discovery SPI
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
new file mode 100644
index 0000000..21703c6
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
@@ -0,0 +1,1219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Op;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Zookeeper Client.
+ */
+public class ZookeeperClient implements Watcher {
+ /** */
+ private static final long RETRY_TIMEOUT =
+ IgniteSystemProperties.getLong("IGNITE_ZOOKEEPER_DISCOVERY_RETRY_TIMEOUT", 2000);
+
+ /** */
+ private static final int MAX_RETRY_COUNT =
+ IgniteSystemProperties.getInteger("IGNITE_ZOOKEEPER_DISCOVERY_MAX_RETRY_COUNT", 10);
+
+ /** */
+ private final AtomicInteger retryCount = new AtomicInteger();
+
+ /** */
+ private static final int MAX_REQ_SIZE = 1048528;
+
+ /** */
+ private static final List<ACL> ZK_ACL = ZooDefs.Ids.OPEN_ACL_UNSAFE;
+
+ /** */
+ private static final byte[] EMPTY_BYTES = {};
+
+ /** */
+ private final ZooKeeper zk;
+
+ /** */
+ private final IgniteLogger log;
+
+ /** */
+ private ConnectionState state = ConnectionState.Disconnected;
+
+ /** */
+ private long connLossTimeout;
+
+ /** */
+ private volatile long connStartTime;
+
+ /** */
+ private final Object stateMux = new Object();
+
+ /** */
+ private final IgniteRunnable connLostC;
+
+ /** */
+ private final Timer connTimer;
+
+ /** */
+ private final ArrayDeque<ZkAsyncOperation> retryQ = new ArrayDeque<>();
+
+ /** */
+ private volatile boolean closing;
+
+ /**
+ * @param log Logger.
+ * @param connectString ZK connection string.
+ * @param sesTimeout ZK session timeout.
+ * @param connLostC Lost connection callback.
+ * @throws Exception If failed.
+ */
+ ZookeeperClient(IgniteLogger log, String connectString, int sesTimeout, IgniteRunnable connLostC) throws Exception {
+ this(null, log, connectString, sesTimeout, connLostC);
+ }
+
+ /**
+ * @param igniteInstanceName Ignite instance name.
+ * @param log Logger.
+ * @param connectString ZK connection string.
+ * @param sesTimeout ZK session timeout.
+ * @param connLostC Lost connection callback.
+ * @throws Exception If failed.
+ */
+ ZookeeperClient(String igniteInstanceName,
+ IgniteLogger log,
+ String connectString,
+ int sesTimeout,
+ IgniteRunnable connLostC)
+ throws Exception
+ {
+ this.log = log.getLogger(getClass());
+ this.connLostC = connLostC;
+
+ connLossTimeout = sesTimeout;
+
+ long connStartTime = this.connStartTime = System.currentTimeMillis();
+
+ connTimer = new Timer("zk-client-timer-" + igniteInstanceName);
+
+ String threadName = Thread.currentThread().getName();
+
+ // ZK generates internal threads' names using current thread name.
+ Thread.currentThread().setName("zk-" + igniteInstanceName);
+
+ try {
+ zk = new ZooKeeper(connectString, sesTimeout, this);
+ }
+ finally {
+ Thread.currentThread().setName(threadName);
+ }
+
+ synchronized (stateMux) {
+ if (connStartTime == this.connStartTime && state == ConnectionState.Disconnected)
+ scheduleConnectionCheck();
+ }
+ }
+
+ /**
+ * @return Zookeeper client.
+ */
+ ZooKeeper zk() {
+ return zk;
+ }
+
+ /**
+ * @return {@code True} if connected to ZooKeeper.
+ */
+ boolean connected() {
+ synchronized (stateMux) {
+ return state == ConnectionState.Connected;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void process(WatchedEvent evt) {
+ if (closing)
+ return;
+
+ if (evt.getType() == Event.EventType.None) {
+ ConnectionState newState;
+
+ synchronized (stateMux) {
+ if (state == ConnectionState.Lost) {
+ U.warn(log, "Received event after connection was lost [evtState=" + evt.getState() + "]");
+
+ return;
+ }
+
+ if (!zk.getState().isAlive())
+ return;
+
+ Event.KeeperState zkState = evt.getState();
+
+ switch (zkState) {
+ case SaslAuthenticated:
+ return; // No-op.
+
+ case AuthFailed:
+ newState = state;
+
+ break;
+
+ case Disconnected:
+ newState = ConnectionState.Disconnected;
+
+ break;
+
+ case SyncConnected:
+ newState = ConnectionState.Connected;
+
+ break;
+
+ case Expired:
+ U.warn(log, "Session expired, changing state to Lost");
+
+ newState = ConnectionState.Lost;
+
+ break;
+
+ default:
+ U.error(log, "Unexpected state for ZooKeeper client, close connection: " + zkState);
+
+ newState = ConnectionState.Lost;
+ }
+
+ if (newState != state) {
+ if (log.isInfoEnabled())
+ log.info("ZooKeeper client state changed [prevState=" + state + ", newState=" + newState + ']');
+
+ state = newState;
+
+ if (newState == ConnectionState.Disconnected) {
+ connStartTime = System.currentTimeMillis();
+
+ scheduleConnectionCheck();
+ }
+ else if (newState == ConnectionState.Connected) {
+ retryCount.set(0);
+
+ stateMux.notifyAll();
+ }
+ else
+ assert state == ConnectionState.Lost : state;
+ }
+ else
+ return;
+ }
+
+ if (newState == ConnectionState.Lost) {
+ closeClient();
+
+ notifyConnectionLost();
+ }
+ else if (newState == ConnectionState.Connected) {
+ for (ZkAsyncOperation op : retryQ)
+ op.execute();
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private void notifyConnectionLost() {
+ if (!closing && state == ConnectionState.Lost && connLostC != null)
+ connLostC.run();
+
+ connTimer.cancel();
+ }
+
+ /**
+ * @param path Path.
+ * @return {@code True} if node exists.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ boolean exists(String path) throws ZookeeperClientFailedException, InterruptedException {
+ for (;;) {
+ long connStartTime = this.connStartTime;
+
+ try {
+ return zk.exists(path, false) != null;
+ }
+ catch (Exception e) {
+ onZookeeperError(connStartTime, e);
+ }
+ }
+ }
+
+ /**
+ *
+ * @param paths Paths to create.
+ * @param createMode Create mode.
+ * @throws KeeperException.NodeExistsException If at least one of target node already exists.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ void createAll(List<String> paths, CreateMode createMode)
+ throws ZookeeperClientFailedException, InterruptedException, KeeperException.NodeExistsException
+ {
+ // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8188
+ List<Op> ops = new ArrayList<>(paths.size());
+
+ for (String path : paths)
+ ops.add(Op.create(path, EMPTY_BYTES, ZK_ACL, createMode));
+
+ for (;;) {
+ long connStartTime = this.connStartTime;
+
+ try {
+ zk.multi(ops);
+
+ return;
+ }
+ catch (KeeperException.NodeExistsException e) {
+ throw e;
+ }
+ catch (Exception e) {
+ onZookeeperError(connStartTime, e);
+ }
+ }
+ }
+
+ /**
+ * @param path Path.
+ * @param data Data.
+ * @param overhead Extra overhead.
+ * @return {@code True} If data size exceeds max request size and should be splitted into multiple parts.
+ */
+ boolean needSplitNodeData(String path, byte[] data, int overhead) {
+ return requestOverhead(path) + data.length + overhead > MAX_REQ_SIZE;
+ }
+
+ /**
+ * @param path Path.
+ * @param data Data.
+ * @param overhead Extra overhead.
+ * @return Splitted data.
+ */
+ List<byte[]> splitNodeData(String path, byte[] data, int overhead) {
+ int partSize = MAX_REQ_SIZE - requestOverhead(path) - overhead;
+
+ int partCnt = data.length / partSize;
+
+ if (data.length % partSize != 0)
+ partCnt++;
+
+ assert partCnt > 1 : "Do not need split";
+
+ List<byte[]> parts = new ArrayList<>(partCnt);
+
+ int remaining = data.length;
+
+ for (int i = 0; i < partCnt; i++) {
+ int partSize0 = Math.min(remaining, partSize);
+
+ byte[] part = new byte[partSize0];
+
+ System.arraycopy(data, i * partSize, part, 0, part.length);
+
+ remaining -= partSize0;
+
+ parts.add(part);
+ }
+
+ assert remaining == 0 : remaining;
+
+ return parts;
+ }
+
+ /**
+ * TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8187
+ * @param path Request path.
+ * @return Marshalled request overhead.
+ */
+ private int requestOverhead(String path) {
+ return path.length();
+ }
+
+ /**
+ * @param path Path.
+ * @param data Data.
+ * @param createMode Create mode.
+ * @return Created path.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ String createIfNeeded(String path, byte[] data, CreateMode createMode)
+ throws ZookeeperClientFailedException, InterruptedException
+ {
+ assert !createMode.isSequential() : createMode;
+
+ if (data == null)
+ data = EMPTY_BYTES;
+
+ for (;;) {
+ long connStartTime = this.connStartTime;
+
+ try {
+ return zk.create(path, data, ZK_ACL, createMode);
+ }
+ catch (KeeperException.NodeExistsException e) {
+ if (log.isDebugEnabled())
+ log.debug("Node already exists: " + path);
+
+ return path;
+ }
+ catch (Exception e) {
+ onZookeeperError(connStartTime, e);
+ }
+ }
+ }
+
+ /**
+ * @param checkPrefix Unique prefix to check in case of retry.
+ * @param parentPath Parent node path.
+ * @param path Node to create.
+ * @param data Node data.
+ * @param createMode Create mode.
+ * @return Create path.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ String createSequential(String checkPrefix, String parentPath, String path, byte[] data, CreateMode createMode)
+ throws ZookeeperClientFailedException, InterruptedException
+ {
+ assert createMode.isSequential() : createMode;
+
+ if (data == null)
+ data = EMPTY_BYTES;
+
+ boolean first = true;
+
+ for (;;) {
+ long connStartTime = this.connStartTime;
+
+ try {
+ if (!first) {
+ List<String> children = zk.getChildren(parentPath, false);
+
+ for (int i = 0; i < children.size(); i++) {
+ String child = children.get(i);
+
+ if (children.get(i).startsWith(checkPrefix)) {
+ String resPath = parentPath + "/" + child;
+
+ if (log.isDebugEnabled())
+ log.debug("Check before retry, node already created: " + resPath);
+
+ return resPath;
+ }
+ }
+ }
+
+ return zk.create(path, data, ZK_ACL, createMode);
+ }
+ catch (KeeperException.NodeExistsException e) {
+ assert !createMode.isSequential() : createMode;
+
+ if (log.isDebugEnabled())
+ log.debug("Node already exists: " + path);
+
+ return path;
+ }
+ catch (Exception e) {
+ onZookeeperError(connStartTime, e);
+ }
+
+ first = false;
+ }
+ }
+
+ /**
+ * @param path Path.
+ * @return Children nodes.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ List<String> getChildren(String path)
+ throws ZookeeperClientFailedException, InterruptedException
+ {
+ for (;;) {
+ long connStartTime = this.connStartTime;
+
+ try {
+ return zk.getChildren(path, false);
+ }
+ catch (Exception e) {
+ onZookeeperError(connStartTime, e);
+ }
+ }
+ }
+
+ /**
+ * @param path Path.
+ * @throws InterruptedException If interrupted.
+ * @throws KeeperException In case of error.
+ * @return {@code True} if given path exists.
+ */
+ boolean existsNoRetry(String path) throws InterruptedException, KeeperException {
+ return zk.exists(path, false) != null;
+ }
+
+ /**
+ * @param path Path.
+ * @param ver Expected version.
+ * @throws InterruptedException If interrupted.
+ * @throws KeeperException In case of error.
+ */
+ void deleteIfExistsNoRetry(String path, int ver) throws InterruptedException, KeeperException {
+ try {
+ zk.delete(path, ver);
+ }
+ catch (KeeperException.NoNodeException e) {
+ // No-op if znode does not exist.
+ }
+ }
+
+ /**
+ * @param path Path.
+ * @param ver Version.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ void deleteIfExists(String path, int ver)
+ throws ZookeeperClientFailedException, InterruptedException
+ {
+ try {
+ delete(path, ver);
+ }
+ catch (KeeperException.NoNodeException e) {
+ // No-op if znode does not exist.
+ }
+ }
+
+ /**
+ * @param parent Parent path.
+ * @param paths Children paths.
+ * @param ver Version.
+ * @throws KeeperException.NoNodeException If at least one of nodes does not exist.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ void deleteAll(@Nullable String parent, List<String> paths, int ver)
+ throws KeeperException.NoNodeException, ZookeeperClientFailedException, InterruptedException
+ {
+ if (paths.isEmpty())
+ return;
+
+ // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8188
+ List<Op> ops = new ArrayList<>(paths.size());
+
+ for (String path : paths) {
+ String path0 = parent != null ? parent + "/" + path : path;
+
+ ops.add(Op.delete(path0, ver));
+ }
+
+ for (;;) {
+ long connStartTime = this.connStartTime;
+
+ try {
+ zk.multi(ops);
+
+ return;
+ }
+ catch (KeeperException.NoNodeException e) {
+ throw e;
+ }
+ catch (Exception e) {
+ onZookeeperError(connStartTime, e);
+ }
+ }
+ }
+
+ /**
+ * @param path Path.
+ * @param ver Version.
+ * @throws KeeperException.NoNodeException If target node does not exist.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ private void delete(String path, int ver)
+ throws KeeperException.NoNodeException, ZookeeperClientFailedException, InterruptedException
+ {
+ for (;;) {
+ long connStartTime = this.connStartTime;
+
+ try {
+ zk.delete(path, ver);
+
+ return;
+ }
+ catch (KeeperException.NoNodeException e) {
+ throw e;
+ }
+ catch (Exception e) {
+ onZookeeperError(connStartTime, e);
+ }
+ }
+ }
+
+ /**
+ * @param path Path.
+ * @param data Data.
+ * @param ver Version.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ * @throws KeeperException.NoNodeException If node does not exist.
+ * @throws KeeperException.BadVersionException If version does not match.
+ */
+ void setData(String path, byte[] data, int ver)
+ throws ZookeeperClientFailedException, InterruptedException, KeeperException.NoNodeException,
+ KeeperException.BadVersionException
+ {
+ if (data == null)
+ data = EMPTY_BYTES;
+
+ for (;;) {
+ long connStartTime = this.connStartTime;
+
+ try {
+ zk.setData(path, data, ver);
+
+ return;
+ }
+ catch (KeeperException.BadVersionException | KeeperException.NoNodeException e) {
+ throw e;
+ }
+ catch (Exception e) {
+ onZookeeperError(connStartTime, e);
+ }
+ }
+ }
+
+ /**
+ * @param path Path.
+ * @param stat Optional {@link Stat} instance to return znode state.
+ * @return Data.
+ * @throws KeeperException.NoNodeException If target node does not exist.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ byte[] getData(String path, @Nullable Stat stat)
+ throws KeeperException.NoNodeException, ZookeeperClientFailedException, InterruptedException {
+ for (;;) {
+ long connStartTime = this.connStartTime;
+
+ try {
+ return zk.getData(path, false, stat);
+ }
+ catch (KeeperException.NoNodeException e) {
+ throw e;
+ }
+ catch (Exception e) {
+ onZookeeperError(connStartTime, e);
+ }
+ }
+ }
+
+ /**
+ * @param path Path.
+ * @return Data.
+ * @throws KeeperException.NoNodeException If target node does not exist.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ byte[] getData(String path)
+ throws KeeperException.NoNodeException, ZookeeperClientFailedException, InterruptedException
+ {
+ return getData(path, null);
+ }
+
+ /**
+ * @param path Path.
+ */
+ void deleteIfExistsAsync(String path) {
+ new DeleteIfExistsOperation(path).execute();
+ }
+
+ /**
+ * @param path Path.
+ * @param watcher Watcher.
+ * @param cb Callback.
+ */
+ void existsAsync(String path, Watcher watcher, AsyncCallback.StatCallback cb) {
+ ExistsOperation op = new ExistsOperation(path, watcher, cb);
+
+ zk.exists(path, watcher, new StatCallbackWrapper(op), null);
+ }
+
+ /**
+ * @param path Path.
+ * @param watcher Watcher.
+ * @param cb Callback.
+ */
+ void getChildrenAsync(String path, Watcher watcher, AsyncCallback.Children2Callback cb) {
+ GetChildrenOperation op = new GetChildrenOperation(path, watcher, cb);
+
+ zk.getChildren(path, watcher, new ChildrenCallbackWrapper(op), null);
+ }
+
+ /**
+ * @param path Path.
+ * @param watcher Watcher.
+ * @param cb Callback.
+ */
+ void getDataAsync(String path, Watcher watcher, AsyncCallback.DataCallback cb) {
+ GetDataOperation op = new GetDataOperation(path, watcher, cb);
+
+ zk.getData(path, watcher, new DataCallbackWrapper(op), null);
+ }
+
+ /**
+ * @param path Path.
+ * @param data Data.
+ * @param createMode Create mode.
+ * @param cb Callback.
+ */
+ private void createAsync(String path, byte[] data, CreateMode createMode, AsyncCallback.StringCallback cb) {
+ if (data == null)
+ data = EMPTY_BYTES;
+
+ CreateOperation op = new CreateOperation(path, data, createMode, cb);
+
+ zk.create(path, data, ZK_ACL, createMode, new CreateCallbackWrapper(op), null);
+ }
+
+ /**
+ *
+ */
+ void onCloseStart() {
+ closing = true;
+
+ synchronized (stateMux) {
+ stateMux.notifyAll();
+ }
+ }
+
+ /**
+ *
+ */
+ public void close() {
+ closeClient();
+ }
+
+ /**
+ * @param prevConnStartTime Time when connection was established.
+ * @param e Error.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ private void onZookeeperError(long prevConnStartTime, Exception e)
+ throws ZookeeperClientFailedException, InterruptedException
+ {
+ ZookeeperClientFailedException err = null;
+
+ synchronized (stateMux) {
+ if (closing)
+ throw new ZookeeperClientFailedException("ZooKeeper client is closed.");
+
+ U.warn(log, "Failed to execute ZooKeeper operation [err=" + e + ", state=" + state + ']');
+
+ if (state == ConnectionState.Lost) {
+ U.error(log, "Operation failed with unexpected error, connection lost: " + e, e);
+
+ throw new ZookeeperClientFailedException(e);
+ }
+
+ boolean retry = (e instanceof KeeperException) && needRetry(((KeeperException)e).code().intValue());
+
+ if (retry) {
+ long remainingTime;
+
+ if (state == ConnectionState.Connected && connStartTime == prevConnStartTime) {
+ state = ConnectionState.Disconnected;
+
+ connStartTime = System.currentTimeMillis();
+
+ remainingTime = connLossTimeout;
+ }
+ else {
+ assert connStartTime != 0;
+
+ assert state == ConnectionState.Disconnected : state;
+
+ remainingTime = connLossTimeout - (System.currentTimeMillis() - connStartTime);
+
+ if (remainingTime <= 0) {
+ state = ConnectionState.Lost;
+
+ U.warn(log, "Failed to establish ZooKeeper connection, close client " +
+ "[timeout=" + connLossTimeout + ']');
+
+ err = new ZookeeperClientFailedException(e);
+ }
+ }
+
+ if (err == null) {
+ U.warn(log, "ZooKeeper operation failed, will retry [err=" + e +
+ ", retryTimeout=" + RETRY_TIMEOUT +
+ ", connLossTimeout=" + connLossTimeout +
+ ", path=" + ((KeeperException)e).getPath() +
+ ", remainingWaitTime=" + remainingTime + ']');
+
+ stateMux.wait(RETRY_TIMEOUT);
+
+ if (closing)
+ throw new ZookeeperClientFailedException("ZooKeeper client is closed.");
+ }
+ }
+ else {
+ U.error(log, "Operation failed with unexpected error, close ZooKeeper client: " + e, e);
+
+ state = ConnectionState.Lost;
+
+ err = new ZookeeperClientFailedException(e);
+ }
+ }
+
+ if (err != null) {
+ closeClient();
+
+ notifyConnectionLost();
+
+ throw err;
+ }
+ }
+
+ /**
+ * @param code Zookeeper error code.
+ * @return {@code True} if can retry operation.
+ */
+ private boolean needRetry(int code) {
+ boolean retryByErrorCode = code == KeeperException.Code.CONNECTIONLOSS.intValue() ||
+ code == KeeperException.Code.SESSIONMOVED.intValue() ||
+ code == KeeperException.Code.OPERATIONTIMEOUT.intValue();
+
+ if (retryByErrorCode) {
+ if (MAX_RETRY_COUNT <= 0 || retryCount.incrementAndGet() < MAX_RETRY_COUNT)
+ return true;
+ else
+ return false;
+ }
+ else
+ return false;
+ }
+
+ /**
+ *
+ */
+ private void closeClient() {
+ try {
+ zk.close();
+ }
+ catch (Exception closeErr) {
+ U.warn(log, "Failed to close ZooKeeper client: " + closeErr, closeErr);
+ }
+
+ connTimer.cancel();
+ }
+
+ /**
+ *
+ */
+ private void scheduleConnectionCheck() {
+ assert state == ConnectionState.Disconnected : state;
+
+ connTimer.schedule(new ConnectionTimeoutTask(connStartTime), connLossTimeout);
+ }
+
+ /**
+ *
+ */
+ interface ZkAsyncOperation {
+ /**
+ *
+ */
+ void execute();
+ }
+
+ /**
+ *
+ */
+ class GetChildrenOperation implements ZkAsyncOperation {
+ /** */
+ private final String path;
+
+ /** */
+ private final Watcher watcher;
+
+ /** */
+ private final AsyncCallback.Children2Callback cb;
+
+ /**
+ * @param path Path.
+ * @param watcher Watcher.
+ * @param cb Callback.
+ */
+ GetChildrenOperation(String path, Watcher watcher, AsyncCallback.Children2Callback cb) {
+ this.path = path;
+ this.watcher = watcher;
+ this.cb = cb;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void execute() {
+ getChildrenAsync(path, watcher, cb);
+ }
+ }
+
+ /**
+ *
+ */
+ class GetDataOperation implements ZkAsyncOperation {
+ /** */
+ private final String path;
+
+ /** */
+ private final Watcher watcher;
+
+ /** */
+ private final AsyncCallback.DataCallback cb;
+
+ /**
+ * @param path Path.
+ * @param watcher Watcher.
+ * @param cb Callback.
+ */
+ GetDataOperation(String path, Watcher watcher, AsyncCallback.DataCallback cb) {
+ this.path = path;
+ this.watcher = watcher;
+ this.cb = cb;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void execute() {
+ getDataAsync(path, watcher, cb);
+ }
+ }
+
+ /**
+ *
+ */
+ class ExistsOperation implements ZkAsyncOperation {
+ /** */
+ private final String path;
+
+ /** */
+ private final Watcher watcher;
+
+ /** */
+ private final AsyncCallback.StatCallback cb;
+
+ /**
+ * @param path Path.
+ * @param watcher Watcher.
+ * @param cb Callback.
+ */
+ ExistsOperation(String path, Watcher watcher, AsyncCallback.StatCallback cb) {
+ this.path = path;
+ this.watcher = watcher;
+ this.cb = cb;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void execute() {
+ existsAsync(path, watcher, cb);
+ }
+ }
+
+ /**
+ *
+ */
+ class CreateOperation implements ZkAsyncOperation {
+ /** */
+ private final String path;
+
+ /** */
+ private final byte[] data;
+
+ /** */
+ private final CreateMode createMode;
+
+ /** */
+ private final AsyncCallback.StringCallback cb;
+
+ /**
+ * @param path path.
+ * @param data Data.
+ * @param createMode Create mode.
+ * @param cb Callback.
+ */
+ CreateOperation(String path, byte[] data, CreateMode createMode, AsyncCallback.StringCallback cb) {
+ this.path = path;
+ this.data = data;
+ this.createMode = createMode;
+ this.cb = cb;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void execute() {
+ createAsync(path, data, createMode, cb);
+ }
+ }
+
+ /**
+ *
+ */
+ class DeleteIfExistsOperation implements AsyncCallback.VoidCallback, ZkAsyncOperation {
+ /** */
+ private final String path;
+
+ /**
+ * @param path Path.
+ */
+ DeleteIfExistsOperation(String path) {
+ this.path = path;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void execute() {
+ zk.delete(path, -1, this, null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void processResult(int rc, String path, Object ctx) {
+ if (closing)
+ return;
+
+ if (rc == KeeperException.Code.NONODE.intValue())
+ return;
+
+ if (needRetry(rc)) {
+ U.warn(log, "Failed to execute async operation, connection lost. Will retry after connection restore [" +
+ "path=" + path + ']');
+
+ retryQ.add(this);
+ }
+ else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue())
+ U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']');
+ else
+ assert rc == 0 : KeeperException.Code.get(rc);
+ }
+ }
+
+ /**
+ *
+ */
+ class CreateCallbackWrapper implements AsyncCallback.StringCallback {
+ /** */
+ final CreateOperation op;
+
+ /**
+ * @param op Operation.
+ */
+ CreateCallbackWrapper(CreateOperation op) {
+ this.op = op;
+ }
+
+ @Override public void processResult(int rc, String path, Object ctx, String name) {
+ if (closing)
+ return;
+
+ if (rc == KeeperException.Code.NODEEXISTS.intValue())
+ return;
+
+ if (needRetry(rc)) {
+ U.warn(log, "Failed to execute async operation, connection lost. Will retry after connection restore [path=" + path + ']');
+
+ retryQ.add(op);
+ }
+ else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue())
+ U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']');
+ else {
+ if (op.cb != null)
+ op.cb.processResult(rc, path, ctx, name);
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ class ChildrenCallbackWrapper implements AsyncCallback.Children2Callback {
+ /** */
+ private final GetChildrenOperation op;
+
+ /**
+ * @param op Operation.
+ */
+ private ChildrenCallbackWrapper(GetChildrenOperation op) {
+ this.op = op;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+ if (closing)
+ return;
+
+ if (needRetry(rc)) {
+ U.warn(log, "Failed to execute async operation, connection lost. Will retry after connection restore [path=" + path + ']');
+
+ retryQ.add(op);
+ }
+ else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue())
+ U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']');
+ else
+ op.cb.processResult(rc, path, ctx, children, stat);
+ }
+ }
+
+ /**
+ *
+ */
+ class DataCallbackWrapper implements AsyncCallback.DataCallback {
+ /** */
+ private final GetDataOperation op;
+
+ /**
+ * @param op Operation.
+ */
+ private DataCallbackWrapper(GetDataOperation op) {
+ this.op = op;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) {
+ if (closing)
+ return;
+
+ if (needRetry(rc)) {
+ U.warn(log, "Failed to execute async operation, connection lost. Will retry after connection restore [path=" + path + ']');
+
+ retryQ.add(op);
+ }
+ else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue())
+ U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']');
+ else
+ op.cb.processResult(rc, path, ctx, data, stat);
+ }
+ }
+
+ /**
+ *
+ */
+ class StatCallbackWrapper implements AsyncCallback.StatCallback {
+ /** */
+ private final ExistsOperation op;
+
+ /**
+ * @param op Operation.
+ */
+ private StatCallbackWrapper(ExistsOperation op) {
+ this.op = op;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void processResult(int rc, String path, Object ctx, Stat stat) {
+ if (closing)
+ return;
+
+ if (needRetry(rc)) {
+ U.warn(log, "Failed to execute async operation, connection lost. Will retry after connection restore [path=" + path + ']');
+
+ retryQ.add(op);
+ }
+ else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue())
+ U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']');
+ else
+ op.cb.processResult(rc, path, ctx, stat);
+ }
+ }
+
+ /**
+ *
+ */
+ private class ConnectionTimeoutTask extends TimerTask {
+ /** */
+ private final long connectStartTime;
+
+ /**
+ * @param connectStartTime Time was connection started.
+ */
+ ConnectionTimeoutTask(long connectStartTime) {
+ this.connectStartTime = connectStartTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ boolean connLoss = false;
+
+ synchronized (stateMux) {
+ if (closing)
+ return;
+
+ if (state == ConnectionState.Disconnected &&
+ ZookeeperClient.this.connStartTime == connectStartTime) {
+
+ state = ConnectionState.Lost;
+
+ U.warn(log, "Failed to establish ZooKeeper connection, close client " +
+ "[timeout=" + connLossTimeout + ']');
+
+ connLoss = true;
+ }
+ }
+
+ if (connLoss) {
+ closeClient();
+
+ notifyConnectionLost();
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private enum ConnectionState {
+ /** */
+ Connected,
+ /** */
+ Disconnected,
+ /** */
+ Lost
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientFailedException.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientFailedException.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientFailedException.java
new file mode 100644
index 0000000..01d011b
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientFailedException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ *
+ */
+class ZookeeperClientFailedException extends Exception {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * @param msg Message.
+ */
+ ZookeeperClientFailedException(String msg) {
+ super(msg);
+ }
+
+ /**
+ * @param cause Cause.
+ */
+ ZookeeperClientFailedException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
new file mode 100644
index 0000000..3cb5fad
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
@@ -0,0 +1,362 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.apache.ignite.spi.discovery.DiscoveryMetricsProvider;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DAEMON;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID;
+
+/**
+ * Zookeeper Cluster Node.
+ */
+public class ZookeeperClusterNode implements IgniteClusterNode, Serializable, Comparable<ZookeeperClusterNode> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private static final byte CLIENT_NODE_MASK = 0x01;
+
+ /** */
+ private UUID id;
+
+ /** */
+ private Serializable consistentId;
+
+ /** */
+ private long internalId;
+
+ /** */
+ private long order;
+
+ /** */
+ private IgniteProductVersion ver;
+
+ /** Node attributes. */
+ private Map<String, Object> attrs;
+
+ /** Internal discovery addresses as strings. */
+ private Collection<String> addrs;
+
+ /** Internal discovery host names as strings. */
+ private Collection<String> hostNames;
+
+ /** */
+ private long sesTimeout;
+
+ /** Metrics provider. */
+ private transient DiscoveryMetricsProvider metricsProvider;
+
+ /** */
+ private transient boolean loc;
+
+ /** */
+ private transient volatile ClusterMetrics metrics;
+
+ /** Node cache metrics. */
+ @GridToStringExclude
+ private transient volatile Map<Integer, CacheMetrics> cacheMetrics;
+
+ /** */
+ private byte flags;
+
+ /** Daemon node flag. */
+ @GridToStringExclude
+ private transient boolean daemon;
+
+ /** Daemon node initialization flag. */
+ @GridToStringExclude
+ private transient volatile boolean daemonInit;
+
+ /**
+ * @param id Node ID.
+ * @param addrs Node addresses.
+ * @param hostNames Node host names.
+ * @param ver Node version.
+ * @param attrs Node attributes.
+ * @param consistentId Consistent ID.
+ * @param sesTimeout Zookeeper session timeout.
+ * @param client Client node flag.
+ * @param metricsProvider Metrics provider.
+ */
+ public ZookeeperClusterNode(
+ UUID id,
+ Collection<String> addrs,
+ Collection<String> hostNames,
+ IgniteProductVersion ver,
+ Map<String, Object> attrs,
+ Serializable consistentId,
+ long sesTimeout,
+ boolean client,
+ DiscoveryMetricsProvider metricsProvider
+ ) {
+ assert id != null;
+ assert consistentId != null;
+
+ this.id = id;
+ this.ver = ver;
+ this.attrs = Collections.unmodifiableMap(attrs);
+ this.addrs = addrs;
+ this.hostNames = hostNames;
+ this.consistentId = consistentId;
+ this.sesTimeout = sesTimeout;
+ this.metricsProvider = metricsProvider;
+
+ if (client)
+ flags |= CLIENT_NODE_MASK;
+ }
+
+ /** {@inheritDoc} */
+ @Override public UUID id() {
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object consistentId() {
+ return consistentId;
+ }
+
+ /** {@inheritDoc} */
+ public void setConsistentId(Serializable consistentId) {
+ this.consistentId = consistentId;
+
+ final Map<String, Object> map = new HashMap<>(attrs);
+
+ map.put(ATTR_NODE_CONSISTENT_ID, consistentId);
+
+ attrs = Collections.unmodifiableMap(map);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isCacheClient() {
+ return isClient();
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public <T> T attribute(String name) {
+ // Even though discovery SPI removes this attribute after authentication, keep this check for safety.
+ if (IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS.equals(name))
+ return null;
+
+ return (T)attrs.get(name);
+ }
+
+ /**
+ * Sets node attributes.
+ *
+ * @param attrs Node attributes.
+ */
+ void setAttributes(Map<String, Object> attrs) {
+ this.attrs = U.sealMap(attrs);
+ }
+
+ /**
+ * Gets node attributes without filtering.
+ *
+ * @return Node attributes without filtering.
+ */
+ Map<String, Object> getAttributes() {
+ return attrs;
+ }
+
+ /** {@inheritDoc} */
+ @Override public ClusterMetrics metrics() {
+ if (metricsProvider != null) {
+ ClusterMetrics metrics0 = metricsProvider.metrics();
+
+ assert metrics0 != null;
+
+ metrics = metrics0;
+
+ return metrics0;
+ }
+
+ return metrics;
+ }
+
+ /** {@inheritDoc} */
+ public void setMetrics(ClusterMetrics metrics) {
+ assert metrics != null;
+
+ this.metrics = metrics;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<Integer, CacheMetrics> cacheMetrics() {
+ if (metricsProvider != null) {
+ Map<Integer, CacheMetrics> cacheMetrics0 = metricsProvider.cacheMetrics();
+
+ cacheMetrics = cacheMetrics0;
+
+ return cacheMetrics0;
+ }
+
+ return cacheMetrics;
+ }
+
+ /** {@inheritDoc} */
+ public void setCacheMetrics(Map<Integer, CacheMetrics> cacheMetrics) {
+ this.cacheMetrics = cacheMetrics != null ? cacheMetrics : Collections.<Integer, CacheMetrics>emptyMap();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, Object> attributes() {
+ // Even though discovery SPI removes this attribute after authentication, keep this check for safety.
+ return F.view(attrs, new IgnitePredicate<String>() {
+ @Override public boolean apply(String s) {
+ return !IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS.equals(s);
+ }
+ });
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<String> addresses() {
+ return addrs;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<String> hostNames() {
+ return hostNames;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long order() {
+ return order;
+ }
+
+ /**
+ * @return Internal ID corresponds to Zookeeper sequential node.
+ */
+ long internalId() {
+ return internalId;
+ }
+
+ /**
+ * @param internalId Internal ID corresponds to Zookeeper sequential node.
+ */
+ void internalId(long internalId) {
+ this.internalId = internalId;
+ }
+
+ /**
+ * @param order Node order.
+ */
+ void order(long order) {
+ assert order > 0 : order;
+
+ this.order = order;
+ }
+
+ /**
+ * @param newId New node ID.
+ */
+ public void onClientDisconnected(UUID newId) {
+ id = newId;
+ }
+
+ /**
+ * @return Session timeout.
+ */
+ long sessionTimeout() {
+ return sesTimeout;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteProductVersion version() {
+ return ver;
+ }
+
+ /**
+ * @param loc Local node flag.
+ */
+ public void local(boolean loc) {
+ this.loc = loc;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isLocal() {
+ return loc;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isDaemon() {
+ if (!daemonInit) {
+ daemon = "true".equalsIgnoreCase((String)attribute(ATTR_DAEMON));
+
+ daemonInit = true;
+ }
+
+ return daemon;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isClient() {
+ return (CLIENT_NODE_MASK & flags) != 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compareTo(@Nullable ZookeeperClusterNode node) {
+ if (node == null)
+ return 1;
+
+ int res = Long.compare(order, node.order);
+
+ if (res == 0) {
+ assert id().equals(node.id()) : "Duplicate order [this=" + this + ", other=" + node + ']';
+
+ res = id().compareTo(node.id());
+ }
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return id.hashCode();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ return F.eqNodes(this, obj);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "ZookeeperClusterNode [id=" + id +
+ ", addrs=" + addrs +
+ ", order=" + order +
+ ", loc=" + loc +
+ ", client=" + isClient() + ']';
+ }
+}
[07/54] [abbrv] ignite git commit: IGNITE-7222 Added ZooKeeper
discovery SPI
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
index 54b3a78..e89a4c8 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
@@ -17,11 +17,21 @@
package org.apache.ignite.spi.communication.tcp;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CyclicBarrier;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.util.nio.GridCommunicationClient;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.spi.IgniteSpiAdapter;
import org.apache.ignite.spi.communication.CommunicationSpi;
import org.apache.ignite.spi.communication.GridAbstractCommunicationSelfTest;
@@ -85,6 +95,67 @@ abstract class GridTcpCommunicationSpiAbstractTest extends GridAbstractCommunica
}
}
+ /**
+ *
+ */
+ public void testCheckConnection1() {
+ for (int i = 0; i < 100; i++) {
+ for (Map.Entry<UUID, CommunicationSpi<Message>> entry : spis.entrySet()) {
+ TcpCommunicationSpi spi = (TcpCommunicationSpi)entry.getValue();
+
+ List<ClusterNode> checkNodes = new ArrayList<>(nodes);
+
+ assert checkNodes.size() > 1;
+
+ IgniteFuture<BitSet> fut = spi.checkConnection(checkNodes);
+
+ BitSet res = fut.get();
+
+ for (int n = 0; n < checkNodes.size(); n++)
+ assertTrue(res.get(n));
+ }
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCheckConnection2() throws Exception {
+ final int THREADS = spis.size();
+
+ final CyclicBarrier b = new CyclicBarrier(THREADS);
+
+ List<IgniteInternalFuture> futs = new ArrayList<>();
+
+ for (Map.Entry<UUID, CommunicationSpi<Message>> entry : spis.entrySet()) {
+ final TcpCommunicationSpi spi = (TcpCommunicationSpi)entry.getValue();
+
+ futs.add(GridTestUtils.runAsync(new Callable() {
+ @Override public Object call() throws Exception {
+ List<ClusterNode> checkNodes = new ArrayList<>(nodes);
+
+ assert checkNodes.size() > 1;
+
+ b.await();
+
+ for (int i = 0; i < 100; i++) {
+ IgniteFuture<BitSet> fut = spi.checkConnection(checkNodes);
+
+ BitSet res = fut.get();
+
+ for (int n = 0; n < checkNodes.size(); n++)
+ assertTrue(res.get(n));
+ }
+
+ return null;
+ }
+ }));
+ }
+
+ for (IgniteInternalFuture f : futs)
+ f.get();
+ }
+
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
super.afterTest();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/spi/discovery/FilterDataForClientNodeDiscoveryTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/FilterDataForClientNodeDiscoveryTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/FilterDataForClientNodeDiscoveryTest.java
index 54b48e5..9a45d2d 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/FilterDataForClientNodeDiscoveryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/FilterDataForClientNodeDiscoveryTest.java
@@ -206,6 +206,11 @@ public class FilterDataForClientNodeDiscoveryTest extends GridCommonAbstractTest
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr, AffinityTopologyVersion topVer,
DiscoCache discoCache) {
return null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
index ca05288..51dcb23 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
@@ -613,6 +613,16 @@ public class GridSpiTestContext implements IgniteSpiContext {
return Collections.emptyMap();
}
+ /** {@inheritDoc} */
+ @Override public boolean communicationFailureResolveSupported() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resolveCommunicationFailure(ClusterNode node, Exception err) {
+ throw new UnsupportedOperationException();
+ }
+
/**
* @param cacheName Cache name.
* @return Map representing cache.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/testframework/config/GridTestProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/config/GridTestProperties.java b/modules/core/src/test/java/org/apache/ignite/testframework/config/GridTestProperties.java
index 4507572..e2594ca 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/config/GridTestProperties.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/config/GridTestProperties.java
@@ -83,6 +83,15 @@ public final class GridTestProperties {
/** "True value" enables {@link BinaryBasicNameMapper} in {@link BinaryTypeConfiguration#getNameMapper()} */
public static final String BINARY_MARSHALLER_USE_SIMPLE_NAME_MAPPER = "binary.marshaller.use.simple.name.mapper";
+ /**
+ * Name of class which provides static method preprocessConfiguration(IgniteConfiguration cfg) to
+ * alter {@link org.apache.ignite.configuration.IgniteConfiguration} before node is started.
+ * <p>
+ * Note: this pre-preprocessor is started only if test starts node using one of GridAbstractTest's startGrid
+ * method.
+ */
+ public static final String IGNITE_CFG_PREPROCESSOR_CLS = "ignite.cfg.preprocessor.class";
+
/** */
static {
// Initialize IGNITE_HOME system property.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index c3b262c..f5784eb 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -46,6 +46,7 @@ import junit.framework.TestCase;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
@@ -84,6 +85,7 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.logger.NullLogger;
import org.apache.ignite.marshaller.Marshaller;
import org.apache.ignite.marshaller.MarshallerContextTestImpl;
@@ -92,6 +94,7 @@ import org.apache.ignite.marshaller.jdk.JdkMarshaller;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.spi.checkpoint.sharedfs.SharedFsCheckpointSpi;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
@@ -124,6 +127,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.internal.GridKernalState.DISCONNECTED;
import static org.apache.ignite.testframework.config.GridTestProperties.BINARY_MARSHALLER_USE_SIMPLE_NAME_MAPPER;
+import static org.apache.ignite.testframework.config.GridTestProperties.IGNITE_CFG_PREPROCESSOR_CLS;
/**
* Common abstract test for Ignite tests.
@@ -203,13 +207,15 @@ public abstract class GridAbstractTest extends TestCase {
if (BINARY_MARSHALLER)
GridTestProperties.setProperty(GridTestProperties.MARSH_CLASS_NAME, BinaryMarshaller.class.getName());
- Thread timer = new Thread(new GridTestClockTimer(), "ignite-clock-for-tests");
+ if (GridTestClockTimer.startTestTimer()) {
+ Thread timer = new Thread(new GridTestClockTimer(), "ignite-clock-for-tests");
- timer.setDaemon(true);
+ timer.setDaemon(true);
- timer.setPriority(10);
+ timer.setPriority(10);
- timer.start();
+ timer.start();
+ }
}
/** */
@@ -838,6 +844,7 @@ public abstract class GridAbstractTest extends TestCase {
protected Ignite startGrid(String igniteInstanceName, GridSpringResourceContext ctx) throws Exception {
return startGrid(igniteInstanceName, optimize(getConfiguration(igniteInstanceName)), ctx);
}
+
/**
* Starts new grid with given name.
*
@@ -852,12 +859,33 @@ public abstract class GridAbstractTest extends TestCase {
startingIgniteInstanceName.set(igniteInstanceName);
try {
+ String cfgProcClsName = System.getProperty(IGNITE_CFG_PREPROCESSOR_CLS);
+
+ if (cfgProcClsName != null) {
+ try {
+ Class<?> cfgProc = Class.forName(cfgProcClsName);
+
+ Method method = cfgProc.getMethod("preprocessConfiguration", IgniteConfiguration.class);
+
+ if (!Modifier.isStatic(method.getModifiers()))
+ throw new Exception("Non-static pre-processor method in pre-processor class: " + cfgProcClsName);
+
+ method.invoke(null, cfg);
+ }
+ catch (Exception e) {
+ log.error("Failed to pre-process IgniteConfiguration using pre-processor class: " + cfgProcClsName);
+
+ throw new IgniteException(e);
+ }
+ }
+
Ignite node = IgnitionEx.start(cfg, ctx);
IgniteConfiguration nodeCfg = node.configuration();
log.info("Node started with the following configuration [id=" + node.cluster().localNode().id()
+ ", marshaller=" + nodeCfg.getMarshaller()
+ + ", discovery=" + nodeCfg.getDiscoverySpi()
+ ", binaryCfg=" + nodeCfg.getBinaryConfiguration()
+ ", lateAff=" + nodeCfg.isLateAffinityAssignment() + "]");
@@ -967,6 +995,26 @@ public abstract class GridAbstractTest extends TestCase {
if (cfg == null)
cfg = optimize(getConfiguration(igniteInstanceName));
+ if (locNode != null) {
+ DiscoverySpi discoverySpi = locNode.configuration().getDiscoverySpi();
+
+ if (discoverySpi != null && !(discoverySpi instanceof TcpDiscoverySpi)) {
+ try {
+ // Clone added to support ZookeeperDiscoverySpi.
+ Method m = discoverySpi.getClass().getDeclaredMethod("cloneSpiConfiguration");
+
+ m.setAccessible(true);
+
+ cfg.setDiscoverySpi((DiscoverySpi) m.invoke(discoverySpi));
+
+ resetDiscovery = false;
+ }
+ catch (NoSuchMethodException e) {
+ // Ignore.
+ }
+ }
+ }
+
return new IgniteProcessProxy(cfg, log, locNode, resetDiscovery);
}
@@ -1075,7 +1123,9 @@ public abstract class GridAbstractTest extends TestCase {
for (Ignite g : srvs)
stopGrid(g.name(), cancel, false);
- assert G.allGrids().isEmpty();
+ List<Ignite> nodes = G.allGrids();
+
+ assert nodes.isEmpty() : nodes;
}
finally {
IgniteProcessProxy.killAll(); // In multi-JVM case.
@@ -1177,6 +1227,14 @@ public abstract class GridAbstractTest extends TestCase {
}
/**
+ * @param nodeIdx Node index.
+ * @return Node ID.
+ */
+ protected final UUID nodeId(int nodeIdx) {
+ return ignite(nodeIdx).cluster().localNode().id();
+ }
+
+ /**
* Gets grid for given test.
*
* @return Grid for given test.
@@ -1217,7 +1275,11 @@ public abstract class GridAbstractTest extends TestCase {
* @throws Exception If failed.
*/
protected Ignite startGrid(String igniteInstanceName, String springCfgPath) throws Exception {
- return startGrid(igniteInstanceName, loadConfiguration(springCfgPath));
+ IgniteConfiguration cfg = loadConfiguration(springCfgPath);
+
+ cfg.setGridLogger(getTestResources().getLogger());
+
+ return startGrid(igniteInstanceName, cfg);
}
/**
@@ -2142,6 +2204,50 @@ public abstract class GridAbstractTest extends TestCase {
}
}
}
+ /**
+ * @param expSize Expected nodes number.
+ * @throws Exception If failed.
+ */
+ protected void waitForTopology(final int expSize) throws Exception {
+ assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ List<Ignite> nodes = G.allGrids();
+
+ if (nodes.size() != expSize) {
+ info("Wait all nodes [size=" + nodes.size() + ", exp=" + expSize + ']');
+
+ return false;
+ }
+
+ for (Ignite node: nodes) {
+ try {
+ IgniteFuture<?> reconnectFut = node.cluster().clientReconnectFuture();
+
+ if (reconnectFut != null && !reconnectFut.isDone()) {
+ info("Wait for size on node, reconnect is in progress [node=" + node.name() + ']');
+
+ return false;
+ }
+
+ int sizeOnNode = node.cluster().nodes().size();
+
+ if (sizeOnNode != expSize) {
+ info("Wait for size on node [node=" + node.name() + ", size=" + sizeOnNode + ", exp=" + expSize + ']');
+
+ return false;
+ }
+ }
+ catch (IgniteClientDisconnectedException e) {
+ info("Wait for size on node, node disconnected [node=" + node.name() + ']');
+
+ return false;
+ }
+ }
+
+ return true;
+ }
+ }, 30_000));
+ }
/**
* @param millis Time to sleep.
@@ -2172,6 +2278,17 @@ public abstract class GridAbstractTest extends TestCase {
}
/**
+ * @return {@code True} if nodes use {@link TcpDiscoverySpi}.
+ */
+ protected static boolean tcpDiscovery() {
+ List<Ignite> nodes = G.allGrids();
+
+ assertFalse("There are no nodes", nodes.isEmpty());
+
+ return nodes.get(0).configuration().getDiscoverySpi() instanceof TcpDiscoverySpi;
+ }
+
+ /**
*
*/
private static interface WriteReplaceOwner {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteNodeRunner.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteNodeRunner.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteNodeRunner.java
index d7be576..2b3a19c 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteNodeRunner.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteNodeRunner.java
@@ -162,6 +162,8 @@ public class IgniteNodeRunner {
cfg.setDiscoverySpi(disco);
}
+ X.println("Configured discovery: " + cfg.getDiscoverySpi().getClass().getName());
+
return cfg;
}
finally {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 55fab8d..14eb296 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -19,6 +19,7 @@ package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.internal.ClusterNodeMetricsSelfTest;
+import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest;
import org.apache.ignite.internal.GridAffinityNoCacheSelfTest;
import org.apache.ignite.internal.GridAffinitySelfTest;
import org.apache.ignite.internal.GridAlwaysFailoverSpiFailSelfTest;
@@ -122,6 +123,7 @@ public class IgniteComputeGridTestSuite {
suite.addTestSuite(GridAlwaysFailoverSpiFailSelfTest.class);
suite.addTestSuite(GridTaskInstanceExecutionSelfTest.class);
suite.addTestSuite(ClusterNodeMetricsSelfTest.class);
+ suite.addTestSuite(ClusterNodeMetricsUpdateTest.class);
suite.addTestSuite(GridNonHistoryMetricsSelfTest.class);
suite.addTestSuite(GridCancelledJobsMetricsSelfTest.class);
suite.addTestSuite(GridCollisionJobsContextSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
index b9ef1e4..e26b211 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
@@ -117,7 +117,7 @@ public class IgniteCacheDistributedQueryCancelSelfTest extends GridCommonAbstrac
}
for (Ignite g : G.allGrids())
- if (!g.configuration().getDiscoverySpi().isClientMode())
+ if (!g.configuration().isClientMode())
stopGrid(g.name(), true);
}
}, 1);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
index 97720d5..bd3b093 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
@@ -89,11 +89,14 @@ public abstract class DynamicIndexAbstractBasicSelfTest extends DynamicIndexAbst
* @param mode Mode.
* @param atomicityMode Atomicity mode.
* @param near Near flag.
+ * @throws Exception If failed.
*/
private void initialize(CacheMode mode, CacheAtomicityMode atomicityMode, boolean near)
- throws IgniteCheckedException {
+ throws Exception {
createSqlCache(node(), cacheConfiguration(mode, atomicityMode, near));
+ awaitPartitionMapExchange();
+
grid(IDX_CLI_NEAR_ONLY).getOrCreateNearCache(CACHE_NAME, new NearCacheConfiguration<>());
assertNoIndex(CACHE_NAME, TBL_NAME, IDX_NAME_1);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java
index a181068..5cad167 100644
--- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java
+++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java
@@ -44,13 +44,7 @@ public class GridJtaTransactionManagerSelfTest extends GridCommonAbstractTest {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName).
setCacheConfiguration(defaultCacheConfiguration().setCacheMode(PARTITIONED));
- cfg.getTransactionConfiguration().setTxManagerFactory(new Factory<TransactionManager>() {
- private static final long serialVersionUID = 0L;
-
- @Override public TransactionManager create() {
- return jotm.getTransactionManager();
- }
- });
+ cfg.getTransactionConfiguration().setTxManagerFactory(new TestTxManagerFactory());
return cfg;
}
@@ -205,4 +199,17 @@ public class GridJtaTransactionManagerSelfTest extends GridCommonAbstractTest {
cache.removeAll();
}
}
+
+ /**
+ *
+ */
+ static class TestTxManagerFactory implements Factory<TransactionManager> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public TransactionManager create() {
+ return jotm.getTransactionManager();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java
index f6fd5c7..14b7fae 100644
--- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java
+++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java
@@ -30,12 +30,19 @@ public class GridPartitionedCacheJtaFactorySelfTest extends AbstractCacheJtaSelf
@Override protected void configureJta(IgniteConfiguration cfg) {
TransactionConfiguration txCfg = cfg.getTransactionConfiguration();
- txCfg.setTxManagerFactory(new Factory<TransactionManager>() {
- private static final long serialVersionUID = 0L;
+ txCfg.setTxManagerFactory(new TestTxManagerFactory());
+ }
+
+ /**
+ *
+ */
+ static class TestTxManagerFactory implements Factory<TransactionManager> {
+ /** */
+ private static final long serialVersionUID = 0L;
- @Override public TransactionManager create() {
- return jotm.getTransactionManager();
- }
- });
+ /** {@inheritDoc} */
+ @Override public TransactionManager create() {
+ return jotm.getTransactionManager();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
index fce47a6..d87ea0a 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
@@ -25,6 +25,7 @@ import org.apache.ignite.internal.processors.cache.query.QueryCursorEx
import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata
import org.apache.ignite.lang.IgniteUuid
import org.apache.ignite.spark.impl._
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi
import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode
import org.apache.spark._
import org.apache.spark.rdd.RDD
@@ -91,8 +92,14 @@ class IgniteRDD[K, V] (
override protected[spark] def getPreferredLocations(split: Partition): Seq[String] = {
ensureCache()
- ic.ignite().affinity(cacheName).mapPartitionToPrimaryAndBackups(split.index)
+ if (ic.ignite().configuration().getDiscoverySpi().isInstanceOf[TcpDiscoverySpi]) {
+ ic.ignite().affinity(cacheName).mapPartitionToPrimaryAndBackups(split.index)
.map(_.asInstanceOf[TcpDiscoveryNode].socketAddresses()).flatten.map(_.getHostName).toList
+ }
+ else {
+ ic.ignite().affinity(cacheName).mapPartitionToPrimaryAndBackups(split.index)
+ .flatten(_.hostNames).toSeq
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/spring/src/test/java/org/apache/ignite/internal/GridFactorySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/spring/src/test/java/org/apache/ignite/internal/GridFactorySelfTest.java b/modules/spring/src/test/java/org/apache/ignite/internal/GridFactorySelfTest.java
index 75128fc..b453858 100644
--- a/modules/spring/src/test/java/org/apache/ignite/internal/GridFactorySelfTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/internal/GridFactorySelfTest.java
@@ -998,7 +998,8 @@ public class GridFactorySelfTest extends GridCommonAbstractTest {
startGrid("1", c);
- assert ((TcpDiscoverySpi)c.getDiscoverySpi()).started();
+ if (tcpDiscovery())
+ assert ((TcpDiscoverySpi)c.getDiscoverySpi()).started();
try {
startGrid("2", c);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/spring/src/test/java/org/apache/ignite/p2p/GridP2PUserVersionChangeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/spring/src/test/java/org/apache/ignite/p2p/GridP2PUserVersionChangeSelfTest.java b/modules/spring/src/test/java/org/apache/ignite/p2p/GridP2PUserVersionChangeSelfTest.java
index b861e19..46da3cc 100644
--- a/modules/spring/src/test/java/org/apache/ignite/p2p/GridP2PUserVersionChangeSelfTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/p2p/GridP2PUserVersionChangeSelfTest.java
@@ -44,6 +44,7 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.testsuites.IgniteIgnore;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.events.EventType.EVT_TASK_UNDEPLOYED;
@@ -255,12 +256,12 @@ public class GridP2PUserVersionChangeSelfTest extends GridCommonAbstractTest {
ignite2.events().localListen(new IgnitePredicate<Event>() {
@Override public boolean apply(Event evt) {
- if (evt.type() == EVT_NODE_LEFT)
+ if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED)
discoLatch.countDown();
return true;
}
- }, EVT_NODE_LEFT);
+ }, EVT_NODE_LEFT, EVT_NODE_FAILED);
Integer res1 = (Integer)ignite1.compute().execute(task1, ignite2.cluster().localNode().id());
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/yardstick/pom-standalone.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/pom-standalone.xml b/modules/yardstick/pom-standalone.xml
index 577a95e..6905d94 100644
--- a/modules/yardstick/pom-standalone.xml
+++ b/modules/yardstick/pom-standalone.xml
@@ -54,6 +54,12 @@
<dependency>
<groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-zookeeper</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
<artifactId>ignite-log4j</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/yardstick/pom.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index 8cad24b..9923bb7 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -55,6 +55,12 @@
<dependency>
<groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-zookeeper</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
<artifactId>ignite-log4j</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/pom.xml
----------------------------------------------------------------------
diff --git a/modules/zookeeper/pom.xml b/modules/zookeeper/pom.xml
index c3c3679..2d47ece 100644
--- a/modules/zookeeper/pom.xml
+++ b/modules/zookeeper/pom.xml
@@ -49,6 +49,12 @@
<dependency>
<groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.curator</groupId>
<artifactId>curator-x-discovery</artifactId>
<version>${curator.version}</version>
</dependency>
@@ -109,6 +115,13 @@
<dependency>
<groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-indexing</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
<artifactId>ignite-log4j</artifactId>
<version>${project.version}</version>
<scope>test</scope>
@@ -122,16 +135,43 @@
</dependency>
<dependency>
+ <groupId>com.thoughtworks.xstream</groupId>
+ <artifactId>xstream</artifactId>
+ <version>1.4.8</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.ignite</groupId>
<artifactId>ignite-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-indexing</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
<!-- Generate the OSGi MANIFEST.MF for this bundle. -->
<plugin>
<groupId>org.apache.felix</groupId>
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
new file mode 100644
index 0000000..860c71c
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
@@ -0,0 +1,557 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.curator.utils.PathUtils;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.IgniteSpiAdapter;
+import org.apache.ignite.spi.IgniteSpiConfiguration;
+import org.apache.ignite.spi.IgniteSpiContext;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiMultipleInstancesSupport;
+import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoveryMetricsProvider;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange;
+import org.apache.ignite.spi.discovery.DiscoverySpiHistorySupport;
+import org.apache.ignite.spi.discovery.DiscoverySpiListener;
+import org.apache.ignite.spi.discovery.DiscoverySpiMutableCustomMessageSupport;
+import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
+import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperClusterNode;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_CONSISTENT_ID_BY_HOST_WITHOUT_PORT;
+import static org.apache.ignite.IgniteSystemProperties.getBoolean;
+
+/**
+ * Zookeeper Discovery Spi.
+ */
+@IgniteSpiMultipleInstancesSupport(true)
+@DiscoverySpiOrderSupport(true)
+@DiscoverySpiHistorySupport(true)
+@DiscoverySpiMutableCustomMessageSupport(false)
+public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi, IgniteDiscoverySpi {
+ /** */
+ public static final String DFLT_ROOT_PATH = "/apacheIgnite";
+
+ /** */
+ public static final long DFLT_JOIN_TIMEOUT = 0;
+
+ /** */
+ @GridToStringInclude
+ private String zkRootPath = DFLT_ROOT_PATH;
+
+ /** */
+ @GridToStringInclude
+ private String zkConnectionString;
+
+ /** */
+ private long joinTimeout = DFLT_JOIN_TIMEOUT;
+
+ /** */
+ @GridToStringInclude
+ private long sesTimeout;
+
+ /** */
+ private boolean clientReconnectDisabled;
+
+ /** */
+ @GridToStringExclude
+ private DiscoverySpiListener lsnr;
+
+ /** */
+ @GridToStringExclude
+ private DiscoverySpiDataExchange exchange;
+
+ /** */
+ @GridToStringExclude
+ private DiscoverySpiNodeAuthenticator nodeAuth;
+
+ /** */
+ @GridToStringExclude
+ private DiscoveryMetricsProvider metricsProvider;
+
+ /** */
+ @GridToStringExclude
+ private ZookeeperDiscoveryImpl impl;
+
+ /** */
+ @GridToStringExclude
+ private Map<String, Object> locNodeAttrs;
+
+ /** */
+ @GridToStringExclude
+ private IgniteProductVersion locNodeVer;
+
+ /** */
+ @GridToStringExclude
+ private Serializable consistentId;
+
+ /** Local node addresses. */
+ private IgniteBiTuple<Collection<String>, Collection<String>> addrs;
+
+ /** */
+ @LoggerResource
+ @GridToStringExclude
+ private IgniteLogger log;
+
+ /** */
+ private IgniteDiscoverySpiInternalListener internalLsnr;
+
+ /**
+ * @return Base path in ZK for znodes created by SPI.
+ */
+ public String getZkRootPath() {
+ return zkRootPath;
+ }
+
+ /**
+ * @param zkRootPath Base path in ZooKeeper for znodes created by SPI.
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = true)
+ public ZookeeperDiscoverySpi setZkRootPath(String zkRootPath) {
+ this.zkRootPath = zkRootPath;
+
+ return this;
+ }
+
+ /**
+ * @return ZooKeeper session timeout.
+ */
+ public long getSessionTimeout() {
+ return sesTimeout;
+ }
+
+ /**
+ * @param sesTimeout ZooKeeper session timeout.
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = true)
+ public ZookeeperDiscoverySpi setSessionTimeout(long sesTimeout) {
+ this.sesTimeout = sesTimeout;
+
+ return this;
+ }
+
+ /**
+ * @return Cluster join timeout.
+ */
+ public long getJoinTimeout() {
+ return joinTimeout;
+ }
+
+ /**
+ * @param joinTimeout Cluster join timeout ({@code 0} means wait forever).
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = true)
+ public ZookeeperDiscoverySpi setJoinTimeout(long joinTimeout) {
+ this.joinTimeout = joinTimeout;
+
+ return this;
+ }
+
+ /**
+ * @return ZooKeeper connection string
+ */
+ public String getZkConnectionString() {
+ return zkConnectionString;
+ }
+
+ /**
+ * @param zkConnectionString ZooKeeper connection string
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = false)
+ public ZookeeperDiscoverySpi setZkConnectionString(String zkConnectionString) {
+ this.zkConnectionString = zkConnectionString;
+
+ return this;
+ }
+
+ /**
+ * If {@code true} client does not try to reconnect.
+ *
+ * @return Client reconnect disabled flag.
+ */
+ public boolean isClientReconnectDisabled() {
+ return clientReconnectDisabled;
+ }
+
+ /**
+ * Sets client reconnect disabled flag.
+ *
+ * @param clientReconnectDisabled Client reconnect disabled flag.
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = true)
+ public ZookeeperDiscoverySpi setClientReconnectDisabled(boolean clientReconnectDisabled) {
+ this.clientReconnectDisabled = clientReconnectDisabled;
+
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean clientReconnectSupported() {
+ return !clientReconnectDisabled;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void clientReconnect() {
+ impl.reconnect();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean knownNode(UUID nodeId) {
+ return impl.knownNode(nodeId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean supportsCommunicationFailureResolve() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resolveCommunicationFailure(ClusterNode node, Exception err) {
+ impl.resolveCommunicationError(node, err);
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Serializable consistentId() throws IgniteSpiException {
+ if (consistentId == null) {
+ consistentId = ignite.configuration().getConsistentId();
+
+ if (consistentId == null) {
+ initAddresses();
+
+ final List<String> sortedAddrs = new ArrayList<>(addrs.get1());
+
+ Collections.sort(sortedAddrs);
+
+ if (getBoolean(IGNITE_CONSISTENT_ID_BY_HOST_WITHOUT_PORT))
+ consistentId = U.consistentId(sortedAddrs);
+ else {
+ Integer commPort = null;
+
+ if (locNodeAttrs != null) {
+ commPort = (Integer)locNodeAttrs.get(
+ TcpCommunicationSpi.class.getSimpleName() + "." + TcpCommunicationSpi.ATTR_PORT);
+ }
+ else {
+ CommunicationSpi commSpi = ignite.configuration().getCommunicationSpi();
+
+ if (commSpi instanceof TcpCommunicationSpi) {
+ commPort = ((TcpCommunicationSpi)commSpi).boundPort();
+
+ if (commPort == -1)
+ commPort = null;
+ }
+ }
+
+ if (commPort == null) {
+ U.warn(log, "Can not initialize default consistentId, TcpCommunicationSpi port is not initialized.");
+
+ consistentId = ignite.configuration().getNodeId();
+ }
+ else
+ consistentId = U.consistentId(sortedAddrs, commPort);
+ }
+ }
+ }
+
+ return consistentId;
+ }
+
+ /**
+ *
+ */
+ private void initAddresses() {
+ if (addrs == null) {
+ String locHost = ignite != null ? ignite.configuration().getLocalHost() : null;
+
+ InetAddress locAddr;
+
+ try {
+ locAddr = U.resolveLocalHost(locHost);
+ }
+ catch (IOException e) {
+ throw new IgniteSpiException("Unknown local address: " + locHost, e);
+ }
+
+ try {
+ addrs = U.resolveLocalAddresses(locAddr);
+ }
+ catch (Exception e) {
+ throw new IgniteSpiException("Failed to resolve local host to set of external addresses: " + locHost,
+ e);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<ClusterNode> getRemoteNodes() {
+ return impl.remoteNodes();
+ }
+
+ /** {@inheritDoc} */
+ @Override public ClusterNode getLocalNode() {
+ return impl != null ? impl.localNode() : null;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public ClusterNode getNode(UUID nodeId) {
+ return impl.node(nodeId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean pingNode(UUID nodeId) {
+ return impl.pingNode(nodeId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setNodeAttributes(Map<String, Object> attrs, IgniteProductVersion ver) {
+ assert locNodeAttrs == null;
+ assert locNodeVer == null;
+
+ if (log.isDebugEnabled()) {
+ log.debug("Node attributes to set: " + attrs);
+ log.debug("Node version to set: " + ver);
+ }
+
+ locNodeAttrs = attrs;
+ locNodeVer = ver;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setListener(@Nullable DiscoverySpiListener lsnr) {
+ this.lsnr = lsnr;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setDataExchange(DiscoverySpiDataExchange exchange) {
+ this.exchange = exchange;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setMetricsProvider(DiscoveryMetricsProvider metricsProvider) {
+ this.metricsProvider = metricsProvider;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void disconnect() throws IgniteSpiException {
+ impl.stop();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setAuthenticator(DiscoverySpiNodeAuthenticator auth) {
+ this.nodeAuth = auth;
+ }
+
+ /**
+ * @return Authenticator.
+ */
+ public DiscoverySpiNodeAuthenticator getAuthenticator() {
+ return nodeAuth;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getGridStartTime() {
+ return impl.gridStartTime();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) {
+ IgniteDiscoverySpiInternalListener internalLsnr = impl.internalLsnr;
+
+ if (internalLsnr != null) {
+ if (!internalLsnr.beforeSendCustomEvent(this, log, msg))
+ return;
+ }
+
+ impl.sendCustomMessage(msg);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void failNode(UUID nodeId, @Nullable String warning) {
+ impl.failNode(nodeId, warning);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isClientMode() throws IllegalStateException {
+ return impl.localNode().isClient();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void onContextInitialized0(IgniteSpiContext spiCtx) throws IgniteSpiException {
+ super.onContextInitialized0(spiCtx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void spiStart(@Nullable String igniteInstanceName) throws IgniteSpiException {
+ if (sesTimeout == 0)
+ sesTimeout = ignite.configuration().getFailureDetectionTimeout().intValue();
+
+ assertParameter(sesTimeout > 0, "sessionTimeout > 0");
+
+ A.notNullOrEmpty(zkConnectionString, "zkConnectionString can not be empty");
+
+ A.notNullOrEmpty(zkRootPath, "zkRootPath can not be empty");
+
+ zkRootPath = zkRootPath.trim();
+
+ if (zkRootPath.endsWith("/"))
+ zkRootPath = zkRootPath.substring(0, zkRootPath.length() - 1);
+
+ try {
+ PathUtils.validatePath(zkRootPath);
+ }
+ catch (IllegalArgumentException e) {
+ throw new IgniteSpiException("zkRootPath is invalid: " + zkRootPath, e);
+ }
+
+ ZookeeperClusterNode locNode = initLocalNode();
+
+ if (log.isInfoEnabled()) {
+ log.info("Start Zookeeper discovery [zkConnectionString=" + zkConnectionString +
+ ", sessionTimeout=" + sesTimeout +
+ ", zkRootPath=" + zkRootPath + ']');
+ }
+
+ impl = new ZookeeperDiscoveryImpl(
+ this,
+ igniteInstanceName,
+ log,
+ zkRootPath,
+ locNode,
+ lsnr,
+ exchange,
+ internalLsnr);
+
+ try {
+ impl.startJoinAndWait();
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteSpiException("Failed to join cluster, thread was interrupted", e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setInternalListener(IgniteDiscoverySpiInternalListener lsnr) {
+ if (impl != null)
+ impl.internalLsnr = lsnr;
+ else
+ internalLsnr = lsnr;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void simulateNodeFailure() {
+ impl.simulateNodeFailure();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void spiStop() throws IgniteSpiException {
+ if (impl != null)
+ impl.stop();
+ }
+
+ /**
+ * @return Local node instance.
+ */
+ private ZookeeperClusterNode initLocalNode() {
+ assert ignite != null;
+
+ initAddresses();
+
+ ZookeeperClusterNode locNode = new ZookeeperClusterNode(
+ ignite.configuration().getNodeId(),
+ addrs.get1(),
+ addrs.get2(),
+ locNodeVer,
+ locNodeAttrs,
+ consistentId(),
+ sesTimeout,
+ ignite.configuration().isClientMode(),
+ metricsProvider);
+
+ locNode.local(true);
+
+ DiscoverySpiListener lsnr = this.lsnr;
+
+ if (lsnr != null)
+ lsnr.onLocalNodeInitialized(locNode);
+
+ if (log.isDebugEnabled())
+ log.debug("Local node initialized: " + locNode);
+
+ if (metricsProvider != null) {
+ locNode.setMetrics(metricsProvider.metrics());
+ locNode.setCacheMetrics(metricsProvider.cacheMetrics());
+ }
+
+ return locNode;
+ }
+
+ /**
+ * Used in tests (called via reflection).
+ *
+ * @return Copy of SPI.
+ */
+ private ZookeeperDiscoverySpi cloneSpiConfiguration() {
+ ZookeeperDiscoverySpi spi = new ZookeeperDiscoverySpi();
+
+ spi.setZkRootPath(zkRootPath);
+ spi.setZkConnectionString(zkConnectionString);
+ spi.setSessionTimeout(sesTimeout);
+ spi.setJoinTimeout(joinTimeout);
+ spi.setClientReconnectDisabled(clientReconnectDisabled);
+
+ return spi;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ZookeeperDiscoverySpi.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractCallabck.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractCallabck.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractCallabck.java
new file mode 100644
index 0000000..b80a9dd
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractCallabck.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.ignite.internal.util.GridSpinBusyLock;
+
+/**
+ *
+ */
+abstract class ZkAbstractCallabck {
+ /** */
+ final ZkRuntimeState rtState;
+
+ /** */
+ private final ZookeeperDiscoveryImpl impl;
+
+ /** */
+ private final GridSpinBusyLock busyLock;
+
+ /**
+ * @param rtState Runtime state.
+ * @param impl Discovery impl.
+ */
+ ZkAbstractCallabck(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+ this.rtState = rtState;
+ this.impl = impl;
+
+ busyLock = impl.busyLock;
+ }
+
+ /**
+ * @return {@code True} if is able to start processing.
+ */
+ final boolean onProcessStart() {
+ boolean start = rtState.errForClose == null && busyLock.enterBusy();
+
+ if (!start) {
+ assert rtState.errForClose != null;
+
+ onStartFailed();
+
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ *
+ */
+ void onStartFailed() {
+ // No-op.
+ }
+
+ /**
+ *
+ */
+ final void onProcessEnd() {
+ busyLock.leaveBusy();
+ }
+
+ /**
+ * @param e Error.
+ */
+ final void onProcessError(Throwable e) {
+ impl.onFatalError(busyLock, e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
new file mode 100644
index 0000000..2292e35
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.List;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ *
+ */
+abstract class ZkAbstractChildrenCallback extends ZkAbstractCallabck implements AsyncCallback.Children2Callback {
+ /**
+ * @param rtState Runtime state.
+ * @param impl Discovery impl.
+ */
+ ZkAbstractChildrenCallback(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+ super(rtState, impl);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+ if (!onProcessStart())
+ return;
+
+ try {
+ processResult0(rc, path, ctx, children, stat);
+
+ onProcessEnd();
+ }
+ catch (Throwable e) {
+ onProcessError(e);
+ }
+ }
+
+ /**
+ * @param rc
+ * @param path
+ * @param ctx
+ * @param children
+ * @param stat
+ * @throws Exception If failed.
+ */
+ abstract void processResult0(int rc, String path, Object ctx, List<String> children, Stat stat)
+ throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractWatcher.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractWatcher.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractWatcher.java
new file mode 100644
index 0000000..9098d05
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractWatcher.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+
+/**
+ *
+ */
+abstract class ZkAbstractWatcher extends ZkAbstractCallabck implements Watcher {
+ /**
+ * @param rtState Runtime state.
+ * @param impl Discovery impl.
+ */
+ ZkAbstractWatcher(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+ super(rtState, impl);
+ }
+
+ /** {@inheritDoc} */
+ @Override public final void process(WatchedEvent evt) {
+ if (!onProcessStart())
+ return;
+
+ try {
+ process0(evt);
+
+ onProcessEnd();
+ }
+ catch (Throwable e) {
+ onProcessError(e);
+ }
+ }
+
+ /**
+ * @param evt Event.
+ * @throws Exception If failed.
+ */
+ protected abstract void process0(WatchedEvent evt) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAliveNodeData.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAliveNodeData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAliveNodeData.java
new file mode 100644
index 0000000..d824377
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAliveNodeData.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Zk Alive Node Data.
+ */
+public class ZkAliveNodeData implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ long lastProcEvt = -1;
+
+ /** */
+ transient boolean needUpdate;
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ZkAliveNodeData.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkBulkJoinContext.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkBulkJoinContext.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkBulkJoinContext.java
new file mode 100644
index 0000000..a186aed
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkBulkJoinContext.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.internal.util.typedef.T2;
+
+/**
+ *
+ */
+class ZkBulkJoinContext {
+ /** */
+ List<T2<ZkJoinedNodeEvtData, Map<Integer, Serializable>>> nodes;
+
+ /**
+ * @param nodeEvtData Node event data.
+ * @param discoData Discovery data for node.
+ */
+ void addJoinedNode(ZkJoinedNodeEvtData nodeEvtData, Map<Integer, Serializable> discoData) {
+ if (nodes == null)
+ nodes = new ArrayList<>();
+
+ nodes.add(new T2<>(nodeEvtData, discoData));
+ }
+
+ /**
+ * @return Number of joined nodes.
+ */
+ int nodes() {
+ return nodes != null ? nodes.size() : 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkClusterNodes.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkClusterNodes.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkClusterNodes.java
new file mode 100644
index 0000000..7e2ea7b
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkClusterNodes.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import org.apache.ignite.cluster.ClusterNode;
+
+/**
+ * Zk Cluster Nodes.
+ */
+public class ZkClusterNodes {
+ /** */
+ final ConcurrentSkipListMap<Long, ZookeeperClusterNode> nodesByOrder = new ConcurrentSkipListMap<>();
+
+ /** */
+ final ConcurrentSkipListMap<Long, ZookeeperClusterNode> nodesByInternalId = new ConcurrentSkipListMap<>();
+
+ /** */
+ final ConcurrentHashMap<UUID, ZookeeperClusterNode> nodesById = new ConcurrentHashMap<>();
+
+ /**
+ * @return Remote nodes.
+ */
+ public Collection<ClusterNode> remoteNodes() {
+ List<ClusterNode> nodes = new ArrayList<>();
+
+ for (ClusterNode node : nodesById.values()) {
+ if (!node.isLocal())
+ nodes.add(node);
+ }
+
+ return nodes;
+ }
+
+ /**
+ * @return Current nodes in topology.
+ */
+ @SuppressWarnings("unchecked")
+ List<ClusterNode> topologySnapshot() {
+ return new ArrayList<>((Collection)nodesByOrder.values());
+ }
+
+ /**
+ * @param node New node.
+ */
+ void addNode(ZookeeperClusterNode node) {
+ assert node.id() != null : node;
+ assert node.order() > 0 : node;
+
+ ZookeeperClusterNode old = nodesById.put(node.id(), node);
+
+ assert old == null : old;
+
+ old = nodesByOrder.put(node.order(), node);
+
+ assert old == null : old;
+
+ old = nodesByInternalId.put(node.internalId(), node);
+
+ assert old == null : old;
+ }
+
+ /**
+ * @param internalId Node internal ID.
+ * @return Removed node.
+ */
+ ZookeeperClusterNode removeNode(long internalId) {
+ ZookeeperClusterNode node = nodesByInternalId.remove(internalId);
+
+ assert node != null : internalId;
+ assert node.order() > 0 : node;
+
+ Object rvmd = nodesByOrder.remove(node.order());
+
+ assert rvmd != null;
+
+ rvmd = nodesById.remove(node.id());
+
+ assert rvmd != null;
+
+ return node;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
new file mode 100644
index 0000000..9c21f13
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.BitSet;
+
+/**
+ *
+ */
+class ZkCommunicationErrorNodeState implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final BitSet commState;
+
+ /** */
+ final Exception err;
+
+ /**
+ * @param commState Communication state.
+ * @param err Error if failed get communication state..
+ */
+ ZkCommunicationErrorNodeState(BitSet commState, Exception err) {
+ assert commState != null || err != null;
+
+ this.commState = commState;
+ this.err = err;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
new file mode 100644
index 0000000..accda6e
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.jboss.netty.util.internal.ConcurrentHashMap;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Future is created on each node when either connection error occurs or resolve communication error request
+ * received.
+ */
+class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implements IgniteSpiTimeoutObject, Runnable {
+ /** */
+ private final ZookeeperDiscoveryImpl impl;
+
+ /** */
+ private final IgniteLogger log;
+
+ /** */
+ private final Map<Long, GridFutureAdapter<Boolean>> nodeFuts = new ConcurrentHashMap<>();
+
+ /** */
+ private final long endTime;
+
+ /** */
+ private final IgniteUuid id;
+
+ /** */
+ private State state;
+
+ /** */
+ private long resolveTopVer;
+
+ /** */
+ private Set<Long> resFailedNodes;
+
+ /** */
+ private Exception resErr;
+
+ /** */
+ private ZkDistributedCollectDataFuture collectResFut;
+
+ /**
+ * @param impl Discovery impl.
+ * @param timeout Timeout to wait before initiating resolve process.
+ * @return Future.
+ */
+ static ZkCommunicationErrorProcessFuture createOnCommunicationError(ZookeeperDiscoveryImpl impl, long timeout) {
+ return new ZkCommunicationErrorProcessFuture(impl, State.WAIT_TIMEOUT, timeout);
+ }
+
+ /**
+ * @param impl Discovery impl.
+ * @return Future.
+ */
+ static ZkCommunicationErrorProcessFuture createOnStartResolveRequest(ZookeeperDiscoveryImpl impl) {
+ return new ZkCommunicationErrorProcessFuture(impl, State.RESOLVE_STARTED, 0);
+ }
+
+ /**
+ * @param impl Discovery implementation.
+ * @param state Initial state.
+ * @param timeout Wait timeout before initiating communication errors resolve.
+ */
+ private ZkCommunicationErrorProcessFuture(ZookeeperDiscoveryImpl impl, State state, long timeout) {
+ assert state != State.DONE;
+
+ this.impl = impl;
+ this.log = impl.log();
+
+ if (state == State.WAIT_TIMEOUT) {
+ assert timeout > 0 : timeout;
+
+ id = IgniteUuid.fromUuid(impl.localNode().id());
+ endTime = System.currentTimeMillis() + timeout;
+ }
+ else {
+ id = null;
+ endTime = 0;
+ }
+
+ this.state = state;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public IgniteLogger logger() {
+ return log;
+ }
+
+ /**
+ * @param collectResFut Collect nodes' communication status future.
+ */
+ void nodeResultCollectFuture(ZkDistributedCollectDataFuture collectResFut) {
+ assert this.collectResFut == null : collectResFut;
+
+ this.collectResFut = collectResFut;
+ }
+
+ /**
+ * @param top Topology.
+ * @throws Exception If failed.
+ */
+ void onTopologyChange(ZkClusterNodes top) throws Exception {
+ for (Map.Entry<Long, GridFutureAdapter<Boolean>> e : nodeFuts.entrySet()) {
+ if (!top.nodesByOrder.containsKey(e.getKey()))
+ e.getValue().onDone(false);
+ }
+
+ if (collectResFut != null)
+ collectResFut.onTopologyChange(top);
+ }
+
+ /**
+ * @param rtState Runtime state.
+ * @param futPath Future path.
+ * @param nodes Nodes to ping.
+ */
+ void checkConnection(final ZkRuntimeState rtState, final String futPath, List<ClusterNode> nodes) {
+ final TcpCommunicationSpi spi = (TcpCommunicationSpi)impl.spi.ignite().configuration().getCommunicationSpi();
+
+ IgniteFuture<BitSet> fut = spi.checkConnection(nodes);
+
+ fut.listen(new IgniteInClosure<IgniteFuture<BitSet>>() {
+ @Override public void apply(final IgniteFuture<BitSet> fut) {
+ // Future completed either from NIO thread or timeout worker, save result from another thread.
+ impl.runInWorkerThread(new ZkRunnable(rtState, impl) {
+ @Override public void run0() throws Exception {
+ BitSet commState = null;
+ Exception err = null;
+
+ try {
+ commState = fut.get();
+ }
+ catch (Exception e) {
+ err = e;
+ }
+
+ ZkCommunicationErrorNodeState state = new ZkCommunicationErrorNodeState(commState, err);
+
+ ZkDistributedCollectDataFuture.saveNodeResult(futPath,
+ rtState.zkClient,
+ impl.localNode().order(),
+ impl.marshalZip(state));
+ }
+
+ @Override void onStartFailed() {
+ onError(rtState.errForClose);
+ }
+ });
+
+ }
+ });
+ }
+
+ /**
+ *
+ */
+ void scheduleCheckOnTimeout() {
+ synchronized (this) {
+ if (state == State.WAIT_TIMEOUT)
+ impl.spi.getSpiContext().addTimeoutObject(this);
+ }
+ }
+
+ /**
+ * @param topVer Topology version.
+ * @return {@code False} if future was already completed and need create another future instance.
+ */
+ boolean onStartResolveRequest(long topVer) {
+ synchronized (this) {
+ if (state == State.DONE)
+ return false;
+
+ if (state == State.WAIT_TIMEOUT)
+ impl.spi.getSpiContext().removeTimeoutObject(this);
+
+ assert resolveTopVer == 0 : resolveTopVer;
+
+ resolveTopVer = topVer;
+
+ state = State.RESOLVE_STARTED;
+ }
+
+ return true;
+ }
+
+ /**
+ * @param err Error.
+ */
+ void onError(Exception err) {
+ assert err != null;
+
+ Map<Long, GridFutureAdapter<Boolean>> futs;
+
+ synchronized (this) {
+ if (state == State.DONE) {
+ assert resErr != null;
+
+ return;
+ }
+
+ state = State.DONE;
+
+ resErr = err;
+
+ futs = nodeFuts; // nodeFuts should not be modified after state changed to DONE.
+ }
+
+ for (Map.Entry<Long, GridFutureAdapter<Boolean>> e : futs.entrySet())
+ e.getValue().onDone(err);
+
+ onDone(err);
+ }
+
+ /**
+ * @param failedNodes Node failed as result of resolve process.
+ */
+ void onFinishResolve(Set<Long> failedNodes) {
+ Map<Long, GridFutureAdapter<Boolean>> futs;
+
+ synchronized (this) {
+ if (state == State.DONE) {
+ assert resErr != null;
+
+ return;
+ }
+
+ assert state == State.RESOLVE_STARTED : state;
+
+ state = State.DONE;
+
+ resFailedNodes = failedNodes;
+
+ futs = nodeFuts; // nodeFuts should not be modified after state changed to DONE.
+ }
+
+ for (Map.Entry<Long, GridFutureAdapter<Boolean>> e : futs.entrySet()) {
+ Boolean res = !F.contains(resFailedNodes, e.getKey());
+
+ e.getValue().onDone(res);
+ }
+
+ onDone();
+ }
+
+ /**
+ * @param node Node.
+ * @return Future finished when communication error resolve is done or {@code null} if another
+ * resolve process should be started.
+ */
+ @Nullable IgniteInternalFuture<Boolean> nodeStatusFuture(ClusterNode node) {
+ GridFutureAdapter<Boolean> fut;
+
+ synchronized (this) {
+ if (state == State.DONE) {
+ if (resolveTopVer != 0 && node.order() <= resolveTopVer) {
+ Boolean res = !F.contains(resFailedNodes, node.order());
+
+ return new GridFinishedFuture<>(res);
+ }
+ else
+ return null;
+ }
+
+ fut = nodeFuts.get(node.order());
+
+ if (fut == null)
+ nodeFuts.put(node.order(), fut = new GridFutureAdapter<>());
+ }
+
+ if (impl.node(node.order()) == null)
+ fut.onDone(false);
+
+ return fut;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ // Run from zk discovery worker pool after timeout.
+ if (needProcessTimeout()) {
+ try {
+ UUID reqId = UUID.randomUUID();
+
+ if (log.isInfoEnabled()) {
+ log.info("Initiate cluster-wide communication error resolve process [reqId=" + reqId +
+ ", errNodes=" + nodeFuts.size() + ']');
+ }
+
+ impl.sendCustomMessage(new ZkCommunicationErrorResolveStartMessage(reqId));
+ }
+ catch (Exception e) {
+ Collection<GridFutureAdapter<Boolean>> futs;
+
+ synchronized (this) {
+ if (state != State.WAIT_TIMEOUT)
+ return;
+
+ state = State.DONE;
+ resErr = e;
+
+ futs = nodeFuts.values(); // nodeFuts should not be modified after state changed to DONE.
+ }
+
+ for (GridFutureAdapter<Boolean> fut : futs)
+ fut.onDone(e);
+
+ onDone(e);
+ }
+ }
+ }
+
+ /**
+ * @return {@code True} if need initiate resolve process after timeout expired.
+ */
+ private boolean needProcessTimeout() {
+ synchronized (this) {
+ if (state != State.WAIT_TIMEOUT)
+ return false;
+
+ for (GridFutureAdapter<Boolean> fut : nodeFuts.values()) {
+ if (!fut.isDone())
+ return true;
+ }
+
+ state = State.DONE;
+ }
+
+ onDone(null, null);
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid id() {
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long endTime() {
+ return endTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onTimeout() {
+ if (needProcessTimeout())
+ impl.runInWorkerThread(this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean onDone(@Nullable Void res, @Nullable Throwable err) {
+ if (super.onDone(res, err)) {
+ impl.clearCommunicationErrorProcessFuture(this);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ZkCommunicationErrorProcessFuture.class, this);
+ }
+
+ /**
+ *
+ */
+ enum State {
+ /** */
+ DONE,
+
+ /** */
+ WAIT_TIMEOUT,
+
+ /** */
+ RESOLVE_STARTED
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
new file mode 100644
index 0000000..9b7476c
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+class ZkCommunicationErrorResolveFinishMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final UUID futId;
+
+ /** */
+ final long topVer;
+
+ /** */
+ transient ZkCommunicationErrorResolveResult res;
+
+ /**
+ * @param futId Future ID.
+ * @param topVer Topology version when resolve process finished.
+ */
+ ZkCommunicationErrorResolveFinishMessage(UUID futId, long topVer) {
+ this.futId = futId;
+ this.topVer = topVer;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isMutable() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ZkCommunicationErrorResolveFinishMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
new file mode 100644
index 0000000..23495aa
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.GridLongList;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+class ZkCommunicationErrorResolveResult implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final GridLongList killedNodes;
+
+ /** */
+ final Exception err;
+
+ /**
+ * @param killedNodes Killed nodes.
+ * @param err Error.
+ */
+ ZkCommunicationErrorResolveResult(@Nullable GridLongList killedNodes, Exception err) {
+ this.killedNodes = killedNodes;
+ this.err = err;
+ }
+}
[48/54] [abbrv] ignite git commit: IGNITE-8230: SQL: Fixed backup
number propagation in CREATE TABLE command. This closes #3803.
Posted by ag...@apache.org.
IGNITE-8230: SQL: Fixed backup number propagation in CREATE TABLE command. This closes #3803.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b93595cb
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b93595cb
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b93595cb
Branch: refs/heads/ignite-6083
Commit: b93595cbe67bb3313d472db58b6d113e10a2f97e
Parents: a57c9e1
Author: devozerov <vo...@gridgain.com>
Authored: Thu Apr 12 15:13:51 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Apr 12 15:13:51 2018 +0300
----------------------------------------------------------------------
.../processors/query/GridQueryProcessor.java | 7 ++--
.../query/h2/sql/GridSqlCreateTable.java | 7 ++--
.../cache/index/H2DynamicTableSelfTest.java | 37 ++++++++++++++++++++
3 files changed, 45 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b93595cb/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index bde9427..03e5254 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -1483,10 +1483,10 @@ public class GridQueryProcessor extends GridProcessorAdapter {
@SuppressWarnings("unchecked")
public void dynamicTableCreate(String schemaName, QueryEntity entity, String templateName, String cacheName,
String cacheGroup, @Nullable String dataRegion, String affinityKey, @Nullable CacheAtomicityMode atomicityMode,
- @Nullable CacheWriteSynchronizationMode writeSyncMode, int backups, boolean ifNotExists)
+ @Nullable CacheWriteSynchronizationMode writeSyncMode, @Nullable Integer backups, boolean ifNotExists)
throws IgniteCheckedException {
assert !F.isEmpty(templateName);
- assert backups >= 0;
+ assert backups == null || backups >= 0;
CacheConfiguration<?, ?> ccfg = ctx.cache().getConfigFromTemplate(templateName);
@@ -1525,7 +1525,8 @@ public class GridQueryProcessor extends GridProcessorAdapter {
if (writeSyncMode != null)
ccfg.setWriteSynchronizationMode(writeSyncMode);
- ccfg.setBackups(backups);
+ if (backups != null)
+ ccfg.setBackups(backups);
ccfg.setSqlSchema(schemaName);
ccfg.setSqlEscapeAll(true);
http://git-wip-us.apache.org/repos/asf/ignite/blob/b93595cb/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlCreateTable.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlCreateTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlCreateTable.java
index 3608aed..de86d6a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlCreateTable.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlCreateTable.java
@@ -22,6 +22,7 @@ import java.util.LinkedHashSet;
import java.util.List;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.jetbrains.annotations.Nullable;
/**
* CREATE TABLE statement.
@@ -57,7 +58,7 @@ public class GridSqlCreateTable extends GridSqlStatement {
private CacheWriteSynchronizationMode writeSyncMode;
/** Backups number for new cache. */
- private int backups;
+ private Integer backups;
/** Quietly ignore this command if table already exists. */
private boolean ifNotExists;
@@ -184,14 +185,14 @@ public class GridSqlCreateTable extends GridSqlStatement {
/**
* @return Backups number for new cache.
*/
- public int backups() {
+ @Nullable public Integer backups() {
return backups;
}
/**
* @param backups Backups number for new cache.
*/
- public void backups(int backups) {
+ public void backups(Integer backups) {
this.backups = backups;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b93595cb/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
index b20bb59..8224711 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
@@ -88,7 +88,14 @@ public class H2DynamicTableSelfTest extends AbstractSchemaSelfTest {
/** Bad data region name. */
public static final String DATA_REGION_NAME_BAD = "my_data_region_bad";
+ /** Cache with backups. */
+ private static final String CACHE_NAME_BACKUPS = CACHE_NAME + "_backups";
+
+ /** Number of backups for backup test. */
+ private static final int DFLT_BACKUPS = 2;
+
/** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
@Override protected void beforeTestsStarted() throws Exception {
super.beforeTestsStarted();
@@ -98,6 +105,8 @@ public class H2DynamicTableSelfTest extends AbstractSchemaSelfTest {
client().addCacheConfiguration(cacheConfiguration());
client().addCacheConfiguration(cacheConfiguration().setName(CACHE_NAME + "_async")
.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_ASYNC));
+
+ client().addCacheConfiguration(cacheConfiguration().setName(CACHE_NAME_BACKUPS).setBackups(DFLT_BACKUPS));
}
/** {@inheritDoc} */
@@ -108,6 +117,7 @@ public class H2DynamicTableSelfTest extends AbstractSchemaSelfTest {
}
/** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
@Override protected void beforeTest() throws Exception {
super.beforeTest();
@@ -120,6 +130,7 @@ public class H2DynamicTableSelfTest extends AbstractSchemaSelfTest {
execute("DROP TABLE IF EXISTS PUBLIC.\"Person\"");
execute("DROP TABLE IF EXISTS PUBLIC.\"City\"");
execute("DROP TABLE IF EXISTS PUBLIC.\"NameTest\"");
+ execute("DROP TABLE IF EXISTS PUBLIC.\"BackupTest\"");
super.afterTest();
}
@@ -496,6 +507,32 @@ public class H2DynamicTableSelfTest extends AbstractSchemaSelfTest {
}
/**
+ * Test backups propagation.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ public void testBackups() throws Exception {
+ String cacheName = "BackupTestCache";
+
+ execute("CREATE TABLE \"BackupTest\" (id BIGINT PRIMARY KEY, name VARCHAR) WITH \"template=" +
+ CACHE_NAME_BACKUPS + ", cache_name=" + cacheName + "\"");
+
+ CacheConfiguration ccfg = grid(0).cache(cacheName).getConfiguration(CacheConfiguration.class);
+
+ assertEquals(DFLT_BACKUPS, ccfg.getBackups());
+
+ execute("DROP TABLE PUBLIC.\"BackupTest\"");
+
+ execute("CREATE TABLE \"BackupTest\" (id BIGINT PRIMARY KEY, name VARCHAR) WITH \"template=" +
+ CACHE_NAME_BACKUPS + ", cache_name=" + cacheName + ", backups=1\"");
+
+ ccfg = grid(0).cache(cacheName).getConfiguration(CacheConfiguration.class);
+
+ assertEquals(1, ccfg.getBackups());
+ }
+
+ /**
* Test that {@code CREATE TABLE} with given template cache name actually creates new cache,
* H2 table and type descriptor on all nodes, optionally with cache type check.
* @param tplCacheName Template cache name.
[28/54] [abbrv] ignite git commit: IGNITE-7222 .NET: Ignore missing
IgniteConfiguration.CommunicationFailureResolver
Posted by ag...@apache.org.
IGNITE-7222 .NET: Ignore missing IgniteConfiguration.CommunicationFailureResolver
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/780fc07b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/780fc07b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/780fc07b
Branch: refs/heads/ignite-6083
Commit: 780fc07be0b257b578647682585c89548e6d695d
Parents: da77b98
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Wed Apr 11 11:51:45 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Wed Apr 11 11:51:45 2018 +0300
----------------------------------------------------------------------
.../ApiParity/IgniteConfigurationParityTest.cs | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/780fc07b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
index d68083f..bf34fc0 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
@@ -63,7 +63,8 @@ namespace Apache.Ignite.Core.Tests.ApiParity
"ClassLoader",
"CacheStoreSessionListenerFactories",
"PlatformConfiguration",
- "ExecutorConfiguration"
+ "ExecutorConfiguration",
+ "CommunicationFailureResolver"
};
/** Properties that are missing on .NET side. */
[06/54] [abbrv] ignite git commit: IGNITE-7222 Added ZooKeeper
discovery SPI
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
new file mode 100644
index 0000000..0c79c36
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Zk Communication Error Resolve Start Message.
+ */
+public class ZkCommunicationErrorResolveStartMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final UUID id;
+
+ /**
+ * @param id Unique ID.
+ */
+ ZkCommunicationErrorResolveStartMessage(UUID id) {
+ this.id = id;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isMutable() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ZkCommunicationErrorResolveStartMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationFailureContext.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationFailureContext.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationFailureContext.java
new file mode 100644
index 0000000..d27b717
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationFailureContext.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.CommunicationFailureContext;
+import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+class ZkCommunicationFailureContext implements CommunicationFailureContext {
+ /** */
+ private static final Comparator<ClusterNode> NODE_ORDER_CMP = new Comparator<ClusterNode>() {
+ @Override public int compare(ClusterNode node1, ClusterNode node2) {
+ return Long.compare(node1.order(), node2.order());
+ }
+ };
+
+ /** */
+ private Set<ClusterNode> killedNodes = new HashSet<>();
+
+ /** */
+ private final Map<UUID, BitSet> nodesState;
+
+ /** */
+ private final List<ClusterNode> initialNodes;
+
+ /** */
+ private final List<ClusterNode> curNodes;
+
+ /** */
+ private final GridCacheSharedContext<?, ?> ctx;
+
+ /**
+ * @param ctx Context.
+ * @param curNodes Current topology snapshot.
+ * @param initialNodes Topology snapshot when communication error resolve started.
+ * @param nodesState Nodes communication state.
+ */
+ ZkCommunicationFailureContext(
+ GridCacheSharedContext<?, ?> ctx,
+ List<ClusterNode> curNodes,
+ List<ClusterNode> initialNodes,
+ Map<UUID, BitSet> nodesState)
+ {
+ this.ctx = ctx;
+ this.curNodes = Collections.unmodifiableList(curNodes);
+ this.initialNodes = initialNodes;
+ this.nodesState = nodesState;
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<ClusterNode> topologySnapshot() {
+ return curNodes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean connectionAvailable(ClusterNode node1, ClusterNode node2) {
+ BitSet nodeState = nodesState.get(node1.id());
+
+ if (nodeState == null)
+ throw new IllegalArgumentException("Invalid node: " + node1);
+
+ int nodeIdx = Collections.binarySearch(initialNodes, node2, NODE_ORDER_CMP);
+
+ if (nodeIdx < 0)
+ throw new IllegalArgumentException("Invalid node: " + node2);
+
+ assert nodeIdx < nodeState.size() : nodeIdx;
+
+ return nodeState.get(nodeIdx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, CacheConfiguration<?, ?>> startedCaches() {
+ Map<Integer, DynamicCacheDescriptor> cachesMap = ctx.affinity().caches();
+
+ Map<String, CacheConfiguration<?, ?>> res = U.newHashMap(cachesMap.size());
+
+ for (DynamicCacheDescriptor desc : cachesMap.values()) {
+ if (desc.cacheType().userCache())
+ res.put(desc.cacheName(), desc.cacheConfiguration());
+ }
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<List<ClusterNode>> cacheAffinity(String cacheName) {
+ if (cacheName == null)
+ throw new NullPointerException("Null cache name.");
+
+ DynamicCacheDescriptor cacheDesc = ctx.affinity().caches().get(CU.cacheId(cacheName));
+
+ if (cacheDesc == null)
+ throw new IllegalArgumentException("Invalid cache name: " + cacheName);
+
+ GridAffinityAssignmentCache aff = ctx.affinity().groupAffinity(cacheDesc.groupId());
+
+ assert aff != null : cacheName;
+
+ return aff.readyAssignments(aff.lastVersion());
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<List<ClusterNode>> cachePartitionOwners(String cacheName) {
+ if (cacheName == null)
+ throw new NullPointerException("Null cache name.");
+
+ DynamicCacheDescriptor cacheDesc = ctx.affinity().caches().get(CU.cacheId(cacheName));
+
+ if (cacheDesc == null)
+ throw new IllegalArgumentException("Invalid cache name: " + cacheName);
+
+ if (cacheDesc.cacheConfiguration().getCacheMode() == CacheMode.LOCAL)
+ return Collections.emptyList();
+
+ CacheGroupContext grp = ctx.cache().cacheGroup(cacheDesc.groupId());
+
+ GridDhtPartitionTopology top;
+
+ if (grp == null) {
+ top = ctx.exchange().clientTopologyIfExists(cacheDesc.groupId());
+
+ assert top != null : cacheName;
+ }
+ else
+ top = grp.topology();
+
+ return top.allOwners();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void killNode(ClusterNode node) {
+ if (node == null)
+ throw new NullPointerException();
+
+ if (Collections.binarySearch(curNodes, node, NODE_ORDER_CMP) < 0)
+ throw new IllegalArgumentException("Invalid node: " + node);
+
+ killedNodes.add(node);
+ }
+
+ /**
+ * @return Nodes to fail.
+ */
+ Set<ClusterNode> killedNodes() {
+ return killedNodes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "ZkCommunicationFailureContext []";
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java
new file mode 100644
index 0000000..21dfe62
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+
+/**
+ *
+ */
+class ZkDiscoveryCustomEventData extends ZkDiscoveryEventData {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final long origEvtId;
+
+ /** */
+ final UUID sndNodeId;
+
+ /** */
+ final String evtPath;
+
+ /** Message instance (can be marshalled as part of ZkDiscoveryCustomEventData or stored in separate znode. */
+ DiscoverySpiCustomMessage msg;
+
+ /** Unmarshalled message. */
+ transient DiscoverySpiCustomMessage resolvedMsg;
+
+ /**
+ * @param evtId Event ID.
+ * @param origEvtId For acknowledge events ID of original event.
+ * @param topVer Topology version.
+ * @param sndNodeId Sender node ID.
+ * @param msg Message instance.
+ * @param evtPath Event path.
+ */
+ ZkDiscoveryCustomEventData(
+ long evtId,
+ long origEvtId,
+ long topVer,
+ UUID sndNodeId,
+ DiscoverySpiCustomMessage msg,
+ String evtPath)
+ {
+ super(evtId, ZK_EVT_CUSTOM_EVT, topVer);
+
+ assert sndNodeId != null;
+ assert msg != null || origEvtId != 0 || !F.isEmpty(evtPath);
+
+ this.origEvtId = origEvtId;
+ this.msg = msg;
+ this.sndNodeId = sndNodeId;
+ this.evtPath = evtPath;
+ }
+
+ /**
+ * @return {@code True} for custom event ack message.
+ */
+ boolean ackEvent() {
+ return origEvtId != 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "ZkDiscoveryCustomEventData [" +
+ "evtId=" + eventId() +
+ ", topVer=" + topologyVersion() +
+ ", sndNode=" + sndNodeId +
+ ", ack=" + ackEvent() +
+ ']';
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventData.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventData.java
new file mode 100644
index 0000000..d667a17
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventData.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Set;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+abstract class ZkDiscoveryEventData implements Serializable {
+ /** */
+ static final byte ZK_EVT_NODE_JOIN = 1;
+
+ /** */
+ static final byte ZK_EVT_NODE_FAILED = 2;
+
+ /** */
+ static final byte ZK_EVT_CUSTOM_EVT = 3;
+
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final long evtId;
+
+ /** */
+ private final byte evtType;
+
+ /** */
+ private final long topVer;
+
+ /** */
+ private transient Set<Long> remainingAcks;
+
+ /** */
+ int flags;
+
+ /**
+ * @param evtId Event ID.
+ * @param evtType Event type.
+ * @param topVer Topology version.
+ */
+ ZkDiscoveryEventData(long evtId, byte evtType, long topVer) {
+ assert evtType == ZK_EVT_NODE_JOIN || evtType == ZK_EVT_NODE_FAILED || evtType == ZK_EVT_CUSTOM_EVT : evtType;
+
+ this.evtId = evtId;
+ this.evtType = evtType;
+ this.topVer = topVer;
+ }
+
+ /**
+ * @param nodes Current nodes in topology.
+ */
+ void initRemainingAcks(Collection<ZookeeperClusterNode> nodes) {
+ assert remainingAcks == null : this;
+
+ remainingAcks = U.newHashSet(nodes.size());
+
+ for (ZookeeperClusterNode node : nodes) {
+ if (!node.isLocal() && node.order() <= topVer) {
+ boolean add = remainingAcks.add(node.internalId());
+
+ assert add : node;
+ }
+ }
+ }
+
+ /**
+ * @param node Node.
+ */
+ void addRemainingAck(ZookeeperClusterNode node) {
+ assert node.order() <= topVer : node;
+
+ boolean add = remainingAcks.add(node.internalId());
+
+ assert add : node;
+ }
+
+ /**
+ * @return {@code True} if all nodes processed event.
+ */
+ boolean allAcksReceived() {
+ return remainingAcks.isEmpty();
+ }
+
+ /**
+ * @return Remaining acks.
+ */
+ Set<Long> remainingAcks() {
+ return remainingAcks;
+ }
+
+ /**
+ * @param nodeInternalId Node ID.
+ * @param ackEvtId Last event ID processed on node.
+ * @return {@code True} if all nodes processed event.
+ */
+ boolean onAckReceived(Long nodeInternalId, long ackEvtId) {
+ assert remainingAcks != null;
+
+ if (ackEvtId >= evtId)
+ remainingAcks.remove(nodeInternalId);
+
+ return remainingAcks.isEmpty();
+ }
+
+ /**
+ * @param node Failed node.
+ * @return {@code True} if all nodes processed event.
+ */
+ boolean onNodeFail(ZookeeperClusterNode node) {
+ assert remainingAcks != null : this;
+
+ remainingAcks.remove(node.internalId());
+
+ return remainingAcks.isEmpty();
+ }
+
+ /**
+ * @param flag Flag mask.
+ * @return {@code True} if flag set.
+ */
+ boolean flagSet(int flag) {
+ return (flags & flag) == flag;
+ }
+
+ /**
+ * @return Event ID.
+ */
+ long eventId() {
+ return evtId;
+ }
+
+ /**
+ * @return Event type.
+ */
+ byte eventType() {
+ return evtType;
+ }
+
+ /**
+ * @return Event topology version.
+ */
+ long topologyVersion() {
+ return topVer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
new file mode 100644
index 0000000..dce861b
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.TreeMap;
+import java.util.UUID;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+class ZkDiscoveryEventsData implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Unique cluster ID (generated when first node in cluster starts). */
+ final UUID clusterId;
+
+ /** Internal order of last processed custom event. */
+ long procCustEvt = -1;
+
+ /** Event ID counter. */
+ long evtIdGen;
+
+ /** Current topology version. */
+ long topVer;
+
+ /** Max node internal order in cluster. */
+ long maxInternalOrder;
+
+ /** Cluster start time (recorded when first node in cluster starts). */
+ final long clusterStartTime;
+
+ /** Events to process. */
+ final TreeMap<Long, ZkDiscoveryEventData> evts;
+
+ /** ID of current active communication error resolve process. */
+ private UUID commErrFutId;
+
+ /**
+ * @param clusterStartTime Start time of first node in cluster.
+ * @return Events.
+ */
+ static ZkDiscoveryEventsData createForNewCluster(long clusterStartTime) {
+ return new ZkDiscoveryEventsData(
+ UUID.randomUUID(),
+ clusterStartTime,
+ 1L,
+ new TreeMap<Long, ZkDiscoveryEventData>()
+ );
+ }
+
+ /**
+ * @param clusterId Cluster ID.
+ * @param topVer Current topology version.
+ * @param clusterStartTime Cluster start time.
+ * @param evts Events history.
+ */
+ private ZkDiscoveryEventsData(
+ UUID clusterId,
+ long clusterStartTime,
+ long topVer,
+ TreeMap<Long, ZkDiscoveryEventData> evts)
+ {
+ this.clusterId = clusterId;
+ this.clusterStartTime = clusterStartTime;
+ this.topVer = topVer;
+ this.evts = evts;
+ }
+
+ /**
+ * @param node Joined node.
+ */
+ void onNodeJoin(ZookeeperClusterNode node) {
+ if (node.internalId() > maxInternalOrder)
+ maxInternalOrder = node.internalId();
+ }
+
+ /**
+ * @return Future ID.
+ */
+ @Nullable UUID communicationErrorResolveFutureId() {
+ return commErrFutId;
+ }
+
+ /**
+ * @param id Future ID.
+ */
+ void communicationErrorResolveFutureId(@Nullable UUID id) {
+ commErrFutId = id;
+ }
+
+ /**
+ * @param nodes Current nodes in topology (these nodes should ack that event processed).
+ * @param evt Event.
+ */
+ void addEvent(Collection<ZookeeperClusterNode> nodes, ZkDiscoveryEventData evt) {
+ Object old = evts.put(evt.eventId(), evt);
+
+ assert old == null : old;
+
+ evt.initRemainingAcks(nodes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeFailEventData.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeFailEventData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeFailEventData.java
new file mode 100644
index 0000000..c76158f
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeFailEventData.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ *
+ */
+class ZkDiscoveryNodeFailEventData extends ZkDiscoveryEventData {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private long failedNodeInternalId;
+
+ /**
+ * @param evtId Event ID.
+ * @param topVer Topology version.
+ * @param failedNodeInternalId Failed node ID.
+ */
+ ZkDiscoveryNodeFailEventData(long evtId, long topVer, long failedNodeInternalId) {
+ super(evtId, ZK_EVT_NODE_FAILED, topVer);
+
+ this.failedNodeInternalId = failedNodeInternalId;
+ }
+
+ /**
+ * @return Failed node ID.
+ */
+ long failedNodeInternalId() {
+ return failedNodeInternalId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "ZkDiscoveryNodeFailEventData [" +
+ "evtId=" + eventId() +
+ ", topVer=" + topologyVersion() +
+ ", nodeId=" + failedNodeInternalId + ']';
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeJoinEventData.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeJoinEventData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeJoinEventData.java
new file mode 100644
index 0000000..e46d52d
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeJoinEventData.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.List;
+
+/**
+ *
+ */
+class ZkDiscoveryNodeJoinEventData extends ZkDiscoveryEventData {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final List<ZkJoinedNodeEvtData> joinedNodes;
+
+ /** */
+ final int dataForJoinedPartCnt;
+
+ /**
+ * @param evtId Event ID.
+ * @param topVer Topology version.
+ * @param joinedNodes Joined nodes data.
+ * @param dataForJoinedPartCnt Data for joined part count.
+ */
+ ZkDiscoveryNodeJoinEventData(
+ long evtId,
+ long topVer,
+ List<ZkJoinedNodeEvtData> joinedNodes,
+ int dataForJoinedPartCnt)
+ {
+ super(evtId, ZK_EVT_NODE_JOIN, topVer);
+
+ this.joinedNodes = joinedNodes;
+ this.dataForJoinedPartCnt = dataForJoinedPartCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "ZkDiscoveryNodeJoinEventData [" +
+ "evtId=" + eventId() +
+ ", topVer=" + topologyVersion() +
+ ", nodes=" + joinedNodes + ']';
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
new file mode 100644
index 0000000..174d698
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ *
+ */
+class ZkDistributedCollectDataFuture extends GridFutureAdapter<Void> {
+ /** */
+ private final IgniteLogger log;
+
+ /** */
+ private final String futPath;
+
+ /** */
+ private final Set<Long> remainingNodes;
+
+ /** */
+ private final Callable<Void> lsnr;
+
+ /**
+ * @param impl Disovery impl
+ * @param rtState Runtime state.
+ * @param futPath Future path.
+ * @param lsnr Future listener.
+ * @throws Exception If listener call failed.
+ */
+ ZkDistributedCollectDataFuture(
+ ZookeeperDiscoveryImpl impl,
+ ZkRuntimeState rtState,
+ String futPath,
+ Callable<Void> lsnr)
+ throws Exception
+ {
+ this.log = impl.log();
+ this.futPath = futPath;
+ this.lsnr = lsnr;
+
+ ZkClusterNodes top = rtState.top;
+
+ // Assume new nodes can not join while future is in progress.
+
+ remainingNodes = U.newHashSet(top.nodesByOrder.size());
+
+ for (ZookeeperClusterNode node : top.nodesByInternalId.values())
+ remainingNodes.add(node.order());
+
+ NodeResultsWatcher watcher = new NodeResultsWatcher(rtState, impl);
+
+ if (remainingNodes.isEmpty())
+ completeAndNotifyListener();
+ else {
+ if (log.isInfoEnabled()) {
+ log.info("Initialize data collect future [futPath=" + futPath + ", " +
+ "remainingNodes=" + remainingNodes.size() + ']');
+ }
+
+ rtState.zkClient.getChildrenAsync(futPath, watcher, watcher);
+ }
+ }
+
+ /**
+ * @throws Exception If listener call failed.
+ */
+ private void completeAndNotifyListener() throws Exception {
+ if (super.onDone())
+ lsnr.call();
+ }
+
+ /**
+ * @param futPath
+ * @param client
+ * @param nodeOrder
+ * @param data
+ * @throws Exception If failed.
+ */
+ static void saveNodeResult(String futPath, ZookeeperClient client, long nodeOrder, byte[] data) throws Exception {
+ client.createIfNeeded(futPath + "/" + nodeOrder, data, CreateMode.PERSISTENT);
+ }
+
+ /**
+ * @param futPath
+ * @param client
+ * @param nodeOrder
+ * @return Node result data.
+ * @throws Exception If fai.ed
+ */
+ static byte[] readNodeResult(String futPath, ZookeeperClient client, long nodeOrder) throws Exception {
+ return client.getData(futPath + "/" + nodeOrder);
+ }
+
+ /**
+ * @param futResPath Result path.
+ * @param client Client.
+ * @param data Result data.
+ * @throws Exception If failed.
+ */
+ static void saveResult(String futResPath, ZookeeperClient client, byte[] data) throws Exception {
+ client.createIfNeeded(futResPath, data, CreateMode.PERSISTENT);
+ }
+
+ static byte[] readResult(ZookeeperClient client, ZkIgnitePaths paths, UUID futId) throws Exception {
+ return client.getData(paths.distributedFutureResultPath(futId));
+ }
+
+ /**
+ * @param client Client.
+ * @param paths Paths utils.
+ * @param futId Future ID.
+ * @param log Ignite Logger.
+ * @throws Exception If failed.
+ */
+ static void deleteFutureData(ZookeeperClient client,
+ ZkIgnitePaths paths,
+ UUID futId,
+ IgniteLogger log
+ ) throws Exception {
+ // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8189
+ String evtDir = paths.distributedFutureBasePath(futId);
+
+ try {
+ client.deleteAll(evtDir,
+ client.getChildren(evtDir),
+ -1);
+ }
+ catch (KeeperException.NoNodeException e) {
+ U.log(log, "Node for deletion was not found: " + e.getPath());
+
+ // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8189
+ }
+
+ client.deleteIfExists(evtDir, -1);
+
+ client.deleteIfExists(paths.distributedFutureResultPath(futId), -1);
+ }
+
+ /**
+ * @param top Current topology.
+ * @throws Exception If listener call failed.
+ */
+ void onTopologyChange(ZkClusterNodes top) throws Exception {
+ if (remainingNodes.isEmpty())
+ return;
+
+ for (Iterator<Long> it = remainingNodes.iterator(); it.hasNext();) {
+ Long nodeOrder = it.next();
+
+ if (!top.nodesByOrder.containsKey(nodeOrder)) {
+ it.remove();
+
+ int remaining = remainingNodes.size();
+
+ if (log.isInfoEnabled()) {
+ log.info("ZkDistributedCollectDataFuture removed remaining failed node [node=" + nodeOrder +
+ ", remaining=" + remaining +
+ ", futPath=" + futPath + ']');
+ }
+
+ if (remaining == 0) {
+ completeAndNotifyListener();
+
+ break;
+ }
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ class NodeResultsWatcher extends ZkAbstractWatcher implements AsyncCallback.Children2Callback {
+ /**
+ * @param rtState Runtime state.
+ * @param impl Discovery impl.
+ */
+ NodeResultsWatcher(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+ super(rtState, impl);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void process0(WatchedEvent evt) {
+ if (evt.getType() == Watcher.Event.EventType.NodeChildrenChanged)
+ rtState.zkClient.getChildrenAsync(evt.getPath(), this, this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+ if (!onProcessStart())
+ return;
+
+ try {
+ if (!isDone()) {
+ assert rc == 0 : KeeperException.Code.get(rc);
+
+ for (int i = 0; i < children.size(); i++) {
+ Long nodeOrder = Long.parseLong(children.get(i));
+
+ if (remainingNodes.remove(nodeOrder)) {
+ int remaining = remainingNodes.size();
+
+ if (log.isInfoEnabled()) {
+ log.info("ZkDistributedCollectDataFuture added new result [node=" + nodeOrder +
+ ", remaining=" + remaining +
+ ", futPath=" + path + ']');
+ }
+
+ if (remaining == 0)
+ completeAndNotifyListener();
+ }
+ }
+ }
+
+ onProcessEnd();
+ }
+ catch (Throwable e) {
+ onProcessError(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkForceNodeFailMessage.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkForceNodeFailMessage.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkForceNodeFailMessage.java
new file mode 100644
index 0000000..de7291c
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkForceNodeFailMessage.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Zk Force Node Fail Message.
+ */
+public class ZkForceNodeFailMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final long nodeInternalId;
+
+ /** */
+ final String warning;
+
+ /**
+ * @param nodeInternalId Node ID.
+ * @param warning Warning to be displayed on all nodes.
+ */
+ ZkForceNodeFailMessage(long nodeInternalId, String warning) {
+ this.nodeInternalId = nodeInternalId;
+ this.warning = warning;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isMutable() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ZkForceNodeFailMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
new file mode 100644
index 0000000..9caf00f
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.UUID;
+
+/**
+ *
+ */
+class ZkIgnitePaths {
+ /** */
+ static final String PATH_SEPARATOR = "/";
+
+ /** */
+ private static final byte CLIENT_NODE_FLAG_MASK = 0x01;
+
+ /** */
+ private static final int UUID_LEN = 36;
+
+ /** Directory to store joined node data. */
+ private static final String JOIN_DATA_DIR = "jd";
+
+ /** Directory to store new custom events. */
+ private static final String CUSTOM_EVTS_DIR = "ce";
+
+ /** Directory to store parts of multi-parts custom events. */
+ private static final String CUSTOM_EVTS_PARTS_DIR = "cp";
+
+ /** Directory to store acknowledge messages for custom events. */
+ private static final String CUSTOM_EVTS_ACKS_DIR = "ca";
+
+ /** Directory to store EPHEMERAL znodes for alive cluster nodes. */
+ static final String ALIVE_NODES_DIR = "n";
+
+ /** Path to store discovery events {@link ZkDiscoveryEventsData}. */
+ private static final String DISCO_EVENTS_PATH = "e";
+
+ /** */
+ final String clusterDir;
+
+ /** */
+ final String aliveNodesDir;
+
+ /** */
+ final String joinDataDir;
+
+ /** */
+ final String evtsPath;
+
+ /** */
+ final String customEvtsDir;
+
+ /** */
+ final String customEvtsPartsDir;
+
+ /** */
+ final String customEvtsAcksDir;
+
+ /**
+ * @param zkRootPath Base Zookeeper directory for all Ignite nodes.
+ */
+ ZkIgnitePaths(String zkRootPath) {
+ clusterDir = zkRootPath;
+
+ aliveNodesDir = zkPath(ALIVE_NODES_DIR);
+ joinDataDir = zkPath(JOIN_DATA_DIR);
+ evtsPath = zkPath(DISCO_EVENTS_PATH);
+ customEvtsDir = zkPath(CUSTOM_EVTS_DIR);
+ customEvtsPartsDir = zkPath(CUSTOM_EVTS_PARTS_DIR);
+ customEvtsAcksDir = zkPath(CUSTOM_EVTS_ACKS_DIR);
+ }
+
+ /**
+ * @param path Relative path.
+ * @return Full path.
+ */
+ private String zkPath(String path) {
+ return clusterDir + "/" + path;
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @param prefixId Unique prefix ID.
+ * @return Path.
+ */
+ String joiningNodeDataPath(UUID nodeId, UUID prefixId) {
+ return joinDataDir + '/' + prefixId + ":" + nodeId.toString();
+ }
+
+ /**
+ * @param path Alive node zk path.
+ * @return Node internal ID.
+ */
+ static long aliveInternalId(String path) {
+ int idx = path.lastIndexOf('|');
+
+ return Integer.parseInt(path.substring(idx + 1));
+ }
+
+ /**
+ * @param prefix Node unique path prefix.
+ * @param node Node.
+ * @return Path.
+ */
+ String aliveNodePathForCreate(String prefix, ZookeeperClusterNode node) {
+ byte flags = 0;
+
+ if (node.isClient())
+ flags |= CLIENT_NODE_FLAG_MASK;
+
+ return aliveNodesDir + "/" + prefix + ":" + node.id() + ":" + encodeFlags(flags) + "|";
+ }
+
+ /**
+ * @param path Alive node zk path.
+ * @return {@code True} if node is client.
+ */
+ static boolean aliveNodeClientFlag(String path) {
+ return (aliveFlags(path) & CLIENT_NODE_FLAG_MASK) != 0;
+ }
+
+ /**
+ * @param path Alive node zk path.
+ * @return Node ID.
+ */
+ static UUID aliveNodePrefixId(String path) {
+ return UUID.fromString(path.substring(0, ZkIgnitePaths.UUID_LEN));
+ }
+
+ /**
+ * @param path Alive node zk path.
+ * @return Node ID.
+ */
+ static UUID aliveNodeId(String path) {
+ // <uuid prefix>:<node id>:<flags>|<alive seq>
+ int startIdx = ZkIgnitePaths.UUID_LEN + 1;
+
+ String idStr = path.substring(startIdx, startIdx + ZkIgnitePaths.UUID_LEN);
+
+ return UUID.fromString(idStr);
+ }
+
+ /**
+ * @param path Event zk path.
+ * @return Event sequence number.
+ */
+ static int customEventSequence(String path) {
+ int idx = path.lastIndexOf('|');
+
+ return Integer.parseInt(path.substring(idx + 1));
+ }
+
+ /**
+ * @param path Custom event zl path.
+ * @return Event node ID.
+ */
+ static UUID customEventSendNodeId(String path) {
+ // <uuid prefix>:<node id>:<partCnt>|<seq>
+ int startIdx = ZkIgnitePaths.UUID_LEN + 1;
+
+ String idStr = path.substring(startIdx, startIdx + ZkIgnitePaths.UUID_LEN);
+
+ return UUID.fromString(idStr);
+ }
+
+ /**
+ * @param path Event path.
+ * @return Event unique prefix.
+ */
+ static String customEventPrefix(String path) {
+ // <uuid prefix>:<node id>:<partCnt>|<seq>
+
+ return path.substring(0, ZkIgnitePaths.UUID_LEN);
+ }
+
+ /**
+ * @param path Custom event zl path.
+ * @return Event node ID.
+ */
+ static int customEventPartsCount(String path) {
+ // <uuid prefix>:<node id>:<partCnt>|<seq>
+ int startIdx = 2 * ZkIgnitePaths.UUID_LEN + 2;
+
+ String cntStr = path.substring(startIdx, startIdx + 4);
+
+ int partCnt = Integer.parseInt(cntStr);
+
+ assert partCnt >= 1 : partCnt;
+
+ return partCnt;
+ }
+
+ /**
+ * @param prefix Prefix.
+ * @param nodeId Node ID.
+ * @param partCnt Parts count.
+ * @return Path.
+ */
+ String createCustomEventPath(String prefix, UUID nodeId, int partCnt) {
+ return customEvtsDir + "/" + prefix + ":" + nodeId + ":" + String.format("%04d", partCnt) + '|';
+ }
+
+ /**
+ * @param prefix Prefix.
+ * @param nodeId Node ID.
+ * @return Path.
+ */
+ String customEventPartsBasePath(String prefix, UUID nodeId) {
+ return customEvtsPartsDir + "/" + prefix + ":" + nodeId + ":";
+ }
+
+ /**
+ * @param prefix Prefix.
+ * @param nodeId Node ID.
+ * @param part Part number.
+ * @return Path.
+ */
+ String customEventPartPath(String prefix, UUID nodeId, int part) {
+ return customEventPartsBasePath(prefix, nodeId) + String.format("%04d", part);
+ }
+
+ /**
+ * @param evtId Event ID.
+ * @return Event zk path.
+ */
+ String joinEventDataPathForJoined(long evtId) {
+ return evtsPath + "/fj-" + evtId;
+ }
+
+ /**
+ * @param topVer Event topology version.
+ * @return Event zk path.
+ */
+ String joinEventSecuritySubjectPath(long topVer) {
+ return evtsPath + "/s-" + topVer;
+ }
+
+ /**
+ * @param origEvtId ID of original custom event.
+ * @return Path for custom event ack.
+ */
+ String ackEventDataPath(long origEvtId) {
+ assert origEvtId != 0;
+
+ return customEvtsAcksDir + "/" + String.valueOf(origEvtId);
+ }
+
+ /**
+ * @param id Future ID.
+ * @return Future path.
+ */
+ String distributedFutureBasePath(UUID id) {
+ return evtsPath + "/f-" + id;
+ }
+
+ /**
+ * @param id Future ID.
+ * @return Future path.
+ */
+ String distributedFutureResultPath(UUID id) {
+ return evtsPath + "/fr-" + id;
+ }
+
+ /**
+ * @param flags Flags.
+ * @return Flags string.
+ */
+ private static String encodeFlags(byte flags) {
+ int intVal = flags + 128;
+
+ String str = Integer.toString(intVal, 16);
+
+ if (str.length() == 1)
+ str = '0' + str;
+
+ assert str.length() == 2 : str;
+
+ return str;
+ }
+
+ /**
+ * @param path Alive node zk path.
+ * @return Flags.
+ */
+ private static byte aliveFlags(String path) {
+ int startIdx = path.lastIndexOf(':') + 1;
+
+ String flagsStr = path.substring(startIdx, startIdx + 2);
+
+ return (byte)(Integer.parseInt(flagsStr, 16) - 128);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalJoinErrorMessage.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalJoinErrorMessage.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalJoinErrorMessage.java
new file mode 100644
index 0000000..a73312c
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalJoinErrorMessage.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ *
+ */
+class ZkInternalJoinErrorMessage implements ZkInternalMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ transient boolean notifyNode = true;
+
+ /** */
+ final long nodeInternalId;
+
+ /** */
+ final String err;
+
+ /**
+ * @param nodeInternalId Joining node internal ID.
+ * @param err Error message.
+ */
+ ZkInternalJoinErrorMessage(long nodeInternalId, String err) {
+ this.nodeInternalId = nodeInternalId;
+ this.err = err;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalMessage.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalMessage.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalMessage.java
new file mode 100644
index 0000000..c1d56f0
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalMessage.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+
+/**
+ *
+ */
+interface ZkInternalMessage extends Serializable {
+ // No-op.
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinEventDataForJoined.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinEventDataForJoined.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinEventDataForJoined.java
new file mode 100644
index 0000000..e4ae4ba0
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinEventDataForJoined.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+class ZkJoinEventDataForJoined implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final List<ZookeeperClusterNode> top;
+
+ /** */
+ private final Map<Long, byte[]> discoData;
+
+ /** */
+ private final Map<Long, Long> dupDiscoData;
+
+ /**
+ * @param top Topology.
+ * @param discoData Discovery data.
+ */
+ ZkJoinEventDataForJoined(List<ZookeeperClusterNode> top, Map<Long, byte[]> discoData, @Nullable Map<Long, Long> dupDiscoData) {
+ assert top != null;
+ assert discoData != null && !discoData.isEmpty();
+
+ this.top = top;
+ this.discoData = discoData;
+ this.dupDiscoData = dupDiscoData;
+ }
+
+ byte[] discoveryDataForNode(long nodeOrder) {
+ assert discoData != null;
+
+ byte[] dataBytes = discoData.get(nodeOrder);
+
+ if (dataBytes != null)
+ return dataBytes;
+
+ assert dupDiscoData != null;
+
+ Long dupDataNode = dupDiscoData.get(nodeOrder);
+
+ assert dupDataNode != null;
+
+ dataBytes = discoData.get(dupDataNode);
+
+ assert dataBytes != null;
+
+ return dataBytes;
+ }
+
+ /**
+ * @return Current topology.
+ */
+ List<ZookeeperClusterNode> topology() {
+ assert top != null;
+
+ return top;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinedNodeEvtData.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinedNodeEvtData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinedNodeEvtData.java
new file mode 100644
index 0000000..3c367cf
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinedNodeEvtData.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.UUID;
+
+/**
+ * Zk Joined Node Evt Data.
+ */
+public class ZkJoinedNodeEvtData implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final long topVer;
+
+ /** */
+ final long joinedInternalId;
+
+ /** */
+ final UUID nodeId;
+
+ /** */
+ final int joinDataPartCnt;
+
+ /** */
+ final int secSubjPartCnt;
+
+ /** */
+ final UUID joinDataPrefixId;
+
+ /** */
+ transient ZkJoiningNodeData joiningNodeData;
+
+ /**
+ * @param topVer Topology version for node join event.
+ * @param nodeId Joined node ID.
+ * @param joinedInternalId Joined node internal ID.
+ * @param joinDataPrefixId Join data unique prefix.
+ * @param joinDataPartCnt Join data part count.
+ * @param secSubjPartCnt Security subject part count.
+ */
+ ZkJoinedNodeEvtData(
+ long topVer,
+ UUID nodeId,
+ long joinedInternalId,
+ UUID joinDataPrefixId,
+ int joinDataPartCnt,
+ int secSubjPartCnt)
+ {
+ this.topVer = topVer;
+ this.nodeId = nodeId;
+ this.joinedInternalId = joinedInternalId;
+ this.joinDataPrefixId = joinDataPrefixId;
+ this.joinDataPartCnt = joinDataPartCnt;
+ this.secSubjPartCnt = secSubjPartCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "ZkJoinedNodeData [id=" + nodeId + ", order=" + topVer + ']';
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoiningNodeData.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoiningNodeData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoiningNodeData.java
new file mode 100644
index 0000000..ff8311d
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoiningNodeData.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.Map;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+class ZkJoiningNodeData implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private int partCnt;
+
+ /** */
+ @GridToStringInclude
+ private ZookeeperClusterNode node;
+
+ /** */
+ @GridToStringInclude
+ private Map<Integer, Serializable> discoData;
+
+ /**
+ * @param partCnt Number of parts in multi-parts message.
+ */
+ ZkJoiningNodeData(int partCnt) {
+ this.partCnt = partCnt;
+ }
+
+ /**
+ * @param node Node.
+ * @param discoData Discovery data.
+ */
+ ZkJoiningNodeData(ZookeeperClusterNode node, Map<Integer, Serializable> discoData) {
+ assert node != null && node.id() != null : node;
+ assert discoData != null;
+
+ this.node = node;
+ this.discoData = discoData;
+ }
+
+ /**
+ * @return Number of parts in multi-parts message.
+ */
+ int partCount() {
+ return partCnt;
+ }
+
+ /**
+ * @return Node.
+ */
+ ZookeeperClusterNode node() {
+ return node;
+ }
+
+ /**
+ * @return Discovery data.
+ */
+ Map<Integer, Serializable> discoveryData() {
+ return discoData;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ZkJoiningNodeData.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNoServersMessage.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNoServersMessage.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNoServersMessage.java
new file mode 100644
index 0000000..626fe74
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNoServersMessage.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+class ZkNoServersMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isMutable() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ZkNoServersMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNodeValidateResult.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNodeValidateResult.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNodeValidateResult.java
new file mode 100644
index 0000000..2abfee3
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNodeValidateResult.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ *
+ */
+class ZkNodeValidateResult {
+ /** */
+ String err;
+
+ /** */
+ byte[] secSubjZipBytes;
+
+ /**
+ * @param err Error.
+ */
+ ZkNodeValidateResult(String err) {
+ this.err = err;
+ }
+
+ /**
+ * @param secSubjZipBytes Marshalled security subject.
+ */
+ ZkNodeValidateResult(byte[] secSubjZipBytes) {
+ this.secSubjZipBytes = secSubjZipBytes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRunnable.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRunnable.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRunnable.java
new file mode 100644
index 0000000..965bdc0
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRunnable.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ * Zk Runnable.
+ */
+public abstract class ZkRunnable extends ZkAbstractCallabck implements Runnable {
+ /**
+ * @param rtState Runtime state.
+ * @param impl Discovery impl.
+ */
+ ZkRunnable(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+ super(rtState, impl);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ if (!onProcessStart())
+ return;
+
+ try {
+ run0();
+
+ onProcessEnd();
+ }
+ catch (Throwable e) {
+ onProcessError(e);
+ }
+ }
+
+ /**
+ *
+ */
+ protected abstract void run0() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java
new file mode 100644
index 0000000..cb04ac3
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.List;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.Watcher;
+
+/**
+ *
+ */
+class ZkRuntimeState {
+ /** */
+ ZkWatcher watcher;
+
+ /** */
+ ZkAliveNodeDataWatcher aliveNodeDataWatcher;
+
+ /** */
+ volatile Exception errForClose;
+
+ /** */
+ final boolean prevJoined;
+
+ /** */
+ ZookeeperClient zkClient;
+
+ /** */
+ long internalOrder;
+
+ /** */
+ int joinDataPartCnt;
+
+ /** */
+ long gridStartTime;
+
+ /** */
+ volatile boolean joined;
+
+ /** */
+ ZkDiscoveryEventsData evtsData;
+
+ /** */
+ boolean crd;
+
+ /** */
+ String locNodeZkPath;
+
+ /** */
+ final ZkAliveNodeData locNodeInfo = new ZkAliveNodeData();
+
+ /** */
+ int procEvtCnt;
+
+ /** */
+ final ZkClusterNodes top = new ZkClusterNodes();
+
+ /** */
+ List<ClusterNode> commErrProcNodes;
+
+ /** Timeout callback registering watcher for join error
+ * (set this watcher after timeout as a minor optimization).
+ */
+ ZkTimeoutObject joinErrTo;
+
+ /** Timeout callback set to wait for join timeout. */
+ ZkTimeoutObject joinTo;
+
+ /** Timeout callback to update processed events counter. */
+ ZkTimeoutObject procEvtsUpdateTo;
+
+ /** */
+ boolean updateAlives;
+
+ /**
+ * @param prevJoined {@code True} if joined topology before reconnect attempt.
+ */
+ ZkRuntimeState(boolean prevJoined) {
+ this.prevJoined = prevJoined;
+ }
+
+ /**
+ * @param watcher Watcher.
+ * @param aliveNodeDataWatcher Alive nodes data watcher.
+ */
+ void init(ZkWatcher watcher, ZkAliveNodeDataWatcher aliveNodeDataWatcher) {
+ this.watcher = watcher;
+ this.aliveNodeDataWatcher = aliveNodeDataWatcher;
+ }
+
+ /**
+ * @param err Error.
+ */
+ void onCloseStart(Exception err) {
+ assert err != null;
+
+ errForClose = err;
+
+ ZookeeperClient zkClient = this.zkClient;
+
+ if (zkClient != null)
+ zkClient.onCloseStart();
+ }
+
+ /**
+ *
+ */
+ interface ZkWatcher extends Watcher, AsyncCallback.Children2Callback, AsyncCallback.DataCallback {
+ // No-op.
+ }
+
+ /**
+ *
+ */
+ interface ZkAliveNodeDataWatcher extends Watcher, AsyncCallback.DataCallback {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkTimeoutObject.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkTimeoutObject.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkTimeoutObject.java
new file mode 100644
index 0000000..4d3d5b4
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkTimeoutObject.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+
+/**
+ *
+ */
+abstract class ZkTimeoutObject implements IgniteSpiTimeoutObject {
+ /** */
+ private final IgniteUuid id = IgniteUuid.randomUuid();
+
+ /** */
+ private final long endTime;
+
+ /** */
+ volatile boolean cancelled;
+
+ /**
+ * @param timeout Timeout.
+ */
+ ZkTimeoutObject(long timeout) {
+ long endTime = timeout >= 0 ? System.currentTimeMillis() + timeout : Long.MAX_VALUE;
+
+ this.endTime = endTime >= 0 ? endTime : Long.MAX_VALUE;
+ }
+
+ /** {@inheritDoc} */
+ @Override public final IgniteUuid id() {
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public final long endTime() {
+ return endTime;
+ }
+}
[39/54] [abbrv] ignite git commit: IGNITE-6679 Clean up some
deprecated cache metrics
Posted by ag...@apache.org.
IGNITE-6679 Clean up some deprecated cache metrics
Signed-off-by: Anton Vinogradov <av...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/32fc6c3c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/32fc6c3c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/32fc6c3c
Branch: refs/heads/ignite-6083
Commit: 32fc6c3c1b013c8477acb9cf51d8326a13307c64
Parents: 14402e4
Author: NSAmelchev <ns...@gmail.com>
Authored: Wed Apr 11 16:59:59 2018 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Wed Apr 11 16:59:59 2018 +0300
----------------------------------------------------------------------
.../src/main/java/org/apache/ignite/cache/CacheMetrics.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/32fc6c3c/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java b/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
index 0b1cb87..c466bee 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
@@ -262,6 +262,7 @@ public interface CacheMetrics {
*
* @return Current size of evict queue.
*/
+ @Deprecated
public int getDhtEvictQueueCurrentSize();
/**
@@ -283,6 +284,7 @@ public interface CacheMetrics {
*
* @return Committed transaction queue size.
*/
+ @Deprecated
public int getTxCommitQueueSize();
/**
@@ -290,6 +292,7 @@ public interface CacheMetrics {
*
* @return Prepared transaction queue size.
*/
+ @Deprecated
public int getTxPrepareQueueSize();
/**
@@ -297,6 +300,7 @@ public interface CacheMetrics {
*
* @return Start version counts map size.
*/
+ @Deprecated
public int getTxStartVersionCountsSize();
/**
@@ -332,6 +336,7 @@ public interface CacheMetrics {
*
* @return Committed DHT transaction queue size.
*/
+ @Deprecated
public int getTxDhtCommitQueueSize();
/**
@@ -339,6 +344,7 @@ public interface CacheMetrics {
*
* @return Prepared DHT transaction queue size.
*/
+ @Deprecated
public int getTxDhtPrepareQueueSize();
/**
@@ -346,6 +352,7 @@ public interface CacheMetrics {
*
* @return DHT start version counts map size.
*/
+ @Deprecated
public int getTxDhtStartVersionCountsSize();
/**
[45/54] [abbrv] ignite git commit: IGNITE-7824: Wrong warning message
fixed. - Fixes #3788.
Posted by ag...@apache.org.
IGNITE-7824: Wrong warning message fixed. - Fixes #3788.
Signed-off-by: Nikolay Izhikov <ni...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/706e7f81
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/706e7f81
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/706e7f81
Branch: refs/heads/ignite-6083
Commit: 706e7f81b44ca9b3a01ffd673e5db61578123b64
Parents: df6356d
Author: NSAmelchev <ns...@gmail.com>
Authored: Thu Apr 12 13:21:15 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Thu Apr 12 13:21:15 2018 +0300
----------------------------------------------------------------------
.../src/main/java/org/apache/ignite/internal/IgniteKernal.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/706e7f81/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 1cb07b9..795601e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1439,7 +1439,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
long safeToUse = ram - Math.max(4L << 30, (long)(ram * 0.2));
if (total > safeToUse) {
- U.quietAndWarn(log, "Nodes started on local machine require more than 20% of physical RAM what can " +
+ U.quietAndWarn(log, "Nodes started on local machine require more than 80% of physical RAM what can " +
"lead to significant slowdown due to swapping (please decrease JVM heap size, data region " +
"size or checkpoint buffer size) [required=" + (total >> 20) + "MB, available=" +
(ram >> 20) + "MB]");
[51/54] [abbrv] ignite git commit: IGNITE-7983: NPE fixed in
transactions
Posted by ag...@apache.org.
IGNITE-7983: NPE fixed in transactions
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/08a700db
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/08a700db
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/08a700db
Branch: refs/heads/ignite-6083
Commit: 08a700db31604d33959e3b5aa3f709dbf66a878b
Parents: adaedb4
Author: Andrey Kuznetsov <st...@gmail.com>
Authored: Thu Apr 12 21:23:28 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Thu Apr 12 21:23:28 2018 +0300
----------------------------------------------------------------------
.../processors/cache/distributed/near/GridNearTxLocal.java | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/08a700db/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index 33f84f0..fc8a9a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -3916,6 +3916,15 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
throw new GridClosureException(e);
}
+ if (isRollbackOnly()) {
+ if (timedOut())
+ throw new GridClosureException(new IgniteTxTimeoutCheckedException(
+ "Transaction has been timed out: " + GridNearTxLocal.this));
+ else
+ throw new GridClosureException(new IgniteTxRollbackCheckedException(
+ "Transaction has been rolled back: " + GridNearTxLocal.this));
+ }
+
return map;
}
},
[42/54] [abbrv] ignite git commit: IGNITE-8042: .NET thin client:
authentication support. This closes #3790.
Posted by ag...@apache.org.
IGNITE-8042: .NET thin client: authentication support. This closes #3790.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/67023a88
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/67023a88
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/67023a88
Branch: refs/heads/ignite-6083
Commit: 67023a88b8c9cf647196b6deb1759b4d01e33890
Parents: e333f30
Author: devozerov <vo...@gridgain.com>
Authored: Thu Apr 12 10:37:36 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Apr 12 10:37:36 2018 +0300
----------------------------------------------------------------------
.../client/ClientAuthenticationException.java | 16 +-
.../internal/client/thin/TcpClientChannel.java | 9 +-
.../odbc/ClientListenerNioListener.java | 19 +++
.../client/ClientConnectionContext.java | 8 +-
.../platform/client/ClientStatus.java | 5 +-
.../Client/ClientConnectionTest.cs | 161 +++++++++++++++++++
.../Client/ClientStatusCode.cs | 12 +-
.../Client/IgniteClientConfiguration.cs | 13 ++
.../IgniteClientConfigurationSection.xsd | 10 ++
.../Impl/Client/ClientProtocolVersion.cs | 22 ++-
.../Impl/Client/ClientSocket.cs | 85 +++++++++-
11 files changed, 329 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/67023a88/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java b/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
index 0c24db8..526690a 100644
--- a/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
+++ b/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
@@ -24,22 +24,10 @@ public class ClientAuthenticationException extends ClientException {
/** Serial version uid. */
private static final long serialVersionUID = 0L;
- /** Message. */
- private static final String MSG = "Invalid user name or password";
-
/**
* Default constructor.
*/
- public ClientAuthenticationException() {
- super(MSG);
- }
-
- /**
- * Constructs a new exception with the specified cause.
- *
- * @param cause the cause.
- */
- public ClientAuthenticationException(Throwable cause) {
- super(MSG, cause);
+ public ClientAuthenticationException(String msg) {
+ super(msg);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67023a88/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
index 8e8294f..10dc865 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
@@ -272,8 +272,13 @@ class TcpClientChannel implements ClientChannel {
try (BinaryReaderExImpl r = new BinaryReaderExImpl(null, res, null, true)) {
String err = r.readString();
- if (err != null && err.toUpperCase().matches(".*USER.*INCORRECT.*"))
- throw new ClientAuthenticationException();
+ int errCode = ClientStatus.FAILED;
+
+ if (res.remaining() > 0)
+ errCode = r.readInt();
+
+ if (errCode == ClientStatus.AUTH_FAILED)
+ throw new ClientAuthenticationException(err);
else if (ver.equals(srvVer))
throw new ClientProtocolError(err);
else if (!supportedVers.contains(srvVer) ||
http://git-wip-us.apache.org/repos/asf/ignite/blob/67023a88/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java
index 53b14d7..407c1a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java
@@ -26,9 +26,11 @@ import org.apache.ignite.internal.binary.BinaryWriterExImpl;
import org.apache.ignite.internal.binary.streams.BinaryHeapInputStream;
import org.apache.ignite.internal.binary.streams.BinaryHeapOutputStream;
import org.apache.ignite.internal.binary.streams.BinaryInputStream;
+import org.apache.ignite.internal.processors.authentication.IgniteAccessControlException;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcConnectionContext;
import org.apache.ignite.internal.processors.odbc.odbc.OdbcConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
+import org.apache.ignite.internal.processors.platform.client.ClientStatus;
import org.apache.ignite.internal.util.GridSpinBusyLock;
import org.apache.ignite.internal.util.nio.GridNioServerListenerAdapter;
import org.apache.ignite.internal.util.nio.GridNioSession;
@@ -228,6 +230,18 @@ public class ClientListenerNioListener extends GridNioServerListenerAdapter<byte
connCtx.handler().writeHandshake(writer);
}
+ catch (IgniteAccessControlException authEx) {
+ writer.writeBoolean(false);
+
+ writer.writeShort((short)0);
+ writer.writeShort((short)0);
+ writer.writeShort((short)0);
+
+ writer.doWriteString(authEx.getMessage());
+
+ if (ver.compareTo(ClientConnectionContext.VER_1_1_0) >= 0)
+ writer.writeInt(ClientStatus.AUTH_FAILED);
+ }
catch (IgniteCheckedException e) {
U.warn(log, "Error during handshake [rmtAddr=" + ses.remoteAddress() + ", msg=" + e.getMessage() + ']');
@@ -239,10 +253,15 @@ public class ClientListenerNioListener extends GridNioServerListenerAdapter<byte
currVer = connCtx.currentVersion();
writer.writeBoolean(false);
+
writer.writeShort(currVer.major());
writer.writeShort(currVer.minor());
writer.writeShort(currVer.maintenance());
+
writer.doWriteString(e.getMessage());
+
+ if (ver.compareTo(ClientConnectionContext.VER_1_1_0) >= 0)
+ writer.writeInt(ClientStatus.FAILED);
}
ses.send(writer.array());
http://git-wip-us.apache.org/repos/asf/ignite/blob/67023a88/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
index 061aab3..056ea83 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
@@ -44,10 +44,10 @@ import static org.apache.ignite.plugin.security.SecuritySubjectType.REMOTE_CLIEN
*/
public class ClientConnectionContext implements ClientListenerConnectionContext {
/** Version 1.0.0. */
- private static final ClientListenerProtocolVersion VER_1_0_0 = ClientListenerProtocolVersion.create(1, 0, 0);
+ public static final ClientListenerProtocolVersion VER_1_0_0 = ClientListenerProtocolVersion.create(1, 0, 0);
/** Version 1.1.0. */
- private static final ClientListenerProtocolVersion VER_1_1_0 = ClientListenerProtocolVersion.create(1, 1, 0);
+ public static final ClientListenerProtocolVersion VER_1_1_0 = ClientListenerProtocolVersion.create(1, 1, 0);
/** Supported versions. */
private static final Collection<ClientListenerProtocolVersion> SUPPORTED_VERS = Arrays.asList(VER_1_1_0, VER_1_0_0);
@@ -144,12 +144,12 @@ public class ClientConnectionContext implements ClientListenerConnectionContext
authCtx = thirdPartyAuthentication(user, pwd).authorizationContext();
else if (kernalCtx.authentication().enabled()) {
if (user == null || user.length() == 0)
- throw new IgniteCheckedException("Unauthenticated sessions are prohibited.");
+ throw new IgniteAccessControlException("Unauthenticated sessions are prohibited.");
authCtx = kernalCtx.authentication().authenticate(user, pwd);
if (authCtx == null)
- throw new IgniteCheckedException("Unknown authentication error.");
+ throw new IgniteAccessControlException("Unknown authentication error.");
}
handler = new ClientRequestHandler(this, authCtx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/67023a88/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
index b8dfb1f..e63812c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
@@ -49,6 +49,9 @@ public final class ClientStatus {
/** Resource does not exist. */
public static final int RESOURCE_DOES_NOT_EXIST = 1011;
- /** Resource does not exist. */
+ /** Authorization failure. */
public static final int SECURITY_VIOLATION = 1012;
+
+ /** Authentication failed. */
+ public static final int AUTH_FAILED = 2000;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67023a88/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
index 9da9a03..2ea17a8 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
@@ -19,13 +19,17 @@ namespace Apache.Ignite.Core.Tests.Client
{
using System;
using System.Collections.Generic;
+ using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
+ using Apache.Ignite.Core.Cache.Configuration;
+ using Apache.Ignite.Core.Cache.Query;
using Apache.Ignite.Core.Client;
+ using Apache.Ignite.Core.Client.Cache;
using Apache.Ignite.Core.Configuration;
using NUnit.Framework;
@@ -34,6 +38,18 @@ namespace Apache.Ignite.Core.Tests.Client
/// </summary>
public class ClientConnectionTest
{
+ /** Temp dir for WAL. */
+ private readonly string _tempDir = TestUtils.GetTempDirectoryName();
+
+ /// <summary>
+ /// Sets up the test.
+ /// </summary>
+ [SetUp]
+ public void SetUp()
+ {
+ TestUtils.ClearWorkDir();
+ }
+
/// <summary>
/// Fixture tear down.
/// </summary>
@@ -41,6 +57,13 @@ namespace Apache.Ignite.Core.Tests.Client
public void TearDown()
{
Ignition.StopAll(true);
+
+ if (Directory.Exists(_tempDir))
+ {
+ Directory.Delete(_tempDir, true);
+ }
+
+ TestUtils.ClearWorkDir();
}
/// <summary>
@@ -55,6 +78,107 @@ namespace Apache.Ignite.Core.Tests.Client
}
/// <summary>
+ /// Tests that empty username or password are not allowed.
+ /// </summary>
+ [Test]
+ public void TestAuthenticationEmptyCredentials()
+ {
+ using (Ignition.Start(SecureServerConfig()))
+ {
+ var cliCfg = SecureClientConfig();
+
+ cliCfg.Password = null;
+ var ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
+ Assert.IsTrue(ex.Message.StartsWith("IgniteClientConfiguration.Password cannot be null"));
+
+ cliCfg.Password = "";
+ ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
+ Assert.IsTrue(ex.Message.StartsWith("IgniteClientConfiguration.Password cannot be empty"));
+
+ cliCfg.Password = "ignite";
+
+ cliCfg.Username = null;
+ ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
+ Assert.IsTrue(ex.Message.StartsWith("IgniteClientConfiguration.Username cannot be null"));
+
+ cliCfg.Username = "";
+ ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
+ Assert.IsTrue(ex.Message.StartsWith("IgniteClientConfiguration.Username cannot be empty"));
+ }
+ }
+
+ /// <summary>
+ /// Test invalid username or password.
+ /// </summary>
+ [Test]
+ public void TestAuthenticationInvalidCredentials()
+ {
+ using (Ignition.Start(SecureServerConfig()))
+ {
+ var cliCfg = SecureClientConfig();
+
+ cliCfg.Username = "invalid";
+
+ var ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
+ Assert.True(ex.StatusCode == ClientStatusCode.AuthenticationFailed);
+
+ cliCfg.Username = "ignite";
+ cliCfg.Password = "invalid";
+
+ ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
+ Assert.True(ex.StatusCode == ClientStatusCode.AuthenticationFailed);
+ }
+ }
+
+ /// <summary>
+ /// Test authentication.
+ /// </summary>
+ [Test]
+ public void TestAuthentication()
+ {
+ using (var srv = Ignition.Start(SecureServerConfig()))
+ {
+ srv.GetCluster().SetActive(true);
+
+ using (var cli = Ignition.StartClient(SecureClientConfig()))
+ {
+ CacheClientConfiguration ccfg = new CacheClientConfiguration()
+ {
+ Name = "TestCache",
+ QueryEntities = new[]
+ {
+ new QueryEntity
+ {
+ KeyType = typeof(string),
+ ValueType = typeof(string),
+ },
+ },
+ };
+
+ ICacheClient<string, string> cache = cli.GetOrCreateCache<string, string>(ccfg);
+
+ cache.Put("key1", "val1");
+
+ cache.Query(new SqlFieldsQuery("CREATE USER \"my_User\" WITH PASSWORD 'my_Password'")).GetAll();
+ }
+
+ var cliCfg = SecureClientConfig();
+
+ cliCfg.Username = "my_User";
+ cliCfg.Password = "my_Password";
+
+ using (var cli = Ignition.StartClient(cliCfg))
+ {
+ ICacheClient<string, string> cache = cli.GetCache<string, string>("TestCache");
+
+ string val = cache.Get("key1");
+
+ Assert.True(val == "val1");
+ }
+ }
+ }
+
+ /// <summary>
/// Tests that multiple clients can connect to one server.
/// </summary>
[Test]
@@ -374,5 +498,42 @@ namespace Apache.Ignite.Core.Tests.Client
throw new Exception("SocketException not found.", origEx);
}
+
+ /// <summary>
+ /// Create server configuration with enabled authentication.
+ /// </summary>
+ /// <returns>Server configuration.</returns>
+ private IgniteConfiguration SecureServerConfig()
+ {
+ return new IgniteConfiguration(TestUtils.GetTestConfiguration())
+ {
+ AuthenticationEnabled = true,
+ DataStorageConfiguration = new DataStorageConfiguration()
+ {
+ StoragePath = Path.Combine(_tempDir, "Store"),
+ WalPath = Path.Combine(_tempDir, "WalStore"),
+ WalArchivePath = Path.Combine(_tempDir, "WalArchive"),
+ DefaultDataRegionConfiguration = new DataRegionConfiguration()
+ {
+ Name = "default",
+ PersistenceEnabled = true
+ }
+ }
+ };
+ }
+
+ /// <summary>
+ /// Create client configuration with enabled authentication.
+ /// </summary>
+ /// <returns>Client configuration.</returns>
+ private static IgniteClientConfiguration SecureClientConfig()
+ {
+ return new IgniteClientConfiguration()
+ {
+ Host = "localhost",
+ Username = "ignite",
+ Password = "ignite"
+ };
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67023a88/modules/platforms/dotnet/Apache.Ignite.Core/Client/ClientStatusCode.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Client/ClientStatusCode.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Client/ClientStatusCode.cs
index 3f5ee8e..3bdd9e1 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Client/ClientStatusCode.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Client/ClientStatusCode.cs
@@ -52,6 +52,16 @@ namespace Apache.Ignite.Core.Client
/// <summary>
/// The too many cursors (see <see cref="ClientConnectorConfiguration.MaxOpenCursorsPerConnection"/>).
/// </summary>
- TooManyCursors = 1010
+ TooManyCursors = 1010,
+
+ /// <summary>
+ /// Authorization failure.
+ /// </summary>
+ SecurityViolation = 1012,
+
+ /// <summary>
+ /// Authentication failed.
+ /// </summary>
+ AuthenticationFailed = 2000
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/67023a88/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs
index 8730f39..3252495 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs
@@ -90,6 +90,9 @@ namespace Apache.Ignite.Core.Client
BinaryProcessor = cfg.BinaryProcessor;
SslStreamFactory = cfg.SslStreamFactory;
+
+ Username = cfg.Username;
+ Password = cfg.Password;
}
/// <summary>
@@ -146,6 +149,16 @@ namespace Apache.Ignite.Core.Client
public ISslStreamFactory SslStreamFactory { get; set; }
/// <summary>
+ /// Username to be used to connect to secured cluster.
+ /// </summary>
+ public string Username { get; set; }
+
+ /// <summary>
+ /// Password to be used to connect to secured cluster.
+ /// </summary>
+ public string Password { get; set; }
+
+ /// <summary>
/// Gets or sets custom binary processor. Internal property for tests.
/// </summary>
internal IBinaryProcessor BinaryProcessor { get; set; }
http://git-wip-us.apache.org/repos/asf/ignite/blob/67023a88/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd
index 569ee6f..7e6caff 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd
@@ -237,6 +237,16 @@
<xs:documentation>Socket operation timeout. Zero or negative for infinite timeout.</xs:documentation>
</xs:annotation>
</xs:attribute>
+ <xs:attribute name="username" type="xs:string">
+ <xs:annotation>
+ <xs:documentation>Username to be used to connect to secured cluster.</xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute name="password" type="xs:string">
+ <xs:annotation>
+ <xs:documentation>Password to be used to connect to secured cluster.</xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
</xs:complexType>
</xs:element>
</xs:schema>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/67023a88/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientProtocolVersion.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientProtocolVersion.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientProtocolVersion.cs
index bfdf5a3..4fe5c71 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientProtocolVersion.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientProtocolVersion.cs
@@ -22,7 +22,7 @@ namespace Apache.Ignite.Core.Impl.Client
/// <summary>
/// Client protocol version.
/// </summary>
- internal struct ClientProtocolVersion : IEquatable<ClientProtocolVersion>
+ internal struct ClientProtocolVersion : IEquatable<ClientProtocolVersion>, IComparable<ClientProtocolVersion>
{
/** */
private readonly short _major;
@@ -68,6 +68,26 @@ namespace Apache.Ignite.Core.Impl.Client
}
/// <summary>
+ /// Compare this version to other version.
+ /// </summary>
+ /// <param name="other"></param>
+ /// <returns></returns>
+ public int CompareTo(ClientProtocolVersion other)
+ {
+ int res = Major - other.Major;
+
+ if (res == 0)
+ {
+ res = Minor - other.Minor;
+
+ if (res == 0)
+ res = Maintenance - other.Maintenance;
+ }
+
+ return res;
+ }
+
+ /// <summary>
/// Returns a value indicating whether specified instance equals to current.
/// </summary>
public bool Equals(ClientProtocolVersion other)
http://git-wip-us.apache.org/repos/asf/ignite/blob/67023a88/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
index fca5dab..27d8f0b 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
@@ -28,6 +28,7 @@ namespace Apache.Ignite.Core.Impl.Client
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
+ using System.Xml.Schema;
using Apache.Ignite.Core.Client;
using Apache.Ignite.Core.Common;
using Apache.Ignite.Core.Impl.Binary;
@@ -38,8 +39,14 @@ namespace Apache.Ignite.Core.Impl.Client
/// </summary>
internal sealed class ClientSocket : IDisposable
{
+ /** Version 1.0.0. */
+ private static readonly ClientProtocolVersion Ver100 = new ClientProtocolVersion(1, 0, 0);
+
+ /** Version 1.1.0. */
+ private static readonly ClientProtocolVersion Ver110 = new ClientProtocolVersion(1, 1, 0);
+
/** Current version. */
- private static readonly ClientProtocolVersion CurrentProtocolVersion = new ClientProtocolVersion(1, 0, 0);
+ private static readonly ClientProtocolVersion CurrentProtocolVersion = Ver110;
/** Handshake opcode. */
private const byte OpHandshake = 1;
@@ -98,7 +105,9 @@ namespace Apache.Ignite.Core.Impl.Client
_socket = Connect(clientConfiguration);
_stream = GetSocketStream(_socket, clientConfiguration);
- Handshake(version ?? CurrentProtocolVersion);
+ Validate(clientConfiguration);
+
+ Handshake(clientConfiguration, version ?? CurrentProtocolVersion);
// Check periodically if any request has timed out.
if (_timeout > TimeSpan.Zero)
@@ -112,6 +121,31 @@ namespace Apache.Ignite.Core.Impl.Client
}
/// <summary>
+ /// Validate configuration.
+ /// </summary>
+ /// <param name="cfg">Configuration.</param>
+ private void Validate(IgniteClientConfiguration cfg)
+ {
+ if (cfg.Username != null)
+ {
+ if (cfg.Username.Length == 0)
+ throw new IgniteClientException("IgniteClientConfiguration.Username cannot be empty.");
+
+ if (cfg.Password == null)
+ throw new IgniteClientException("IgniteClientConfiguration.Password cannot be null when Username is set.");
+ }
+
+ if (cfg.Password != null)
+ {
+ if (cfg.Password.Length == 0)
+ throw new IgniteClientException("IgniteClientConfiguration.Password cannot be empty.");
+
+ if (cfg.Username == null)
+ throw new IgniteClientException("IgniteClientConfiguration.Username cannot be null when Password is set.");
+ }
+ }
+
+ /// <summary>
/// Performs a send-receive operation.
/// </summary>
public T DoOutInOp<T>(ClientOp opId, Action<IBinaryStream> writeAction,
@@ -226,8 +260,10 @@ namespace Apache.Ignite.Core.Impl.Client
/// <summary>
/// Performs client protocol handshake.
/// </summary>
- private void Handshake(ClientProtocolVersion version)
+ private void Handshake(IgniteClientConfiguration clientConfiguration, ClientProtocolVersion version)
{
+ bool auth = version.CompareTo(Ver110) >= 0 && clientConfiguration.Username != null;
+
// Send request.
int messageLen;
var buf = WriteMessage(stream =>
@@ -242,10 +278,19 @@ namespace Apache.Ignite.Core.Impl.Client
// Client type: platform.
stream.WriteByte(ClientType);
- }, 12, out messageLen);
- Debug.Assert(messageLen == 12);
+ // Authentication data.
+ if (auth)
+ {
+ var writer = BinaryUtils.Marshaller.StartMarshal(stream);
+
+ writer.WriteString(clientConfiguration.Username);
+ writer.WriteString(clientConfiguration.Password);
+ BinaryUtils.Marshaller.FinishMarshal(writer);
+ }
+ }, 12, out messageLen);
+
_stream.Write(buf, 0, messageLen);
// Decode response.
@@ -253,6 +298,7 @@ namespace Apache.Ignite.Core.Impl.Client
using (var stream = new BinaryHeapStream(res))
{
+ // Read input.
var success = stream.ReadBool();
if (success)
@@ -265,9 +311,32 @@ namespace Apache.Ignite.Core.Impl.Client
var errMsg = BinaryUtils.Marshaller.Unmarshal<string>(stream);
- throw new IgniteClientException(string.Format(
- "Client handshake failed: '{0}'. Client version: {1}. Server version: {2}",
- errMsg, version, serverVersion));
+ ClientStatusCode errCode = ClientStatusCode.Fail;
+
+ if (stream.Remaining > 0)
+ {
+ errCode = (ClientStatusCode) stream.ReadInt();
+ }
+
+ // Authentication error is handled immediately.
+ if (errCode == ClientStatusCode.AuthenticationFailed)
+ {
+ throw new IgniteClientException(errMsg, null, ClientStatusCode.AuthenticationFailed);
+ }
+
+ // Re-try if possible.
+ bool retry = serverVersion.CompareTo(version) < 0 && serverVersion.Equals(Ver100);
+
+ if (retry)
+ {
+ Handshake(clientConfiguration, serverVersion);
+ }
+ else
+ {
+ throw new IgniteClientException(string.Format(
+ "Client handshake failed: '{0}'. Client version: {1}. Server version: {2}",
+ errMsg, version, serverVersion), null, errCode);
+ }
}
}
[14/54] [abbrv] ignite git commit: IGNITE-8059: Integrate decision
tree with partition based dataset.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureTest.java
new file mode 100644
index 0000000..3d11d9d
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.mse;
+
+import java.util.Random;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+
+/**
+ * Tests for {@link MSEImpurityMeasure}.
+ */
+public class MSEImpurityMeasureTest {
+ /** */
+ @Test
+ public void testImpurityOnEmptyData() {
+ MSEImpurityMeasure impurity = new MSEImpurityMeasure(0, 0, 0, 0, 0, 0);
+
+ assertEquals(0.0, impurity.impurity(), 1e-10);
+ }
+
+ /** */
+ @Test
+ public void testImpurityLeftPart() {
+ // Test on left part [1, 2, 2, 1, 1, 1].
+ MSEImpurityMeasure impurity = new MSEImpurityMeasure(8, 12, 6, 0, 0, 0);
+
+ assertEquals(1.333, impurity.impurity(), 1e-3);
+ }
+
+ /** */
+ @Test
+ public void testImpurityRightPart() {
+ // Test on right part [1, 2, 2, 1, 1, 1].
+ MSEImpurityMeasure impurity = new MSEImpurityMeasure(0, 0, 0, 8, 12, 6);
+
+ assertEquals(1.333, impurity.impurity(), 1e-3);
+ }
+
+ /** */
+ @Test
+ public void testImpurityLeftAndRightPart() {
+ // Test on left part [1, 2, 2] and right part [1, 1, 1].
+ MSEImpurityMeasure impurity = new MSEImpurityMeasure(5, 9, 3, 3, 3, 3);
+
+ assertEquals(0.666, impurity.impurity(), 1e-3);
+ }
+
+ /** */
+ @Test
+ public void testAdd() {
+ Random rnd = new Random(0);
+
+ MSEImpurityMeasure a = new MSEImpurityMeasure(
+ rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt(), rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt()
+ );
+
+ MSEImpurityMeasure b = new MSEImpurityMeasure(
+ rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt(), rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt()
+ );
+
+ MSEImpurityMeasure c = a.add(b);
+
+ assertEquals(a.getLeftY() + b.getLeftY(), c.getLeftY(), 1e-10);
+ assertEquals(a.getLeftY2() + b.getLeftY2(), c.getLeftY2(), 1e-10);
+ assertEquals(a.getLeftCnt() + b.getLeftCnt(), c.getLeftCnt());
+ assertEquals(a.getRightY() + b.getRightY(), c.getRightY(), 1e-10);
+ assertEquals(a.getRightY2() + b.getRightY2(), c.getRightY2(), 1e-10);
+ assertEquals(a.getRightCnt() + b.getRightCnt(), c.getRightCnt());
+ }
+
+ /** */
+ @Test
+ public void testSubtract() {
+ Random rnd = new Random(0);
+
+ MSEImpurityMeasure a = new MSEImpurityMeasure(
+ rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt(), rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt()
+ );
+
+ MSEImpurityMeasure b = new MSEImpurityMeasure(
+ rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt(), rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt()
+ );
+
+ MSEImpurityMeasure c = a.subtract(b);
+
+ assertEquals(a.getLeftY() - b.getLeftY(), c.getLeftY(), 1e-10);
+ assertEquals(a.getLeftY2() - b.getLeftY2(), c.getLeftY2(), 1e-10);
+ assertEquals(a.getLeftCnt() - b.getLeftCnt(), c.getLeftCnt());
+ assertEquals(a.getRightY() - b.getRightY(), c.getRightY(), 1e-10);
+ assertEquals(a.getRightY2() - b.getRightY2(), c.getRightY2(), 1e-10);
+ assertEquals(a.getRightCnt() - b.getRightCnt(), c.getRightCnt());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressorTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressorTest.java
new file mode 100644
index 0000000..001404f
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressorTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link SimpleStepFunctionCompressor}.
+ */
+public class SimpleStepFunctionCompressorTest {
+ /** */
+ @Test
+ public void testCompressSmallFunction() {
+ StepFunction<TestImpurityMeasure> function = new StepFunction<>(
+ new double[]{1, 2, 3, 4},
+ TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3, 4)
+ );
+
+ SimpleStepFunctionCompressor<TestImpurityMeasure> compressor = new SimpleStepFunctionCompressor<>(5, 0, 0);
+
+ StepFunction<TestImpurityMeasure> resFunction = compressor.compress(function);
+
+ assertArrayEquals(new double[]{1, 2, 3, 4}, resFunction.getX(), 1e-10);
+ assertArrayEquals(TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3, 4), resFunction.getY());
+ }
+
+ /** */
+ @Test
+ public void testCompressIncreasingFunction() {
+ StepFunction<TestImpurityMeasure> function = new StepFunction<>(
+ new double[]{1, 2, 3, 4, 5},
+ TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3, 4, 5)
+ );
+
+ SimpleStepFunctionCompressor<TestImpurityMeasure> compressor = new SimpleStepFunctionCompressor<>(1, 0.4, 0);
+
+ StepFunction<TestImpurityMeasure> resFunction = compressor.compress(function);
+
+ assertArrayEquals(new double[]{1, 3, 5}, resFunction.getX(), 1e-10);
+ assertArrayEquals(TestImpurityMeasure.asTestImpurityMeasures(1, 3, 5), resFunction.getY());
+ }
+
+ /** */
+ @Test
+ public void testCompressDecreasingFunction() {
+ StepFunction<TestImpurityMeasure> function = new StepFunction<>(
+ new double[]{1, 2, 3, 4, 5},
+ TestImpurityMeasure.asTestImpurityMeasures(5, 4, 3, 2, 1)
+ );
+
+ SimpleStepFunctionCompressor<TestImpurityMeasure> compressor = new SimpleStepFunctionCompressor<>(1, 0, 0.4);
+
+ StepFunction<TestImpurityMeasure> resFunction = compressor.compress(function);
+
+ assertArrayEquals(new double[]{1, 3, 5}, resFunction.getX(), 1e-10);
+ assertArrayEquals(TestImpurityMeasure.asTestImpurityMeasures(5, 3, 1), resFunction.getY());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionTest.java
new file mode 100644
index 0000000..2a0279c
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link StepFunction}.
+ */
+public class StepFunctionTest {
+ /** */
+ @Test
+ public void testAddIncreasingFunctions() {
+ StepFunction<TestImpurityMeasure> a = new StepFunction<>(
+ new double[]{1, 3, 5},
+ TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3)
+ );
+
+ StepFunction<TestImpurityMeasure> b = new StepFunction<>(
+ new double[]{0, 2, 4},
+ TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3)
+ );
+
+ StepFunction<TestImpurityMeasure> c = a.add(b);
+
+ assertArrayEquals(new double[]{0, 1, 2, 3, 4, 5}, c.getX(), 1e-10);
+ assertArrayEquals(
+ TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3, 4, 5, 6),
+ c.getY()
+ );
+ }
+
+ /** */
+ @Test
+ public void testAddDecreasingFunctions() {
+ StepFunction<TestImpurityMeasure> a = new StepFunction<>(
+ new double[]{1, 3, 5},
+ TestImpurityMeasure.asTestImpurityMeasures(3, 2, 1)
+ );
+
+ StepFunction<TestImpurityMeasure> b = new StepFunction<>(
+ new double[]{0, 2, 4},
+ TestImpurityMeasure.asTestImpurityMeasures(3, 2, 1)
+ );
+
+ StepFunction<TestImpurityMeasure> c = a.add(b);
+
+ assertArrayEquals(new double[]{0, 1, 2, 3, 4, 5}, c.getX(), 1e-10);
+ assertArrayEquals(
+ TestImpurityMeasure.asTestImpurityMeasures(3, 6, 5, 4, 3, 2),
+ c.getY()
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/TestImpurityMeasure.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/TestImpurityMeasure.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/TestImpurityMeasure.java
new file mode 100644
index 0000000..c0d1911
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/TestImpurityMeasure.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import java.util.Objects;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Utils class used as impurity measure in tests.
+ */
+class TestImpurityMeasure implements ImpurityMeasure<TestImpurityMeasure> {
+ /** */
+ private static final long serialVersionUID = 2414020770162797847L;
+
+ /** Impurity. */
+ private final double impurity;
+
+ /**
+ * Constructs a new instance of test impurity measure.
+ *
+ * @param impurity Impurity.
+ */
+ private TestImpurityMeasure(double impurity) {
+ this.impurity = impurity;
+ }
+
+ /**
+ * Convert doubles to array of test impurity measures.
+ *
+ * @param impurity Impurity as array of doubles.
+ * @return Test impurity measure objects as array.
+ */
+ static TestImpurityMeasure[] asTestImpurityMeasures(double... impurity) {
+ TestImpurityMeasure[] res = new TestImpurityMeasure[impurity.length];
+
+ for (int i = 0; i < impurity.length; i++)
+ res[i] = new TestImpurityMeasure(impurity[i]);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double impurity() {
+ return impurity;
+ }
+
+ /** {@inheritDoc} */
+ @Override public TestImpurityMeasure add(TestImpurityMeasure measure) {
+ return new TestImpurityMeasure(impurity + measure.impurity);
+ }
+
+ /** {@inheritDoc} */
+ @Override public TestImpurityMeasure subtract(TestImpurityMeasure measure) {
+ return new TestImpurityMeasure(impurity - measure.impurity);
+ }
+
+ /** */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ TestImpurityMeasure measure = (TestImpurityMeasure)o;
+
+ return Double.compare(measure.impurity, impurity) == 0;
+ }
+
+ /** */
+ @Override public int hashCode() {
+
+ return Objects.hash(impurity);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
new file mode 100644
index 0000000..b259ec9
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.performance;
+
+import java.io.IOException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.nn.performance.MnistMLPTestUtil;
+import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
+import org.apache.ignite.ml.tree.DecisionTreeNode;
+import org.apache.ignite.ml.tree.impurity.util.SimpleStepFunctionCompressor;
+import org.apache.ignite.ml.util.MnistUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests {@link DecisionTreeClassificationTrainer} on the MNIST dataset that require to start the whole Ignite
+ * infrastructure. For manual run.
+ */
+public class DecisionTreeMNISTIntegrationTest extends GridCommonAbstractTest {
+ /** Number of nodes in grid */
+ private static final int NODE_COUNT = 3;
+
+ /** Ignite instance. */
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ for (int i = 1; i <= NODE_COUNT; i++)
+ startGrid(i);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() {
+ stopAllGrids();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void beforeTest() throws Exception {
+ /* Grid instance. */
+ ignite = grid(NODE_COUNT);
+ ignite.configuration().setPeerClassLoadingEnabled(true);
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+ }
+
+ /** Tests on the MNIST dataset. For manual run. */
+ public void testMNIST() throws IOException {
+ CacheConfiguration<Integer, MnistUtils.MnistLabeledImage> trainingSetCacheCfg = new CacheConfiguration<>();
+ trainingSetCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+ trainingSetCacheCfg.setName("MNIST_TRAINING_SET");
+
+ IgniteCache<Integer, MnistUtils.MnistLabeledImage> trainingSet = ignite.createCache(trainingSetCacheCfg);
+
+ int i = 0;
+ for (MnistUtils.MnistLabeledImage e : MnistMLPTestUtil.loadTrainingSet(60_000))
+ trainingSet.put(i++, e);
+
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(
+ 8,
+ 0,
+ new SimpleStepFunctionCompressor<>());
+
+ DecisionTreeNode mdl = trainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+ (k, v) -> v.getPixels(),
+ (k, v) -> (double) v.getLabel()
+ );
+
+ int correctAnswers = 0;
+ int incorrectAnswers = 0;
+
+ for (MnistUtils.MnistLabeledImage e : MnistMLPTestUtil.loadTestSet(10_000)) {
+ double res = mdl.apply(e.getPixels());
+
+ if (res == e.getLabel())
+ correctAnswers++;
+ else
+ incorrectAnswers++;
+ }
+
+ double accuracy = 1.0 * correctAnswers / (correctAnswers + incorrectAnswers);
+
+ assertTrue(accuracy > 0.8);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
new file mode 100644
index 0000000..6dbd44c
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.performance;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.nn.performance.MnistMLPTestUtil;
+import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
+import org.apache.ignite.ml.tree.DecisionTreeNode;
+import org.apache.ignite.ml.tree.impurity.util.SimpleStepFunctionCompressor;
+import org.apache.ignite.ml.util.MnistUtils;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * Tests {@link DecisionTreeClassificationTrainer} on the MNIST dataset using locally stored data. For manual run.
+ */
+public class DecisionTreeMNISTTest {
+ /** Tests on the MNIST dataset. For manual run. */
+ @Test
+ public void testMNIST() throws IOException {
+ Map<Integer, MnistUtils.MnistLabeledImage> trainingSet = new HashMap<>();
+
+ int i = 0;
+ for (MnistUtils.MnistLabeledImage e : MnistMLPTestUtil.loadTrainingSet(60_000))
+ trainingSet.put(i++, e);
+
+
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(
+ 8,
+ 0,
+ new SimpleStepFunctionCompressor<>());
+
+ DecisionTreeNode mdl = trainer.fit(
+ new LocalDatasetBuilder<>(trainingSet, 10),
+ (k, v) -> v.getPixels(),
+ (k, v) -> (double) v.getLabel()
+ );
+
+ int correctAnswers = 0;
+ int incorrectAnswers = 0;
+
+ for (MnistUtils.MnistLabeledImage e : MnistMLPTestUtil.loadTestSet(10_000)) {
+ double res = mdl.apply(e.getPixels());
+
+ if (res == e.getLabel())
+ correctAnswers++;
+ else
+ incorrectAnswers++;
+ }
+
+ double accuracy = 1.0 * correctAnswers / (correctAnswers + incorrectAnswers);
+
+ assertTrue(accuracy > 0.8);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/BaseDecisionTreeTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/BaseDecisionTreeTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/BaseDecisionTreeTest.java
deleted file mode 100644
index 65f0ae4..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/BaseDecisionTreeTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.Arrays;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Base class for decision trees test.
- */
-public class BaseDecisionTreeTest extends GridCommonAbstractTest {
- /** Count of nodes. */
- private static final int NODE_COUNT = 4;
-
- /** Grid instance. */
- protected Ignite ignite;
-
- /**
- * Default constructor.
- */
- public BaseDecisionTreeTest() {
- super(false);
- }
-
- /**
- * {@inheritDoc}
- */
- @Override protected void beforeTest() throws Exception {
- ignite = grid(NODE_COUNT);
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- for (int i = 1; i <= NODE_COUNT; i++)
- startGrid(i);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- stopAllGrids();
- }
-
- /**
- * Convert double array to {@link LabeledVectorDouble}
- *
- * @param arr Array for conversion.
- * @return LabeledVectorDouble.
- */
- protected static LabeledVectorDouble<DenseLocalOnHeapVector> asLabeledVector(double arr[]) {
- return new LabeledVectorDouble<>(new DenseLocalOnHeapVector(Arrays.copyOf(arr, arr.length - 1)), arr[arr.length - 1]);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/ColumnDecisionTreeTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/ColumnDecisionTreeTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/ColumnDecisionTreeTrainerTest.java
deleted file mode 100644
index b090f43..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/ColumnDecisionTreeTrainerTest.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Tracer;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.MatrixColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-
-/** Tests behaviour of ColumnDecisionTreeTrainer. */
-public class ColumnDecisionTreeTrainerTest extends BaseDecisionTreeTest {
- /**
- * Test {@link ColumnDecisionTreeTrainerTest} for mixed (continuous and categorical) data with Gini impurity.
- */
- public void testCacheMixedGini() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int totalPts = 1 << 10;
- int featCnt = 2;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(1, 3);
-
- Random rnd = new Random(12349L);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
- split(0, 1, new int[] {0, 2}).
- split(1, 0, -10.0);
-
- testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MEAN, rnd);
- }
-
- /**
- * Test {@link ColumnDecisionTreeTrainerTest} for mixed (continuous and categorical) data with Variance impurity.
- */
- public void testCacheMixed() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int totalPts = 1 << 10;
- int featCnt = 2;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(1, 3);
-
- Random rnd = new Random(12349L);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
- split(0, 1, new int[] {0, 2}).
- split(1, 0, -10.0);
-
- testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, rnd);
- }
-
- /**
- * Test {@link ColumnDecisionTreeTrainerTest} for continuous data with Variance impurity.
- */
- public void testCacheCont() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int totalPts = 1 << 10;
- int featCnt = 12;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
-
- Random rnd = new Random(12349L);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
- split(0, 0, -10.0).
- split(1, 0, 0.0).
- split(1, 1, 2.0).
- split(3, 7, 50.0);
-
- testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, rnd);
- }
-
- /**
- * Test {@link ColumnDecisionTreeTrainerTest} for continuous data with Gini impurity.
- */
- public void testCacheContGini() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int totalPts = 1 << 10;
- int featCnt = 12;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
-
- Random rnd = new Random(12349L);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
- split(0, 0, -10.0).
- split(1, 0, 0.0).
- split(1, 1, 2.0).
- split(3, 7, 50.0);
-
- testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MEAN, rnd);
- }
-
- /**
- * Test {@link ColumnDecisionTreeTrainerTest} for categorical data with Variance impurity.
- */
- public void testCacheCat() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int totalPts = 1 << 10;
- int featCnt = 12;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(5, 7);
-
- Random rnd = new Random(12349L);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
- split(0, 5, new int[] {0, 2, 5});
-
- testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, rnd);
- }
-
- /** */
- private <D extends ContinuousRegionInfo> void testByGen(int totalPts, HashMap<Integer, Integer> catsInfo,
- SplitDataGenerator<DenseLocalOnHeapVector> gen,
- IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> calc,
- IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> catImpCalc,
- IgniteFunction<DoubleStream, Double> regCalc, Random rnd) {
-
- List<IgniteBiTuple<Integer, DenseLocalOnHeapVector>> lst = gen.
- points(totalPts, (i, rn) -> i).
- collect(Collectors.toList());
-
- int featCnt = gen.featuresCnt();
-
- Collections.shuffle(lst, rnd);
-
- SparseDistributedMatrix m = new SparseDistributedMatrix(totalPts, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
-
- int i = 0;
- for (IgniteBiTuple<Integer, DenseLocalOnHeapVector> bt : lst) {
- byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
- byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
- m.setRow(i, bt.get2().getStorage().data());
- i++;
- }
-
- ColumnDecisionTreeTrainer<D> trainer =
- new ColumnDecisionTreeTrainer<>(3, calc, catImpCalc, regCalc, ignite);
-
- DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catsInfo));
-
- byRegion.keySet().forEach(k -> {
- LabeledVectorDouble sp = byRegion.get(k).get(0);
- Tracer.showAscii(sp.features());
- X.println("Actual and predicted vectors [act=" + sp.label() + " " + ", pred=" + mdl.apply(sp.features()) + "]");
- assert mdl.apply(sp.features()) == sp.doubleLabel();
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/DecisionTreesTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/DecisionTreesTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/DecisionTreesTestSuite.java
deleted file mode 100644
index 3343503..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/DecisionTreesTestSuite.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
-/**
- * Test suite for all tests located in org.apache.ignite.ml.trees package
- */
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- ColumnDecisionTreeTrainerTest.class,
- GiniSplitCalculatorTest.class,
- VarianceSplitCalculatorTest.class
-})
-public class DecisionTreesTestSuite {
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/GiniSplitCalculatorTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/GiniSplitCalculatorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/GiniSplitCalculatorTest.java
deleted file mode 100644
index c92b4f5..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/GiniSplitCalculatorTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.GiniSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-import org.junit.Test;
-
-/**
- * Test of {@link GiniSplitCalculator}.
- */
-public class GiniSplitCalculatorTest {
- /** Test calculation of region info consisting from one point. */
- @Test
- public void testCalculateRegionInfoSimple() {
- double labels[] = new double[] {0.0};
-
- assert new GiniSplitCalculator(labels).calculateRegionInfo(DoubleStream.of(labels), 0).impurity() == 0.0;
- }
-
- /** Test calculation of region info consisting from two distinct classes. */
- @Test
- public void testCalculateRegionInfoTwoClasses() {
- double labels[] = new double[] {0.0, 1.0};
-
- assert new GiniSplitCalculator(labels).calculateRegionInfo(DoubleStream.of(labels), 0).impurity() == 0.5;
- }
-
- /** Test calculation of region info consisting from three distinct classes. */
- @Test
- public void testCalculateRegionInfoThreeClasses() {
- double labels[] = new double[] {0.0, 1.0, 2.0};
-
- assert Math.abs(new GiniSplitCalculator(labels).calculateRegionInfo(DoubleStream.of(labels), 0).impurity() - 2.0 / 3) < 1E-5;
- }
-
- /** Test calculation of split of region consisting from one point. */
- @Test
- public void testSplitSimple() {
- double labels[] = new double[] {0.0};
- double values[] = new double[] {0.0};
- Integer[] samples = new Integer[] {0};
-
- int cnts[] = new int[] {1};
-
- GiniSplitCalculator.GiniData data = new GiniSplitCalculator.GiniData(0.0, 1, cnts, 1);
-
- assert new GiniSplitCalculator(labels).splitRegion(samples, values, labels, 0, data) == null;
- }
-
- /** Test calculation of split of region consisting from two points. */
- @Test
- public void testSplitTwoClassesTwoPoints() {
- double labels[] = new double[] {0.0, 1.0};
- double values[] = new double[] {0.0, 1.0};
- Integer[] samples = new Integer[] {0, 1};
-
- int cnts[] = new int[] {1, 1};
-
- GiniSplitCalculator.GiniData data = new GiniSplitCalculator.GiniData(0.5, 2, cnts, 1.0 * 1.0 + 1.0 * 1.0);
-
- SplitInfo<GiniSplitCalculator.GiniData> split = new GiniSplitCalculator(labels).splitRegion(samples, values, labels, 0, data);
-
- assert split.leftData().impurity() == 0;
- assert split.leftData().counts()[0] == 1;
- assert split.leftData().counts()[1] == 0;
- assert split.leftData().getSize() == 1;
-
- assert split.rightData().impurity() == 0;
- assert split.rightData().counts()[0] == 0;
- assert split.rightData().counts()[1] == 1;
- assert split.rightData().getSize() == 1;
- }
-
- /** Test calculation of split of region consisting from four distinct values. */
- @Test
- public void testSplitTwoClassesFourPoints() {
- double labels[] = new double[] {0.0, 0.0, 1.0, 1.0};
- double values[] = new double[] {0.0, 1.0, 2.0, 3.0};
-
- Integer[] samples = new Integer[] {0, 1, 2, 3};
-
- int[] cnts = new int[] {2, 2};
-
- GiniSplitCalculator.GiniData data = new GiniSplitCalculator.GiniData(0.5, 4, cnts, 2.0 * 2.0 + 2.0 * 2.0);
-
- SplitInfo<GiniSplitCalculator.GiniData> split = new GiniSplitCalculator(labels).splitRegion(samples, values, labels, 0, data);
-
- assert split.leftData().impurity() == 0;
- assert split.leftData().counts()[0] == 2;
- assert split.leftData().counts()[1] == 0;
- assert split.leftData().getSize() == 2;
-
- assert split.rightData().impurity() == 0;
- assert split.rightData().counts()[0] == 0;
- assert split.rightData().counts()[1] == 2;
- assert split.rightData().getSize() == 2;
- }
-
- /** Test calculation of split of region consisting from three distinct values. */
- @Test
- public void testSplitThreePoints() {
- double labels[] = new double[] {0.0, 1.0, 2.0};
- double values[] = new double[] {0.0, 1.0, 2.0};
- Integer[] samples = new Integer[] {0, 1, 2};
-
- int[] cnts = new int[] {1, 1, 1};
-
- GiniSplitCalculator.GiniData data = new GiniSplitCalculator.GiniData(2.0 / 3, 3, cnts, 1.0 * 1.0 + 1.0 * 1.0 + 1.0 * 1.0);
-
- SplitInfo<GiniSplitCalculator.GiniData> split = new GiniSplitCalculator(labels).splitRegion(samples, values, labels, 0, data);
-
- assert split.leftData().impurity() == 0.0;
- assert split.leftData().counts()[0] == 1;
- assert split.leftData().counts()[1] == 0;
- assert split.leftData().counts()[2] == 0;
- assert split.leftData().getSize() == 1;
-
- assert split.rightData().impurity() == 0.5;
- assert split.rightData().counts()[0] == 0;
- assert split.rightData().counts()[1] == 1;
- assert split.rightData().counts()[2] == 1;
- assert split.rightData().getSize() == 2;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/SplitDataGenerator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/SplitDataGenerator.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/SplitDataGenerator.java
deleted file mode 100644
index 279e685..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/SplitDataGenerator.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.ml.trees;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-import org.apache.ignite.ml.util.Utils;
-
-/**
- * Utility class for generating data which has binary tree split structure.
- *
- * @param <V>
- */
-public class SplitDataGenerator<V extends Vector> {
- /** */
- private static final double DELTA = 100.0;
-
- /** Map of the form of (is categorical -> list of region indexes). */
- private final Map<Boolean, List<Integer>> di;
-
- /** List of regions. */
- private final List<Region> regs;
-
- /** Data of bounds of regions. */
- private final Map<Integer, IgniteBiTuple<Double, Double>> boundsData;
-
- /** Random numbers generator. */
- private final Random rnd;
-
- /** Supplier of vectors. */
- private final Supplier<V> supplier;
-
- /** Features count. */
- private final int featCnt;
-
- /**
- * Create SplitDataGenerator.
- *
- * @param featCnt Features count.
- * @param catFeaturesInfo Information about categorical features in form of map (feature index -> categories
- * count).
- * @param supplier Supplier of vectors.
- * @param rnd Random numbers generator.
- */
- public SplitDataGenerator(int featCnt, Map<Integer, Integer> catFeaturesInfo, Supplier<V> supplier, Random rnd) {
- regs = new LinkedList<>();
- boundsData = new HashMap<>();
- this.rnd = rnd;
- this.supplier = supplier;
- this.featCnt = featCnt;
-
- // Divide indexes into indexes of categorical coordinates and indexes of continuous coordinates.
- di = IntStream.range(0, featCnt).
- boxed().
- collect(Collectors.partitioningBy(catFeaturesInfo::containsKey));
-
- // Categorical coordinates info.
- Map<Integer, CatCoordInfo> catCoords = new HashMap<>();
- di.get(true).forEach(i -> {
- BitSet bs = new BitSet();
- bs.set(0, catFeaturesInfo.get(i));
- catCoords.put(i, new CatCoordInfo(bs));
- });
-
- // Continuous coordinates info.
- Map<Integer, ContCoordInfo> contCoords = new HashMap<>();
- di.get(false).forEach(i -> {
- contCoords.put(i, new ContCoordInfo());
- boundsData.put(i, new IgniteBiTuple<>(-1.0, 1.0));
- });
-
- Region firstReg = new Region(catCoords, contCoords, 0);
- regs.add(firstReg);
- }
-
- /**
- * Categorical coordinate info.
- */
- private static class CatCoordInfo implements Serializable {
- /**
- * Defines categories which are included in this region
- */
- private final BitSet bs;
-
- /**
- * Construct CatCoordInfo.
- *
- * @param bs Bitset.
- */
- CatCoordInfo(BitSet bs) {
- this.bs = bs;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "CatCoordInfo [" +
- "bs=" + bs +
- ']';
- }
- }
-
- /**
- * Continuous coordinate info.
- */
- private static class ContCoordInfo implements Serializable {
- /**
- * Left (min) bound of region.
- */
- private double left;
-
- /**
- * Right (max) bound of region.
- */
- private double right;
-
- /**
- * Construct ContCoordInfo.
- */
- ContCoordInfo() {
- left = Double.NEGATIVE_INFINITY;
- right = Double.POSITIVE_INFINITY;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "ContCoordInfo [" +
- "left=" + left +
- ", right=" + right +
- ']';
- }
- }
-
- /**
- * Class representing information about region.
- */
- private static class Region implements Serializable {
- /**
- * Information about categorical coordinates restrictions of this region in form of
- * (coordinate index -> restriction)
- */
- private final Map<Integer, CatCoordInfo> catCoords;
-
- /**
- * Information about continuous coordinates restrictions of this region in form of
- * (coordinate index -> restriction)
- */
- private final Map<Integer, ContCoordInfo> contCoords;
-
- /**
- * Region should contain {@code 1/2^twoPow * totalPoints} points.
- */
- private int twoPow;
-
- /**
- * Construct region by information about restrictions on coordinates (features) values.
- *
- * @param catCoords Restrictions on categorical coordinates.
- * @param contCoords Restrictions on continuous coordinates
- * @param twoPow Region should contain {@code 1/2^twoPow * totalPoints} points.
- */
- Region(Map<Integer, CatCoordInfo> catCoords, Map<Integer, ContCoordInfo> contCoords, int twoPow) {
- this.catCoords = catCoords;
- this.contCoords = contCoords;
- this.twoPow = twoPow;
- }
-
- /** */
- int divideBy() {
- return 1 << twoPow;
- }
-
- /** */
- void incTwoPow() {
- twoPow++;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "Region [" +
- "catCoords=" + catCoords +
- ", contCoords=" + contCoords +
- ", twoPow=" + twoPow +
- ']';
- }
-
- /**
- * Generate continuous coordinate for this region.
- *
- * @param coordIdx Coordinate index.
- * @param boundsData Data with bounds
- * @param rnd Random numbers generator.
- * @return Categorical coordinate value.
- */
- double generateContCoord(int coordIdx, Map<Integer, IgniteBiTuple<Double, Double>> boundsData,
- Random rnd) {
- ContCoordInfo cci = contCoords.get(coordIdx);
- double left = cci.left;
- double right = cci.right;
-
- if (left == Double.NEGATIVE_INFINITY)
- left = boundsData.get(coordIdx).get1() - DELTA;
-
- if (right == Double.POSITIVE_INFINITY)
- right = boundsData.get(coordIdx).get2() + DELTA;
-
- double size = right - left;
-
- return left + rnd.nextDouble() * size;
- }
-
- /**
- * Generate categorical coordinate value for this region.
- *
- * @param coordIdx Coordinate index.
- * @param rnd Random numbers generator.
- * @return Categorical coordinate value.
- */
- double generateCatCoord(int coordIdx, Random rnd) {
- // Pick random bit.
- BitSet bs = catCoords.get(coordIdx).bs;
- int j = rnd.nextInt(bs.length());
-
- int i = 0;
- int bn = 0;
- int bnp = 0;
-
- while ((bn = bs.nextSetBit(bn)) != -1 && i <= j) {
- i++;
- bnp = bn;
- bn++;
- }
-
- return bnp;
- }
-
- /**
- * Generate points for this region.
- *
- * @param ptsCnt Count of points to generate.
- * @param val Label for all points in this region.
- * @param boundsData Data about bounds of continuous coordinates.
- * @param catCont Data about which categories can be in this region in the form (coordinate index -> list of
- * categories indexes).
- * @param s Vectors supplier.
- * @param rnd Random numbers generator.
- * @param <V> Type of vectors.
- * @return Stream of generated points for this region.
- */
- <V extends Vector> Stream<V> generatePoints(int ptsCnt, double val,
- Map<Integer, IgniteBiTuple<Double, Double>> boundsData, Map<Boolean, List<Integer>> catCont,
- Supplier<V> s,
- Random rnd) {
- return IntStream.range(0, ptsCnt / divideBy()).mapToObj(i -> {
- V v = s.get();
- int coordsCnt = v.size();
- catCont.get(false).forEach(ci -> v.setX(ci, generateContCoord(ci, boundsData, rnd)));
- catCont.get(true).forEach(ci -> v.setX(ci, generateCatCoord(ci, rnd)));
-
- v.setX(coordsCnt - 1, val);
- return v;
- });
- }
- }
-
- /**
- * Split region by continuous coordinate.using given threshold.
- *
- * @param regIdx Region index.
- * @param coordIdx Coordinate index.
- * @param threshold Threshold.
- * @return {@code this}.
- */
- public SplitDataGenerator<V> split(int regIdx, int coordIdx, double threshold) {
- Region regToSplit = regs.get(regIdx);
- ContCoordInfo cci = regToSplit.contCoords.get(coordIdx);
-
- double left = cci.left;
- double right = cci.right;
-
- if (threshold < left || threshold > right)
- throw new MathIllegalArgumentException("Threshold is out of region bounds.");
-
- regToSplit.incTwoPow();
-
- Region newReg = Utils.copy(regToSplit);
- newReg.contCoords.get(coordIdx).left = threshold;
-
- regs.add(regIdx + 1, newReg);
- cci.right = threshold;
-
- IgniteBiTuple<Double, Double> bounds = boundsData.get(coordIdx);
- double min = bounds.get1();
- double max = bounds.get2();
- boundsData.put(coordIdx, new IgniteBiTuple<>(Math.min(threshold, min), Math.max(max, threshold)));
-
- return this;
- }
-
- /**
- * Split region by categorical coordinate.
- *
- * @param regIdx Region index.
- * @param coordIdx Coordinate index.
- * @param cats Categories allowed for the left sub region.
- * @return {@code this}.
- */
- public SplitDataGenerator<V> split(int regIdx, int coordIdx, int[] cats) {
- BitSet subset = new BitSet();
- Arrays.stream(cats).forEach(subset::set);
- Region regToSplit = regs.get(regIdx);
- CatCoordInfo cci = regToSplit.catCoords.get(coordIdx);
-
- BitSet ssc = (BitSet)subset.clone();
- BitSet set = cci.bs;
- ssc.and(set);
- if (ssc.length() != subset.length())
- throw new MathIllegalArgumentException("Splitter set is not a subset of a parent subset.");
-
- ssc.xor(set);
- set.and(subset);
-
- regToSplit.incTwoPow();
- Region newReg = Utils.copy(regToSplit);
- newReg.catCoords.put(coordIdx, new CatCoordInfo(ssc));
-
- regs.add(regIdx + 1, newReg);
-
- return this;
- }
-
- /**
- * Get stream of points generated by this generator.
- *
- * @param ptsCnt Points count.
- */
- public Stream<IgniteBiTuple<Integer, V>> points(int ptsCnt, BiFunction<Double, Random, Double> f) {
- regs.forEach(System.out::println);
-
- return IntStream.range(0, regs.size()).
- boxed().
- map(i -> regs.get(i).generatePoints(ptsCnt, f.apply((double)i, rnd), boundsData, di, supplier, rnd).map(v -> new IgniteBiTuple<>(i, v))).flatMap(Function.identity());
- }
-
- /**
- * Count of regions.
- *
- * @return Count of regions.
- */
- public int regsCount() {
- return regs.size();
- }
-
- /**
- * Get features count.
- *
- * @return Features count.
- */
- public int featuresCnt() {
- return featCnt;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/VarianceSplitCalculatorTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/VarianceSplitCalculatorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/VarianceSplitCalculatorTest.java
deleted file mode 100644
index d67cbc6..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/VarianceSplitCalculatorTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.VarianceSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-import org.junit.Test;
-
-/**
- * Test for {@link VarianceSplitCalculator}.
- */
-public class VarianceSplitCalculatorTest {
- /** Test calculation of region info consisting from one point. */
- @Test
- public void testCalculateRegionInfoSimple() {
- double labels[] = new double[] {0.0};
-
- assert new VarianceSplitCalculator().calculateRegionInfo(DoubleStream.of(labels), 1).impurity() == 0.0;
- }
-
- /** Test calculation of region info consisting from two classes. */
- @Test
- public void testCalculateRegionInfoTwoClasses() {
- double labels[] = new double[] {0.0, 1.0};
-
- assert new VarianceSplitCalculator().calculateRegionInfo(DoubleStream.of(labels), 2).impurity() == 0.25;
- }
-
- /** Test calculation of region info consisting from three classes. */
- @Test
- public void testCalculateRegionInfoThreeClasses() {
- double labels[] = new double[] {1.0, 2.0, 3.0};
-
- assert Math.abs(new VarianceSplitCalculator().calculateRegionInfo(DoubleStream.of(labels), 3).impurity() - 2.0 / 3) < 1E-10;
- }
-
- /** Test calculation of split of region consisting from one point. */
- @Test
- public void testSplitSimple() {
- double labels[] = new double[] {0.0};
- double values[] = new double[] {0.0};
- Integer[] samples = new Integer[] {0};
-
- VarianceSplitCalculator.VarianceData data = new VarianceSplitCalculator.VarianceData(0.0, 1, 0.0);
-
- assert new VarianceSplitCalculator().splitRegion(samples, values, labels, 0, data) == null;
- }
-
- /** Test calculation of split of region consisting from two classes. */
- @Test
- public void testSplitTwoClassesTwoPoints() {
- double labels[] = new double[] {0.0, 1.0};
- double values[] = new double[] {0.0, 1.0};
- Integer[] samples = new Integer[] {0, 1};
-
- VarianceSplitCalculator.VarianceData data = new VarianceSplitCalculator.VarianceData(0.25, 2, 0.5);
-
- SplitInfo<VarianceSplitCalculator.VarianceData> split = new VarianceSplitCalculator().splitRegion(samples, values, labels, 0, data);
-
- assert split.leftData().impurity() == 0;
- assert split.leftData().mean() == 0;
- assert split.leftData().getSize() == 1;
-
- assert split.rightData().impurity() == 0;
- assert split.rightData().mean() == 1;
- assert split.rightData().getSize() == 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/test/java/org/apache/ignite/ml/trees/performance/ColumnDecisionTreeTrainerBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/performance/ColumnDecisionTreeTrainerBenchmark.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/performance/ColumnDecisionTreeTrainerBenchmark.java
deleted file mode 100644
index 21fd692..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/performance/ColumnDecisionTreeTrainerBenchmark.java
+++ /dev/null
@@ -1,456 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.performance;
-
-import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Random;
-import java.util.UUID;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.estimators.Estimators;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Tracer;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distributed.keys.impl.SparseMatrixKey;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteTriFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.ml.trees.BaseDecisionTreeTest;
-import org.apache.ignite.ml.trees.SplitDataGenerator;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.BiIndex;
-import org.apache.ignite.ml.trees.trainers.columnbased.BiIndexedCacheColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.MatrixColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ContextCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ProjectionsCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.SplitCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.GiniSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.VarianceSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.ml.util.MnistUtils;
-import org.apache.ignite.stream.StreamTransformer;
-import org.apache.ignite.testframework.junits.IgniteTestResources;
-import org.apache.log4j.Level;
-import org.junit.Assert;
-
-/**
- * Various benchmarks for hand runs.
- */
-public class ColumnDecisionTreeTrainerBenchmark extends BaseDecisionTreeTest {
- /** Name of the property specifying path to training set images. */
- private static final String PROP_TRAINING_IMAGES = "mnist.training.images";
-
- /** Name of property specifying path to training set labels. */
- private static final String PROP_TRAINING_LABELS = "mnist.training.labels";
-
- /** Name of property specifying path to test set images. */
- private static final String PROP_TEST_IMAGES = "mnist.test.images";
-
- /** Name of property specifying path to test set labels. */
- private static final String PROP_TEST_LABELS = "mnist.test.labels";
-
- /** Function to approximate. */
- private static final Function<Vector, Double> f1 = v -> v.get(0) * v.get(0) + 2 * Math.sin(v.get(1)) + v.get(2);
-
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return 6000000;
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName,
- IgniteTestResources rsrcs) throws Exception {
- IgniteConfiguration configuration = super.getConfiguration(igniteInstanceName, rsrcs);
- // We do not need any extra event types.
- configuration.setIncludeEventTypes();
- configuration.setPeerClassLoadingEnabled(false);
-
- resetLog4j(Level.INFO, false, GridCacheProcessor.class.getPackage().getName());
-
- return configuration;
- }
-
- /**
- * This test is for manual run only.
- * To run this test rename this method so it starts from 'test'.
- */
- public void tstCacheMixed() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int ptsPerReg = 150;
- int featCnt = 10;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(1, 3);
-
- Random rnd = new Random(12349L);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
- split(0, 1, new int[] {0, 2}).
- split(1, 0, -10.0).
- split(0, 0, 0.0);
-
- testByGenStreamerLoad(ptsPerReg, catsInfo, gen, rnd);
- }
-
- /**
- * Run decision tree classifier on MNIST using bi-indexed cache as a storage for dataset.
- * To run this test rename this method so it starts from 'test'.
- *
- * @throws IOException In case of loading MNIST dataset errors.
- */
- public void tstMNISTBiIndexedCache() throws IOException {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
- int ptsCnt = 40_000;
- int featCnt = 28 * 28;
-
- Properties props = loadMNISTProperties();
-
- Stream<DenseLocalOnHeapVector> trainingMnistStream = MnistUtils.mnistAsStream(props.getProperty(PROP_TRAINING_IMAGES), props.getProperty(PROP_TRAINING_LABELS), new Random(123L), ptsCnt);
- Stream<DenseLocalOnHeapVector> testMnistStream = MnistUtils.mnistAsStream(props.getProperty(PROP_TEST_IMAGES), props.getProperty(PROP_TEST_LABELS), new Random(123L), 10_000);
-
- IgniteCache<BiIndex, Double> cache = createBiIndexedCache();
-
- loadVectorsIntoBiIndexedCache(cache.getName(), trainingMnistStream.iterator(), featCnt + 1);
-
- ColumnDecisionTreeTrainer<GiniSplitCalculator.GiniData> trainer =
- new ColumnDecisionTreeTrainer<>(10, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MOST_COMMON, ignite);
-
- X.println("Training started.");
- long before = System.currentTimeMillis();
- DecisionTreeModel mdl = trainer.train(new BiIndexedCacheColumnDecisionTreeTrainerInput(cache, new HashMap<>(), ptsCnt, featCnt));
- X.println("Training finished in " + (System.currentTimeMillis() - before));
-
- IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse = Estimators.errorsPercentage();
- Double accuracy = mse.apply(mdl, testMnistStream.map(v -> new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
- X.println("Errors percentage: " + accuracy);
-
- Assert.assertEquals(0, SplitCache.getOrCreate(ignite).size());
- Assert.assertEquals(0, FeaturesCache.getOrCreate(ignite).size());
- Assert.assertEquals(0, ContextCache.getOrCreate(ignite).size());
- Assert.assertEquals(0, ProjectionsCache.getOrCreate(ignite).size());
- }
-
- /**
- * Run decision tree classifier on MNIST using sparse distributed matrix as a storage for dataset.
- * To run this test rename this method so it starts from 'test'.
- *
- * @throws IOException In case of loading MNIST dataset errors.
- */
- public void tstMNISTSparseDistributedMatrix() throws IOException {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
- int ptsCnt = 30_000;
- int featCnt = 28 * 28;
-
- Properties props = loadMNISTProperties();
-
- Stream<DenseLocalOnHeapVector> trainingMnistStream = MnistUtils.mnistAsStream(props.getProperty(PROP_TRAINING_IMAGES), props.getProperty(PROP_TRAINING_LABELS), new Random(123L), ptsCnt);
- Stream<DenseLocalOnHeapVector> testMnistStream = MnistUtils.mnistAsStream(props.getProperty(PROP_TEST_IMAGES), props.getProperty(PROP_TEST_LABELS), new Random(123L), 10_000);
-
- SparseDistributedMatrix m = new SparseDistributedMatrix(ptsCnt, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- SparseDistributedMatrixStorage sto = (SparseDistributedMatrixStorage)m.getStorage();
-
- loadVectorsIntoSparseDistributedMatrixCache(sto.cache().getName(), sto.getUUID(), trainingMnistStream.iterator(), featCnt + 1);
-
- ColumnDecisionTreeTrainer<GiniSplitCalculator.GiniData> trainer =
- new ColumnDecisionTreeTrainer<>(10, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MOST_COMMON, ignite);
-
- X.println("Training started");
- long before = System.currentTimeMillis();
- DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, new HashMap<>()));
- X.println("Training finished in " + (System.currentTimeMillis() - before));
-
- IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse = Estimators.errorsPercentage();
- Double accuracy = mse.apply(mdl, testMnistStream.map(v -> new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
- X.println("Errors percentage: " + accuracy);
-
- Assert.assertEquals(0, SplitCache.getOrCreate(ignite).size());
- Assert.assertEquals(0, FeaturesCache.getOrCreate(ignite).size());
- Assert.assertEquals(0, ContextCache.getOrCreate(ignite).size());
- Assert.assertEquals(0, ProjectionsCache.getOrCreate(ignite).size());
- }
-
- /** Load properties for MNIST tests. */
- private static Properties loadMNISTProperties() throws IOException {
- Properties res = new Properties();
-
- InputStream is = ColumnDecisionTreeTrainerBenchmark.class.getClassLoader().getResourceAsStream("manualrun/trees/columntrees.manualrun.properties");
-
- res.load(is);
-
- return res;
- }
-
- /** */
- private void testByGenStreamerLoad(int ptsPerReg, HashMap<Integer, Integer> catsInfo,
- SplitDataGenerator<DenseLocalOnHeapVector> gen, Random rnd) {
-
- List<IgniteBiTuple<Integer, DenseLocalOnHeapVector>> lst = gen.
- points(ptsPerReg, (i, rn) -> i).
- collect(Collectors.toList());
-
- int featCnt = gen.featuresCnt();
-
- Collections.shuffle(lst, rnd);
-
- int numRegs = gen.regsCount();
-
- SparseDistributedMatrix m = new SparseDistributedMatrix(numRegs * ptsPerReg, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- IgniteFunction<DoubleStream, Double> regCalc = s -> s.average().orElse(0.0);
-
- Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
-
- SparseDistributedMatrixStorage sto = (SparseDistributedMatrixStorage)m.getStorage();
- long before = System.currentTimeMillis();
- X.println("Batch loading started...");
- loadVectorsIntoSparseDistributedMatrixCache(sto.cache().getName(), sto.getUUID(), gen.
- points(ptsPerReg, (i, rn) -> i).map(IgniteBiTuple::get2).iterator(), featCnt + 1);
- X.println("Batch loading took " + (System.currentTimeMillis() - before) + " ms.");
-
- for (IgniteBiTuple<Integer, DenseLocalOnHeapVector> bt : lst) {
- byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
- byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
- }
-
- ColumnDecisionTreeTrainer<VarianceSplitCalculator.VarianceData> trainer =
- new ColumnDecisionTreeTrainer<>(2, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, regCalc, ignite);
-
- before = System.currentTimeMillis();
- DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catsInfo));
-
- X.println("Training took: " + (System.currentTimeMillis() - before) + " ms.");
-
- byRegion.keySet().forEach(k -> {
- LabeledVectorDouble sp = byRegion.get(k).get(0);
- Tracer.showAscii(sp.features());
- X.println("Predicted value and label [pred=" + mdl.apply(sp.features()) + ", label=" + sp.doubleLabel() + "]");
- assert mdl.apply(sp.features()) == sp.doubleLabel();
- });
- }
-
- /**
- * Test decision tree regression.
- * To run this test rename this method so it starts from 'test'.
- */
- public void tstF1() {
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- int ptsCnt = 10000;
- Map<Integer, double[]> ranges = new HashMap<>();
-
- ranges.put(0, new double[] {-100.0, 100.0});
- ranges.put(1, new double[] {-100.0, 100.0});
- ranges.put(2, new double[] {-100.0, 100.0});
-
- int featCnt = 100;
- double[] defRng = {-1.0, 1.0};
-
- Vector[] trainVectors = vecsFromRanges(ranges, featCnt, defRng, new Random(123L), ptsCnt, f1);
-
- SparseDistributedMatrix m = new SparseDistributedMatrix(ptsCnt, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- SparseDistributedMatrixStorage sto = (SparseDistributedMatrixStorage)m.getStorage();
-
- loadVectorsIntoSparseDistributedMatrixCache(sto.cache().getName(), sto.getUUID(), Arrays.stream(trainVectors).iterator(), featCnt + 1);
-
- IgniteFunction<DoubleStream, Double> regCalc = s -> s.average().orElse(0.0);
-
- ColumnDecisionTreeTrainer<VarianceSplitCalculator.VarianceData> trainer =
- new ColumnDecisionTreeTrainer<>(10, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, regCalc, ignite);
-
- X.println("Training started.");
- long before = System.currentTimeMillis();
- DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, new HashMap<>()));
- X.println("Training finished in: " + (System.currentTimeMillis() - before) + " ms.");
-
- Vector[] testVectors = vecsFromRanges(ranges, featCnt, defRng, new Random(123L), 20, f1);
-
- IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse = Estimators.MSE();
- Double accuracy = mse.apply(mdl, Arrays.stream(testVectors).map(v -> new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
- X.println("MSE: " + accuracy);
- }
-
- /**
- * Load vectors into sparse distributed matrix.
- *
- * @param cacheName Name of cache where matrix is stored.
- * @param uuid UUID of matrix.
- * @param iter Iterator over vectors.
- * @param vectorSize size of vectors.
- */
- private void loadVectorsIntoSparseDistributedMatrixCache(String cacheName, UUID uuid,
- Iterator<? extends org.apache.ignite.ml.math.Vector> iter, int vectorSize) {
- try (IgniteDataStreamer<SparseMatrixKey, Map<Integer, Double>> streamer =
- Ignition.localIgnite().dataStreamer(cacheName)) {
- int sampleIdx = 0;
- streamer.allowOverwrite(true);
-
- streamer.receiver(StreamTransformer.from((e, arg) -> {
- Map<Integer, Double> val = e.getValue();
-
- if (val == null)
- val = new Int2DoubleOpenHashMap();
-
- val.putAll((Map<Integer, Double>)arg[0]);
-
- e.setValue(val);
-
- return null;
- }));
-
- // Feature index -> (sample index -> value)
- Map<Integer, Map<Integer, Double>> batch = new HashMap<>();
- IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
- int batchSize = 1000;
-
- while (iter.hasNext()) {
- org.apache.ignite.ml.math.Vector next = iter.next();
-
- for (int i = 0; i < vectorSize; i++)
- batch.get(i).put(sampleIdx, next.getX(i));
-
- X.println("Sample index: " + sampleIdx);
- if (sampleIdx % batchSize == 0) {
- batch.keySet().forEach(fi -> streamer.addData(new SparseMatrixKey(fi, uuid, fi), batch.get(fi)));
- IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
- }
- sampleIdx++;
- }
- if (sampleIdx % batchSize != 0) {
- batch.keySet().forEach(fi -> streamer.addData(new SparseMatrixKey(fi, uuid, fi), batch.get(fi)));
- IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
- }
- }
- }
-
- /**
- * Load vectors into bi-indexed cache.
- *
- * @param cacheName Name of cache.
- * @param iter Iterator over vectors.
- * @param vectorSize size of vectors.
- */
- private void loadVectorsIntoBiIndexedCache(String cacheName,
- Iterator<? extends org.apache.ignite.ml.math.Vector> iter, int vectorSize) {
- try (IgniteDataStreamer<BiIndex, Double> streamer =
- Ignition.localIgnite().dataStreamer(cacheName)) {
- int sampleIdx = 0;
-
- streamer.perNodeBufferSize(10000);
-
- while (iter.hasNext()) {
- org.apache.ignite.ml.math.Vector next = iter.next();
-
- for (int i = 0; i < vectorSize; i++)
- streamer.addData(new BiIndex(sampleIdx, i), next.getX(i));
-
- sampleIdx++;
-
- if (sampleIdx % 1000 == 0)
- System.out.println("Loaded: " + sampleIdx + " vectors.");
- }
- }
- }
-
- /**
- * Create bi-indexed cache for tests.
- *
- * @return Bi-indexed cache.
- */
- private IgniteCache<BiIndex, Double> createBiIndexedCache() {
- CacheConfiguration<BiIndex, Double> cfg = new CacheConfiguration<>();
-
- // Write to primary.
- cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
- // Atomic transactions only.
- cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
- // No eviction.
- cfg.setEvictionPolicy(null);
-
- // No copying of values.
- cfg.setCopyOnRead(false);
-
- // Cache is partitioned.
- cfg.setCacheMode(CacheMode.PARTITIONED);
-
- cfg.setBackups(0);
-
- cfg.setName("TMP_BI_INDEXED_CACHE");
-
- return Ignition.localIgnite().getOrCreateCache(cfg);
- }
-
- /** */
- private Vector[] vecsFromRanges(Map<Integer, double[]> ranges, int featCnt, double[] defRng, Random rnd, int ptsCnt,
- Function<Vector, Double> f) {
- int vs = featCnt + 1;
- DenseLocalOnHeapVector[] res = new DenseLocalOnHeapVector[ptsCnt];
- for (int pt = 0; pt < ptsCnt; pt++) {
- DenseLocalOnHeapVector v = new DenseLocalOnHeapVector(vs);
- for (int i = 0; i < featCnt; i++) {
- double[] range = ranges.getOrDefault(i, defRng);
- double from = range[0];
- double to = range[1];
- double rng = to - from;
-
- v.setX(i, rnd.nextDouble() * rng);
- }
- v.setX(featCnt, f.apply(v));
- res[pt] = v;
- }
-
- return res;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
deleted file mode 100644
index f8a7c08..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.trees;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteColumnDecisionTreeGiniBenchmark extends IgniteAbstractBenchmark {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public boolean test(Map<Object, Object> ctx) throws Exception {
- // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
- // because we create ignite cache internally.
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- this.getClass().getSimpleName(), new Runnable() {
- /** {@inheritDoc} */
- @Override public void run() {
- // IMPL NOTE originally taken from ColumnDecisionTreeTrainerTest#testCacheMixedGini
- int totalPts = 1 << 10;
- int featCnt = 2;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(1, 3);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1)).
- split(0, 1, new int[] {0, 2}).
- split(1, 0, -10.0);
-
- gen.testByGen(totalPts, ContinuousSplitCalculators.GINI.apply(ignite),
- RegionCalculators.GINI, RegionCalculators.MEAN, ignite);
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
-
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
deleted file mode 100644
index f9d417f..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.trees;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteColumnDecisionTreeVarianceBenchmark extends IgniteAbstractBenchmark {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public boolean test(Map<Object, Object> ctx) throws Exception {
- // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
- // because we create ignite cache internally.
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- this.getClass().getSimpleName(), new Runnable() {
- /** {@inheritDoc} */
- @Override public void run() {
- // IMPL NOTE originally taken from ColumnDecisionTreeTrainerTest#testCacheMixed
- int totalPts = 1 << 10;
- int featCnt = 2;
-
- HashMap<Integer, Integer> catsInfo = new HashMap<>();
- catsInfo.put(1, 3);
-
- SplitDataGenerator<DenseLocalOnHeapVector> gen
- = new SplitDataGenerator<>(
- featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1)).
- split(0, 1, new int[] {0, 2}).
- split(1, 0, -10.0);
-
- gen.testByGen(totalPts,
- ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, ignite);
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
-
- return true;
- }
-}
[43/54] [abbrv] ignite git commit: IGNITE-8176: Integrate gradient
descent linear regression with partition based dataset
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
new file mode 100644
index 0000000..fa8fac4
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.regressions.linear;
+
+import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
+import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link LinearRegressionSGDTrainer}.
+ */
+@RunWith(Parameterized.class)
+public class LinearRegressionSGDTrainerTest {
+ /** Parameters. */
+ @Parameterized.Parameters(name = "Data divided on {0} partitions")
+ public static Iterable<Integer[]> data() {
+ return Arrays.asList(
+ new Integer[] {1},
+ new Integer[] {2},
+ new Integer[] {3},
+ new Integer[] {5},
+ new Integer[] {7},
+ new Integer[] {100}
+ );
+ }
+
+ /** Number of partitions. */
+ @Parameterized.Parameter
+ public int parts;
+
+ /**
+ * Tests {@code fit()} method on a simple small dataset.
+ */
+ @Test
+ public void testSmallDataFit() {
+ Map<Integer, double[]> data = new HashMap<>();
+ data.put(0, new double[] {-1.0915526, 1.81983527, -0.91409478, 0.70890712, -24.55724107});
+ data.put(1, new double[] {-0.61072904, 0.37545517, 0.21705352, 0.09516495, -26.57226867});
+ data.put(2, new double[] {0.05485406, 0.88219898, -0.80584547, 0.94668307, 61.80919728});
+ data.put(3, new double[] {-0.24835094, -0.34000053, -1.69984651, -1.45902635, -161.65525991});
+ data.put(4, new double[] {0.63675392, 0.31675535, 0.38837437, -1.1221971, -14.46432611});
+ data.put(5, new double[] {0.14194017, 2.18158997, -0.28397346, -0.62090588, -3.2122197});
+ data.put(6, new double[] {-0.53487507, 1.4454797, 0.21570443, -0.54161422, -46.5469012});
+ data.put(7, new double[] {-1.58812173, -0.73216803, -2.15670676, -1.03195988, -247.23559889});
+ data.put(8, new double[] {0.20702671, 0.92864654, 0.32721202, -0.09047503, 31.61484949});
+ data.put(9, new double[] {-0.37890345, -0.04846179, -0.84122753, -1.14667474, -124.92598583});
+
+ LinearRegressionSGDTrainer<?> trainer = new LinearRegressionSGDTrainer<>(new UpdatesStrategy<>(
+ new RPropUpdateCalculator(),
+ RPropParameterUpdate::sumLocal,
+ RPropParameterUpdate::avg
+ ), 100000, 10, 100, 123L);
+
+ LinearRegressionModel mdl = trainer.fit(
+ data,
+ parts,
+ (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
+ (k, v) -> v[4]
+ );
+
+ assertArrayEquals(
+ new double[] {72.26948107, 15.95144674, 24.07403921, 66.73038781},
+ mdl.getWeights().getStorage().data(),
+ 1e-1
+ );
+
+ assertEquals(2.8421709430404007e-14, mdl.getIntercept(), 1e-1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LocalLinearRegressionSGDTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LocalLinearRegressionSGDTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LocalLinearRegressionSGDTrainerTest.java
deleted file mode 100644
index bea164d..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LocalLinearRegressionSGDTrainerTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link DenseLocalOnHeapMatrix}.
- */
-public class LocalLinearRegressionSGDTrainerTest extends GenericLinearRegressionTrainerTest {
- /** */
- public LocalLinearRegressionSGDTrainerTest() {
- super(
- new LinearRegressionSGDTrainer(100_000, 1e-12),
- DenseLocalOnHeapMatrix::new,
- DenseLocalOnHeapVector::new,
- 1e-2);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
index 26ba2fb..0befd9b 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
@@ -17,14 +17,14 @@
package org.apache.ignite.ml.svm;
+import org.apache.ignite.ml.TestUtils;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.junit.Test;
+
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.junit.Test;
/**
* Tests for {@link SVMLinearBinaryClassificationTrainer}.
@@ -62,7 +62,8 @@ public class SVMBinaryTrainerTest {
SVMLinearBinaryClassificationTrainer trainer = new SVMLinearBinaryClassificationTrainer();
SVMLinearBinaryClassificationModel mdl = trainer.fit(
- new LocalDatasetBuilder<>(data, 10),
+ data,
+ 10,
(k, v) -> Arrays.copyOfRange(v, 1, v.length),
(k, v) -> v[0]
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java
index ad95eb4..31ab4d7 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java
@@ -17,14 +17,14 @@
package org.apache.ignite.ml.svm;
+import org.apache.ignite.ml.TestUtils;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.junit.Test;
+
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.junit.Test;
/**
* Tests for {@link SVMLinearBinaryClassificationTrainer}.
@@ -65,7 +65,8 @@ public class SVMMultiClassTrainerTest {
.withAmountOfIterations(20);
SVMLinearMultiClassClassificationModel mdl = trainer.fit(
- new LocalDatasetBuilder<>(data, 10),
+ data,
+ 10,
(k, v) -> Arrays.copyOfRange(v, 1, v.length),
(k, v) -> v[0]
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
index 94bca3f..d5b0b86 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
@@ -17,16 +17,16 @@
package org.apache.ignite.ml.tree;
-import java.util.Arrays;
-import java.util.Random;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import java.util.Arrays;
+import java.util.Random;
+
/**
* Tests for {@link DecisionTreeClassificationTrainer} that require to start the whole Ignite infrastructure.
*/
@@ -77,7 +77,8 @@ public class DecisionTreeClassificationTrainerIntegrationTest extends GridCommon
DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(1, 0);
DecisionTreeNode tree = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, data),
+ ignite,
+ data,
(k, v) -> Arrays.copyOf(v, v.length - 1),
(k, v) -> v[v.length - 1]
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
index 2599bfe..12ef698 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
@@ -17,17 +17,12 @@
package org.apache.ignite.ml.tree;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import java.util.*;
+
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertTrue;
@@ -68,7 +63,8 @@ public class DecisionTreeClassificationTrainerTest {
DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(1, 0);
DecisionTreeNode tree = trainer.fit(
- new LocalDatasetBuilder<>(data, parts),
+ data,
+ parts,
(k, v) -> Arrays.copyOf(v, v.length - 1),
(k, v) -> v[v.length - 1]
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
index 754ff20..c2a4638 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
@@ -17,16 +17,16 @@
package org.apache.ignite.ml.tree;
-import java.util.Arrays;
-import java.util.Random;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import java.util.Arrays;
+import java.util.Random;
+
/**
* Tests for {@link DecisionTreeRegressionTrainer} that require to start the whole Ignite infrastructure.
*/
@@ -77,7 +77,8 @@ public class DecisionTreeRegressionTrainerIntegrationTest extends GridCommonAbst
DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(1, 0);
DecisionTreeNode tree = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, data),
+ ignite,
+ data,
(k, v) -> Arrays.copyOf(v, v.length - 1),
(k, v) -> v[v.length - 1]
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
index 3bdbf60..bcfb53f 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
@@ -17,17 +17,12 @@
package org.apache.ignite.ml.tree;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import java.util.*;
+
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertTrue;
@@ -68,7 +63,8 @@ public class DecisionTreeRegressionTrainerTest {
DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(1, 0);
DecisionTreeNode tree = trainer.fit(
- new LocalDatasetBuilder<>(data, parts),
+ data,
+ parts,
(k, v) -> Arrays.copyOf(v, v.length - 1),
(k, v) -> v[v.length - 1]
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
index b259ec9..35f805e 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
@@ -17,13 +17,11 @@
package org.apache.ignite.ml.tree.performance;
-import java.io.IOException;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.nn.performance.MnistMLPTestUtil;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
@@ -31,6 +29,8 @@ import org.apache.ignite.ml.tree.impurity.util.SimpleStepFunctionCompressor;
import org.apache.ignite.ml.util.MnistUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import java.io.IOException;
+
/**
* Tests {@link DecisionTreeClassificationTrainer} on the MNIST dataset that require to start the whole Ignite
* infrastructure. For manual run.
@@ -81,7 +81,8 @@ public class DecisionTreeMNISTIntegrationTest extends GridCommonAbstractTest {
new SimpleStepFunctionCompressor<>());
DecisionTreeNode mdl = trainer.fit(
- new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+ ignite,
+ trainingSet,
(k, v) -> v.getPixels(),
(k, v) -> (double) v.getLabel()
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df6356d5/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
index 6dbd44c..b40c7ac 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
@@ -17,10 +17,6 @@
package org.apache.ignite.ml.tree.performance;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
import org.apache.ignite.ml.nn.performance.MnistMLPTestUtil;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
@@ -28,6 +24,10 @@ import org.apache.ignite.ml.tree.impurity.util.SimpleStepFunctionCompressor;
import org.apache.ignite.ml.util.MnistUtils;
import org.junit.Test;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
import static junit.framework.TestCase.assertTrue;
/**
@@ -50,7 +50,8 @@ public class DecisionTreeMNISTTest {
new SimpleStepFunctionCompressor<>());
DecisionTreeNode mdl = trainer.fit(
- new LocalDatasetBuilder<>(trainingSet, 10),
+ trainingSet,
+ 10,
(k, v) -> v.getPixels(),
(k, v) -> (double) v.getLabel()
);
[53/54] [abbrv] ignite git commit: IGNITE-8240 .NET: Use default
scheduler when starting Tasks
Posted by ag...@apache.org.
IGNITE-8240 .NET: Use default scheduler when starting Tasks
This closes #3812
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/44c4656c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/44c4656c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/44c4656c
Branch: refs/heads/ignite-6083
Commit: 44c4656ce8cc304b0858a18438f57fd9daf9ecb0
Parents: 6df5f99
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Fri Apr 13 12:28:19 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Fri Apr 13 12:28:19 2018 +0300
----------------------------------------------------------------------
.gitignore | 1 +
.../IgniteSessionStateStoreProviderTest.cs | 7 +-
.../Apache.Ignite.Benchmarks/BenchmarkRunner.cs | 1 -
.../Binary/BinaryDynamicRegistrationTest.cs | 2 +-
.../Cache/CacheAbstractTransactionalTest.cs | 5 +-
.../Client/ClientConnectionTest.cs | 3 +-
.../Apache.Ignite.Core.Tests/EventsTest.cs | 9 +--
.../Apache.Ignite.Core.Tests/ExceptionsTest.cs | 3 +-
.../IgniteStartStopTest.cs | 5 +-
.../Apache.Ignite.Core.Tests/MessagingTest.cs | 5 +-
.../Apache.Ignite.Core.csproj | 1 +
.../Impl/Client/ClientSocket.cs | 6 +-
.../Impl/Common/TaskRunner.cs | 70 ++++++++++++++++++++
.../Impl/Datastream/DataStreamerBatch.cs | 2 +-
.../Impl/Datastream/DataStreamerImpl.cs | 2 +-
.../Apache.Ignite.Core/Impl/Events/Events.cs | 2 +-
.../Impl/Transactions/TransactionImpl.cs | 3 +-
17 files changed, 103 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 535a8ff..47220b2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,7 @@ git-patch-prop-local.sh
**/dotnet/libs/
*.classname*
*.exe
+.mvn/
#Visual Studio files
*.[Oo]bj
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/IgniteSessionStateStoreProviderTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/IgniteSessionStateStoreProviderTest.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/IgniteSessionStateStoreProviderTest.cs
index 25700c6..08c44a6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/IgniteSessionStateStoreProviderTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/IgniteSessionStateStoreProviderTest.cs
@@ -28,6 +28,7 @@ namespace Apache.Ignite.AspNet.Tests
using System.Web.SessionState;
using Apache.Ignite.Core;
using Apache.Ignite.Core.Common;
+ using Apache.Ignite.Core.Impl.Common;
using Apache.Ignite.Core.Tests;
using NUnit.Framework;
@@ -265,7 +266,7 @@ namespace Apache.Ignite.AspNet.Tests
Assert.AreEqual(SessionStateActions.None, actions);
// Try to get it in a different thread.
- Task.Factory.StartNew(() =>
+ TaskRunner.Run(() =>
{
object lockId1; // do not overwrite lockId
res = provider.GetItem(HttpContext, Id, out locked, out lockAge, out lockId1, out actions);
@@ -277,7 +278,7 @@ namespace Apache.Ignite.AspNet.Tests
}).Wait();
// Try to get it in a different thread.
- Task.Factory.StartNew(() =>
+ TaskRunner.Run(() =>
{
object lockId1; // do not overwrite lockId
res = provider.GetItemExclusive(HttpContext, Id, out locked, out lockAge, out lockId1, out actions);
@@ -292,7 +293,7 @@ namespace Apache.Ignite.AspNet.Tests
provider.ReleaseItemExclusive(HttpContext, Id, lockId);
// Make sure it is accessible in a different thread.
- Task.Factory.StartNew(() =>
+ TaskRunner.Run(() =>
{
res = provider.GetItem(HttpContext, Id, out locked, out lockAge, out lockId, out actions);
Assert.IsNotNull(res);
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Benchmarks/BenchmarkRunner.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/BenchmarkRunner.cs b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/BenchmarkRunner.cs
index e152ffb..fb2fbd2 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/BenchmarkRunner.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/BenchmarkRunner.cs
@@ -22,7 +22,6 @@ namespace Apache.Ignite.Benchmarks
using System.IO;
using System.Text;
using Apache.Ignite.Benchmarks.Interop;
- using Apache.Ignite.Benchmarks.ThinClient;
/// <summary>
/// Benchmark runner.
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryDynamicRegistrationTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryDynamicRegistrationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryDynamicRegistrationTest.cs
index e635bd1..272a0ca 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryDynamicRegistrationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryDynamicRegistrationTest.cs
@@ -399,7 +399,7 @@ namespace Apache.Ignite.Core.Tests.Binary
};
var tasks = Enumerable.Range(0, threads)
- .Select(x => Task.Factory.StartNew(registerType))
+ .Select(x => TaskRunner.Run(registerType))
.ToArray();
Task.WaitAll(tasks);
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
index 2602a02..3d0168c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
@@ -25,6 +25,7 @@ namespace Apache.Ignite.Core.Tests.Cache
using System.Transactions;
using Apache.Ignite.Core.Cache;
using Apache.Ignite.Core.Cache.Configuration;
+ using Apache.Ignite.Core.Impl.Common;
using Apache.Ignite.Core.Transactions;
using NUnit.Framework;
@@ -563,8 +564,8 @@ namespace Apache.Ignite.Core.Tests.Cache
var aex = Assert.Throws<AggregateException>(() =>
Task.WaitAll(new[]
{
- Task.Factory.StartNew(() => increment(keys0)),
- Task.Factory.StartNew(() => increment(keys0.Reverse().ToArray()))
+ TaskRunner.Run(() => increment(keys0)),
+ TaskRunner.Run(() => increment(keys0.Reverse().ToArray()))
},
TimeSpan.FromSeconds(40)));
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
index 2ea17a8..cb30f40 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
@@ -31,6 +31,7 @@ namespace Apache.Ignite.Core.Tests.Client
using Apache.Ignite.Core.Client;
using Apache.Ignite.Core.Client.Cache;
using Apache.Ignite.Core.Configuration;
+ using Apache.Ignite.Core.Impl.Common;
using NUnit.Framework;
/// <summary>
@@ -310,7 +311,7 @@ namespace Apache.Ignite.Core.Tests.Client
var evt = new ManualResetEventSlim();
var ignite = Ignition.Start(TestUtils.GetTestConfiguration());
- var putGetTask = Task.Factory.StartNew(() =>
+ var putGetTask = TaskRunner.Run(() =>
{
using (var client = StartClient())
{
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs
index a7c0534..e9bac02 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs
@@ -34,6 +34,7 @@ namespace Apache.Ignite.Core.Tests
using Apache.Ignite.Core.Common;
using Apache.Ignite.Core.Events;
using Apache.Ignite.Core.Impl;
+ using Apache.Ignite.Core.Impl.Common;
using Apache.Ignite.Core.Impl.Events;
using Apache.Ignite.Core.Resource;
using Apache.Ignite.Core.Tests.Compute;
@@ -385,14 +386,14 @@ namespace Apache.Ignite.Core.Tests
/// </summary>
private static IEnumerable<Func<IEventFilter<IEvent>, int[], Task<IEvent>>> GetWaitTasks(IEvents events)
{
- yield return (filter, types) => Task.Factory.StartNew(() => events.WaitForLocal(types));
- yield return (filter, types) => Task.Factory.StartNew(() => events.WaitForLocal(types.ToList()));
+ yield return (filter, types) => TaskRunner.Run(() => events.WaitForLocal(types));
+ yield return (filter, types) => TaskRunner.Run(() => events.WaitForLocal(types.ToList()));
yield return (filter, types) => events.WaitForLocalAsync(types);
yield return (filter, types) => events.WaitForLocalAsync(types.ToList());
- yield return (filter, types) => Task.Factory.StartNew(() => events.WaitForLocal(filter, types));
- yield return (filter, types) => Task.Factory.StartNew(() => events.WaitForLocal(filter, types.ToList()));
+ yield return (filter, types) => TaskRunner.Run(() => events.WaitForLocal(filter, types));
+ yield return (filter, types) => TaskRunner.Run(() => events.WaitForLocal(filter, types.ToList()));
yield return (filter, types) => events.WaitForLocalAsync(filter, types);
yield return (filter, types) => events.WaitForLocalAsync(filter, types.ToList());
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExceptionsTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExceptionsTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExceptionsTest.cs
index f7568ef..0b06ea3 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExceptionsTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExceptionsTest.cs
@@ -29,6 +29,7 @@ namespace Apache.Ignite.Core.Tests
using Apache.Ignite.Core.Cluster;
using Apache.Ignite.Core.Common;
using Apache.Ignite.Core.Compute;
+ using Apache.Ignite.Core.Impl.Common;
using Apache.Ignite.Core.Services;
using Apache.Ignite.Core.Transactions;
using NUnit.Framework;
@@ -348,7 +349,7 @@ namespace Apache.Ignite.Core.Tests
cache = cache.WithKeepBinary<TK, int>();
// Do cache puts in parallel
- var putTask = Task.Factory.StartNew(() =>
+ var putTask = TaskRunner.Run(() =>
{
try
{
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
index 792b33d..f9c1cad 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
@@ -23,6 +23,7 @@ namespace Apache.Ignite.Core.Tests
using System.Threading;
using System.Threading.Tasks;
using Apache.Ignite.Core.Common;
+ using Apache.Ignite.Core.Impl.Common;
using Apache.Ignite.Core.Messaging;
using Apache.Ignite.Core.Tests.Process;
using NUnit.Framework;
@@ -207,7 +208,7 @@ namespace Apache.Ignite.Core.Tests
if (i % 2 == 0) // Try to stop ignite from another thread.
{
- Task.Factory.StartNew(() => grid.Dispose()).Wait();
+ TaskRunner.Run(() => grid.Dispose()).Wait();
}
else
{
@@ -306,7 +307,7 @@ namespace Apache.Ignite.Core.Tests
// Spam message subscriptions on a separate thread
// to test race conditions during processor init on remote node
- var listenTask = Task.Factory.StartNew(() =>
+ var listenTask = TaskRunner.Run(() =>
{
var filter = new MessageListener();
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core.Tests/MessagingTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/MessagingTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/MessagingTest.cs
index e644e31..7db4eef 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/MessagingTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/MessagingTest.cs
@@ -27,6 +27,7 @@ namespace Apache.Ignite.Core.Tests
using Apache.Ignite.Core.Cache.Configuration;
using Apache.Ignite.Core.Cluster;
using Apache.Ignite.Core.Common;
+ using Apache.Ignite.Core.Impl.Common;
using Apache.Ignite.Core.Messaging;
using Apache.Ignite.Core.Resource;
using Apache.Ignite.Core.Tests.Cache;
@@ -252,7 +253,7 @@ namespace Apache.Ignite.Core.Tests
var messaging = _grid1.GetMessaging();
- var senders = Task.Factory.StartNew(() => TestUtils.RunMultiThreaded(() =>
+ var senders = TaskRunner.Run(() => TestUtils.RunMultiThreaded(() =>
{
messaging.Send(NextMessage());
Thread.Sleep(50);
@@ -423,7 +424,7 @@ namespace Apache.Ignite.Core.Tests
var messaging = _grid1.GetMessaging();
- var senders = Task.Factory.StartNew(() => TestUtils.RunMultiThreaded(() =>
+ var senders = TaskRunner.Run(() => TestUtils.RunMultiThreaded(() =>
{
MessagingTestHelper.ClearReceived(int.MaxValue);
messaging.Send(NextMessage());
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
index ec84a38..93c45c3 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
@@ -72,6 +72,7 @@
<Compile Include="Client\IgniteClientException.cs" />
<Compile Include="Client\IIgniteClient.cs" />
<Compile Include="Common\ExceptionFactory.cs" />
+ <Compile Include="Impl\Common\TaskRunner.cs" />
<Compile Include="Ssl\ISslContextFactory.cs" />
<Compile Include="Configuration\Package-Info.cs" />
<Compile Include="Configuration\ClientConnectorConfiguration.cs" />
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
index 27d8f0b..bce681f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
@@ -28,11 +28,11 @@ namespace Apache.Ignite.Core.Impl.Client
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
- using System.Xml.Schema;
using Apache.Ignite.Core.Client;
using Apache.Ignite.Core.Common;
using Apache.Ignite.Core.Impl.Binary;
using Apache.Ignite.Core.Impl.Binary.IO;
+ using Apache.Ignite.Core.Impl.Common;
/// <summary>
/// Wrapper over framework socket for Ignite thin client operations.
@@ -117,7 +117,7 @@ namespace Apache.Ignite.Core.Impl.Client
}
// Continuously and asynchronously wait for data from server.
- Task.Factory.StartNew(WaitForMessages);
+ TaskRunner.Run(WaitForMessages);
}
/// <summary>
@@ -174,7 +174,7 @@ namespace Apache.Ignite.Core.Impl.Client
var task = SendRequestAsync(ref reqMsg);
// Decode.
- return task.ContinueWith(responseTask => DecodeResponse(responseTask.Result, readFunc, errorFunc));
+ return task.ContWith(responseTask => DecodeResponse(responseTask.Result, readFunc, errorFunc));
}
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/TaskRunner.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/TaskRunner.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/TaskRunner.cs
new file mode 100644
index 0000000..51a7c6a
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/TaskRunner.cs
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Core.Impl.Common
+{
+ using System;
+ using System.Threading;
+ using System.Threading.Tasks;
+
+ /// <summary>
+ /// Extensions for <see cref="Task"/> classes.
+ /// Fixes the issue with <see cref="TaskScheduler.Current"/> being used by defaut by system APIs.
+ /// </summary>
+ internal static class TaskRunner
+ {
+ /// <summary>
+ /// ContinueWith using default scheduler.
+ /// </summary>
+ public static Task<TNewResult> ContWith<TResult, TNewResult>(this Task<TResult> task,
+ Func<Task<TResult>, TNewResult> continuationFunction)
+ {
+ IgniteArgumentCheck.NotNull(task, "task");
+
+ return task.ContinueWith(continuationFunction, TaskScheduler.Default);
+ }
+
+ /// <summary>
+ /// ContinueWith using default scheduler.
+ /// </summary>
+ public static Task ContWith(this Task task,
+ Action<Task> continuationFunction)
+ {
+ IgniteArgumentCheck.NotNull(task, "task");
+
+ return task.ContinueWith(continuationFunction, TaskScheduler.Default);
+ }
+
+ /// <summary>
+ /// Run new task using default scheduler.
+ /// </summary>
+ public static Task Run(Action action)
+ {
+ return Task.Factory.StartNew(action, CancellationToken.None, TaskCreationOptions.None,
+ TaskScheduler.Default);
+ }
+
+ /// <summary>
+ /// Run new task using default scheduler.
+ /// </summary>
+ public static Task<TResult> Run<TResult>(Func<TResult> func)
+ {
+ return Task.Factory.StartNew(func, CancellationToken.None, TaskCreationOptions.None,
+ TaskScheduler.Default);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerBatch.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerBatch.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerBatch.cs
index 38a8ea8..0026701 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerBatch.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerBatch.cs
@@ -69,7 +69,7 @@ namespace Apache.Ignite.Core.Impl.Datastream
if (prev != null)
Thread.MemoryBarrier(); // Prevent "prev" field escape.
- _fut.Task.ContinueWith(x => ParentsCompleted());
+ _fut.Task.ContWith(x => ParentsCompleted());
}
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs
index 555c6e6..7aaa84a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs
@@ -897,7 +897,7 @@ namespace Apache.Ignite.Core.Impl.Datastream
/// </summary>
public void RunThread()
{
- Task.Factory.StartNew(Run);
+ TaskRunner.Run(Run);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs
index a81523a..04cc210 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs
@@ -241,7 +241,7 @@ namespace Apache.Ignite.Core.Impl.Events
if (hnd != null)
{
// Dispose handle as soon as future ends.
- task.ContinueWith(x => Ignite.HandleRegistry.Release(hnd.Value));
+ task.ContWith(x => Ignite.HandleRegistry.Release(hnd.Value));
}
return task;
http://git-wip-us.apache.org/repos/asf/ignite/blob/44c4656c/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionImpl.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionImpl.cs
index 0b04a68..c800859 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionImpl.cs
@@ -21,6 +21,7 @@ namespace Apache.Ignite.Core.Impl.Transactions
using System.Globalization;
using System.Threading;
using System.Threading.Tasks;
+ using Apache.Ignite.Core.Impl.Common;
using Apache.Ignite.Core.Transactions;
/// <summary>
@@ -457,7 +458,7 @@ namespace Apache.Ignite.Core.Impl.Transactions
/// </summary>
private Task CloseWhenComplete(Task task)
{
- return task.ContinueWith(x => Close());
+ return task.ContWith(x => Close());
}
/** <inheritdoc /> */
[24/54] [abbrv] ignite git commit: IGNITE-8101 Ability to terminate
system workers by JMX for test purposes.
Posted by ag...@apache.org.
IGNITE-8101 Ability to terminate system workers by JMX for test purposes.
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a96ac047
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a96ac047
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a96ac047
Branch: refs/heads/ignite-6083
Commit: a96ac04755d2c7448508990fe9e1dad62bcccc90
Parents: 3a71765
Author: Dmitriy Sorokin <d....@gmail.com>
Authored: Tue Apr 10 22:20:41 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Apr 10 22:20:41 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 7 ++
.../failure/StopNodeOrHaltFailureHandler.java | 2 +-
.../ignite/internal/GridKernalContext.java | 8 ++
.../ignite/internal/GridKernalContextImpl.java | 10 +++
.../apache/ignite/internal/IgniteKernal.java | 16 +++-
.../discovery/GridDiscoveryManager.java | 2 +-
.../GridCachePartitionExchangeManager.java | 3 +-
.../cache/GridCacheSharedTtlCleanupManager.java | 3 +-
.../wal/reader/StandaloneGridKernalContext.java | 6 ++
.../timeout/GridTimeoutProcessor.java | 3 +-
.../ignite/internal/util/IgniteUtils.java | 7 +-
.../worker/WorkersControlMXBeanImpl.java | 62 +++++++++++++++
.../ignite/internal/worker/WorkersRegistry.java | 80 ++++++++++++++++++++
.../ignite/internal/worker/package-info.java | 22 ++++++
.../ignite/mxbean/WorkersControlMXBean.java | 49 ++++++++++++
15 files changed, 271 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 152d845..9da123e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -423,6 +423,13 @@ public final class IgniteSystemProperties {
public static final String IGNITE_MBEANS_DISABLED = "IGNITE_MBEANS_DISABLED";
/**
+ * If property is set to {@code true}, then test features will be enabled.
+ *
+ * Default is {@code false}.
+ */
+ public static final String IGNITE_TEST_FEATURES_ENABLED = "IGNITE_TEST_FEATURES_ENABLED";
+
+ /**
* Property controlling size of buffer holding last exception. Default value of {@code 1000}.
*/
public static final String IGNITE_EXCEPTION_REGISTRY_MAX_SIZE = "IGNITE_EXCEPTION_REGISTRY_MAX_SIZE";
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
index 4f74406..3ce4ff6 100644
--- a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
@@ -92,7 +92,7 @@ public class StopNodeOrHaltFailureHandler implements FailureHandler {
).start();
}
else {
- U.error(log, "JVM will be halted immediately on ignite failure: [failureCtx=" + failureCtx + ']');
+ U.error(log, "JVM will be halted immediately due to the failure: [failureCtx=" + failureCtx + ']');
Runtime.getRuntime().halt(Ignition.KILL_EXIT_CODE);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index 0b40054..505c3d6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -32,6 +32,7 @@ import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.managers.failover.GridFailoverManager;
import org.apache.ignite.internal.managers.indexing.GridIndexingManager;
import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager;
+import org.apache.ignite.internal.worker.WorkersRegistry;
import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
import org.apache.ignite.internal.processors.authentication.IgniteAuthenticationProcessor;
import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
@@ -423,6 +424,13 @@ public interface GridKernalContext extends Iterable<GridComponent> {
public GridIndexingManager indexing();
/**
+ * Gets workers registry.
+ *
+ * @return Workers registry.
+ */
+ public WorkersRegistry workersRegistry();
+
+ /**
* Gets data structures processor.
*
* @return Data structures processor.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 3408334..ac49708 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -46,6 +46,7 @@ import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.managers.failover.GridFailoverManager;
import org.apache.ignite.internal.managers.indexing.GridIndexingManager;
import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager;
+import org.apache.ignite.internal.worker.WorkersRegistry;
import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
import org.apache.ignite.internal.processors.authentication.IgniteAuthenticationProcessor;
import org.apache.ignite.internal.processors.cache.CacheConflictResolutionManager;
@@ -361,6 +362,10 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
private Map<String, Object> attrs = new HashMap<>();
/** */
+ @GridToStringExclude
+ private final WorkersRegistry workersRegistry = new WorkersRegistry();
+
+ /** */
private IgniteEx grid;
/** */
@@ -780,6 +785,11 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
}
/** {@inheritDoc} */
+ @Override public WorkersRegistry workersRegistry() {
+ return workersRegistry;
+ }
+
+ /** {@inheritDoc} */
@Override public GridAffinityProcessor affinity() {
return affProc;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 0b102e5..1cb07b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -125,7 +125,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
import org.apache.ignite.internal.processors.cache.GridCacheUtilityKey;
-import org.apache.ignite.internal.processors.cache.GridCacheUtils;
import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
@@ -185,6 +184,8 @@ import org.apache.ignite.internal.util.typedef.internal.LT;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.worker.WorkersControlMXBeanImpl;
+import org.apache.ignite.internal.worker.WorkersRegistry;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgnitePredicate;
@@ -197,6 +198,7 @@ import org.apache.ignite.marshaller.jdk.JdkMarshaller;
import org.apache.ignite.mxbean.ClusterMetricsMXBean;
import org.apache.ignite.mxbean.IgniteMXBean;
import org.apache.ignite.mxbean.StripedExecutorMXBean;
+import org.apache.ignite.mxbean.WorkersControlMXBean;
import org.apache.ignite.mxbean.ThreadPoolMXBean;
import org.apache.ignite.plugin.IgnitePlugin;
import org.apache.ignite.plugin.PluginNotFoundException;
@@ -1085,7 +1087,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
// Register MBeans.
mBeansMgr.registerAllMBeans(utilityCachePool, execSvc, svcExecSvc, sysExecSvc, stripedExecSvc, p2pExecSvc,
mgmtExecSvc, igfsExecSvc, dataStreamExecSvc, restExecSvc, affExecSvc, idxExecSvc, callbackExecSvc,
- qryExecSvc, schemaExecSvc, customExecSvcs);
+ qryExecSvc, schemaExecSvc, customExecSvcs, ctx.workersRegistry());
// Lifecycle bean notifications.
notifyLifecycleBeans(AFTER_NODE_START);
@@ -4175,7 +4177,8 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
IgniteStripedThreadPoolExecutor callbackExecSvc,
ExecutorService qryExecSvc,
ExecutorService schemaExecSvc,
- @Nullable final Map<String, ? extends ExecutorService> customExecSvcs
+ @Nullable final Map<String, ? extends ExecutorService> customExecSvcs,
+ WorkersRegistry workersRegistry
) throws IgniteCheckedException {
if (U.IGNITE_MBEANS_DISABLED)
return;
@@ -4221,6 +4224,13 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
for (Map.Entry<String, ? extends ExecutorService> entry : customExecSvcs.entrySet())
registerExecutorMBean(entry.getKey(), entry.getValue());
}
+
+ if (U.IGNITE_TEST_FEATURES_ENABLED) {
+ WorkersControlMXBean workerCtrlMXBean = new WorkersControlMXBeanImpl(workersRegistry);
+
+ registerMBean("Kernal", workerCtrlMXBean.getClass().getSimpleName(),
+ workerCtrlMXBean, WorkersControlMXBean.class);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index b0d3256..a1d84e5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -2591,7 +2591,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
*
*/
private DiscoveryWorker() {
- super(ctx.igniteInstanceName(), "disco-event-worker", GridDiscoveryManager.this.log);
+ super(ctx.igniteInstanceName(), "disco-event-worker", GridDiscoveryManager.this.log, ctx.workersRegistry());
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index e40493f..1a0e65f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -2077,7 +2077,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
* Constructor.
*/
private ExchangeWorker() {
- super(cctx.igniteInstanceName(), "partition-exchanger", GridCachePartitionExchangeManager.this.log);
+ super(cctx.igniteInstanceName(), "partition-exchanger", GridCachePartitionExchangeManager.this.log,
+ cctx.kernalContext().workersRegistry());
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
index 613e93b..7adabc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
@@ -121,7 +121,8 @@ public class GridCacheSharedTtlCleanupManager extends GridCacheSharedManagerAdap
* Creates cleanup worker.
*/
CleanupWorker() {
- super(cctx.igniteInstanceName(), "ttl-cleanup-worker", cctx.logger(GridCacheSharedTtlCleanupManager.class));
+ super(cctx.igniteInstanceName(), "ttl-cleanup-worker", cctx.logger(GridCacheSharedTtlCleanupManager.class),
+ cctx.kernalContext().workersRegistry());
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index 429a5ce..cb04575 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -86,6 +86,7 @@ import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
import org.apache.ignite.internal.util.IgniteExceptionRegistry;
import org.apache.ignite.internal.util.StripedExecutor;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.worker.WorkersRegistry;
import org.apache.ignite.marshaller.Marshaller;
import org.apache.ignite.plugin.PluginNotFoundException;
import org.apache.ignite.plugin.PluginProvider;
@@ -454,6 +455,11 @@ public class StandaloneGridKernalContext implements GridKernalContext {
}
/** {@inheritDoc} */
+ @Override public WorkersRegistry workersRegistry() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Override public DataStructuresProcessor dataStructures() {
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
index a09d6fa..25151cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
@@ -145,7 +145,8 @@ public class GridTimeoutProcessor extends GridProcessorAdapter {
*
*/
TimeoutWorker() {
- super(ctx.config().getIgniteInstanceName(), "grid-timeout-worker", GridTimeoutProcessor.this.log);
+ super(ctx.config().getIgniteInstanceName(), "grid-timeout-worker",
+ GridTimeoutProcessor.this.log, ctx.workersRegistry());
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 93f4fb4..42e96fb 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -522,7 +522,12 @@ public abstract class IgniteUtils {
};
/** Ignite MBeans disabled flag. */
- public static boolean IGNITE_MBEANS_DISABLED = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_MBEANS_DISABLED);
+ public static boolean IGNITE_MBEANS_DISABLED =
+ IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_MBEANS_DISABLED);
+
+ /** Ignite test features enabled flag. */
+ public static boolean IGNITE_TEST_FEATURES_ENABLED =
+ IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_TEST_FEATURES_ENABLED);
/** */
private static final boolean assertionsEnabled;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
new file mode 100644
index 0000000..9e427e8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.worker;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.mxbean.WorkersControlMXBean;
+
+/**
+ * MBean that provides control of system workersRegistry.
+ */
+public class WorkersControlMXBeanImpl implements WorkersControlMXBean {
+ /** System worker registry. */
+ private final WorkersRegistry workerRegistry;
+
+ /**
+ * Constructor.
+ *
+ * @param registry System worker registry.
+ */
+ public WorkersControlMXBeanImpl(WorkersRegistry registry) {
+ workerRegistry = registry;
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<String> getWorkerNames() {
+ return new ArrayList<>(workerRegistry.names());
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean terminateWorker(String name) {
+ GridWorker w = workerRegistry.worker(name);
+
+ if (w == null || w.isCancelled())
+ return false;
+
+ Thread t = w.runner();
+
+ if (t == null)
+ return false;
+
+ t.interrupt();
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
new file mode 100644
index 0000000..e8d46fb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.worker;
+
+import java.util.Collection;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.internal.util.worker.GridWorkerListener;
+
+/**
+ * Workers registry.
+ */
+public class WorkersRegistry implements GridWorkerListener {
+ /** Registered workers. */
+ private final ConcurrentMap<String, GridWorker> registeredWorkers = new ConcurrentHashMap<>();
+
+ /**
+ * Adds worker to the registry.
+ *
+ * @param w Worker.
+ */
+ public void register(GridWorker w) {
+ if (registeredWorkers.putIfAbsent(w.name(), w) != null)
+ throw new IllegalStateException("Worker is already registered [worker=" + w + ']');
+ }
+
+ /**
+ * Removes worker from the registry.
+ *
+ * @param name Worker name.
+ */
+ public void unregister(String name) {
+ registeredWorkers.remove(name);
+ }
+
+ /**
+ * Returns names of all registered workers.
+ *
+ * @return Registered worker names.
+ */
+ public Collection<String> names() {
+ return registeredWorkers.keySet();
+ }
+
+ /**
+ * Returns worker with given name.
+ *
+ * @param name Name.
+ * @return Registered {@link GridWorker} with name {@code name} or {@code null} if not found.
+ */
+ public GridWorker worker(String name) {
+ return registeredWorkers.get(name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onStarted(GridWorker w) {
+ register(w);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onStopped(GridWorker w) {
+ unregister(w.name());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java
new file mode 100644
index 0000000..03ca621
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * System worker registry and control MBean implementation.
+ */
+package org.apache.ignite.internal.worker;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a96ac047/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java
new file mode 100644
index 0000000..0f5419b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mxbean;
+
+import java.util.List;
+
+/**
+ * MBean that provides ability to terminate worker that registered in the workers registry.
+ */
+@MXBeanDescription("MBean that provides ability to terminate worker that registered in the workers registry.")
+public interface WorkersControlMXBean {
+ /**
+ * Returns names of all registered workers.
+ *
+ * @return Worker names.
+ */
+ @MXBeanDescription("Names of registered workers.")
+ public List<String> getWorkerNames();
+
+ /**
+ * Terminates worker.
+ *
+ * @param name Worker name.
+ * @return {@code True} if worker has been terminated successfully, {@code false} otherwise.
+ */
+ @MXBeanDescription("Terminates worker.")
+ @MXBeanParametersNames(
+ "name"
+ )
+ @MXBeanParametersDescriptions(
+ "Name of worker to terminate."
+ )
+ public boolean terminateWorker(String name);
+}
[50/54] [abbrv] ignite git commit: IGNITE-8110
GridCacheWriteBehindStore.Flusher thread uses the wrong transformation from
milliseconds to nanoseconds. - Fixes #3742.
Posted by ag...@apache.org.
IGNITE-8110 GridCacheWriteBehindStore.Flusher thread uses the wrong transformation from milliseconds to nanoseconds. - Fixes #3742.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/adaedb44
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/adaedb44
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/adaedb44
Branch: refs/heads/ignite-6083
Commit: adaedb44b767da49c4ec6c57c655b13caa3edd35
Parents: 47ca3e0
Author: Anton Kurbanov <an...@gmail.com>
Authored: Thu Apr 12 20:31:50 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Thu Apr 12 20:31:50 2018 +0300
----------------------------------------------------------------------
.../cache/store/GridCacheWriteBehindStore.java | 2 +-
.../GridCacheWriteBehindStoreSelfTest.java | 41 +++++++++++++++++++-
2 files changed, 40 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/adaedb44/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
index 44cadd6..82ff3aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
@@ -895,7 +895,7 @@ public class GridCacheWriteBehindStore<K, V> implements CacheStore<K, V>, Lifecy
protected Thread thread;
/** Cache flushing frequence in nanos. */
- protected long cacheFlushFreqNanos = cacheFlushFreq * 1000;
+ protected long cacheFlushFreqNanos = cacheFlushFreq * 1000 * 1000;
/** Writer lock. */
private final Lock flusherWriterLock = new ReentrantLock();
http://git-wip-us.apache.org/repos/asf/ignite/blob/adaedb44/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java
index 9a487a4..af21fc8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java
@@ -107,6 +107,43 @@ public class GridCacheWriteBehindStoreSelfTest extends GridCacheWriteBehindStore
}
/**
+ * Checks that write behind cache flush frequency was correctly adjusted to nanos expecting putAllCnt to be
+ * less or equal than elapsed time divided by flush frequency.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSimpleStoreFlushFrequencyWithoutCoalescing() throws Exception {
+ initStore(1, false);
+
+ long writeBehindFlushFreqNanos = FLUSH_FREQUENCY * 1000 * 1000;
+
+ int threshold = store.getWriteBehindStoreBatchSize() / 10;
+
+ try {
+ long start = System.nanoTime();
+
+ for (int i = 0; i < threshold / 2; i++)
+ store.write(new CacheEntryImpl<>(i, "v" + i));
+
+ U.sleep(FLUSH_FREQUENCY + 300);
+
+ for (int i = threshold / 2; i < threshold; i++)
+ store.write(new CacheEntryImpl<>(i, "v" + i));
+
+ long elapsed = System.nanoTime() - start;
+
+ U.sleep(FLUSH_FREQUENCY + 300);
+
+ int expFlushOps = (int)(1 + elapsed / writeBehindFlushFreqNanos);
+
+ assertTrue(delegate.getPutAllCount() <= expFlushOps);
+ }
+ finally {
+ shutdownStore();
+ }
+ }
+
+ /**
* Simple store test.
*
* @param writeCoalescing Write coalescing flag.
@@ -254,7 +291,6 @@ public class GridCacheWriteBehindStoreSelfTest extends GridCacheWriteBehindStore
int delegatePutCnt = delegate.getPutAllCount();
-
fut.get();
log().info(">>> [putCnt = " + actualPutCnt.get() + ", delegatePutCnt=" + delegatePutCnt + "]");
@@ -262,7 +298,8 @@ public class GridCacheWriteBehindStoreSelfTest extends GridCacheWriteBehindStore
assertTrue("No puts were made to the underlying store", delegatePutCnt > 0);
if (store.getWriteCoalescing()) {
assertTrue("Too many puts were made to the underlying store", delegatePutCnt < actualPutCnt.get() / 10);
- } else {
+ }
+ else {
assertTrue("Too few puts cnt=" + actualPutCnt.get() + " << storePutCnt=" + delegatePutCnt, delegatePutCnt > actualPutCnt.get() / 2);
}
}
[34/54] [abbrv] ignite git commit: IGNITE-4756 Print info about
partition distribution to log
Posted by ag...@apache.org.
IGNITE-4756 Print info about partition distribution to log
Signed-off-by: Anton Vinogradov <av...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a3eb1f5d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a3eb1f5d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a3eb1f5d
Branch: refs/heads/ignite-6083
Commit: a3eb1f5d753a38c4019440e1bf39d00bc6136455
Parents: 0e73fa2
Author: Vyacheslav Daradur <da...@gmail.com>
Authored: Wed Apr 11 14:41:29 2018 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Wed Apr 11 14:41:29 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 7 +
.../affinity/GridAffinityAssignmentCache.java | 50 +++-
.../AffinityDistributionLoggingTest.java | 268 +++++++++++++++++++
.../testsuites/IgniteCacheTestSuite5.java | 9 +-
4 files changed, 327 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a3eb1f5d/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 9da123e..04eb425 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -857,6 +857,13 @@ public final class IgniteSystemProperties {
public static final String IGNITE_BPLUS_TREE_LOCK_RETRIES = "IGNITE_BPLUS_TREE_LOCK_RETRIES";
/**
+ * The threshold of uneven distribution above which partition distribution will be logged.
+ *
+ * The default is '50', that means: warn about nodes with 50+% difference.
+ */
+ public static final String IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD = "IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD";
+
+ /**
* Enforces singleton.
*/
private IgniteSystemProperties() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a3eb1f5d/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
index 18edd02..b1899e3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
@@ -34,13 +34,14 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cache.affinity.AffinityCentralizedFunction;
import org.apache.ignite.cache.affinity.AffinityFunction;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.cluster.NodeOrderComparator;
import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.cluster.NodeOrderComparator;
import org.apache.ignite.internal.managers.discovery.DiscoCache;
import org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents;
import org.apache.ignite.internal.processors.cluster.BaselineTopology;
@@ -53,7 +54,10 @@ import org.apache.ignite.lang.IgnitePredicate;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_AFFINITY_HISTORY_SIZE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD;
+import static org.apache.ignite.IgniteSystemProperties.getFloat;
import static org.apache.ignite.IgniteSystemProperties.getInteger;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
/**
@@ -63,6 +67,9 @@ public class GridAffinityAssignmentCache {
/** Cleanup history size. */
private final int MAX_HIST_SIZE = getInteger(IGNITE_AFFINITY_HISTORY_SIZE, 500);
+ /** Partition distribution. */
+ private final float partDistribution = getFloat(IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD, 50f);
+
/** Group name if specified or cache name. */
private final String cacheOrGrpName;
@@ -367,6 +374,9 @@ public class GridAffinityAssignmentCache {
idealAssignment = assignment;
+ if (ctx.cache().cacheMode(cacheOrGrpName) == PARTITIONED)
+ printDistributionIfThresholdExceeded(assignment, sorted.size());
+
if (hasBaseline) {
baselineTopology = discoCache.state().baselineTopology();
assert baselineAssignment != null;
@@ -418,6 +428,44 @@ public class GridAffinityAssignmentCache {
}
/**
+ * Calculates and logs partitions distribution if threshold of uneven distribution {@link #partDistribution} is exceeded.
+ *
+ * @param assignments Assignments to calculate partitions distribution.
+ * @param nodes Affinity nodes number.
+ * @see IgniteSystemProperties#IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD
+ */
+ private void printDistributionIfThresholdExceeded(List<List<ClusterNode>> assignments, int nodes) {
+ int locPrimaryCnt = 0;
+ int locBackupCnt = 0;
+
+ for (List<ClusterNode> assignment : assignments) {
+ for (int i = 0; i < assignment.size(); i++) {
+ ClusterNode node = assignment.get(i);
+
+ if (node.isLocal()) {
+ if (i == 0)
+ locPrimaryCnt++;
+ else
+ locBackupCnt++;
+ }
+ }
+ }
+
+ float expCnt = (float)partsCnt / nodes;
+
+ float deltaPrimary = Math.abs(1 - (float)locPrimaryCnt / expCnt) * 100;
+ float deltaBackup = Math.abs(1 - (float)locBackupCnt / (expCnt * backups)) * 100;
+
+ if (deltaPrimary > partDistribution || deltaBackup > partDistribution) {
+ log.info(String.format("Local node affinity assignment distribution is not ideal " +
+ "[cache=%s, expectedPrimary=%.2f, actualPrimary=%d, " +
+ "expectedBackups=%.2f, actualBackups=%d, warningThreshold=%.2f%%]",
+ cacheOrGrpName, expCnt, locPrimaryCnt,
+ expCnt * backups, locBackupCnt, partDistribution));
+ }
+ }
+
+ /**
* Copies previous affinity assignment when discovery event does not cause affinity assignment changes
* (e.g. client node joins on leaves).
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/a3eb1f5d/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityDistributionLoggingTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityDistributionLoggingTest.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityDistributionLoggingTest.java
new file mode 100644
index 0000000..813c830
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityDistributionLoggingTest.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.affinity;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.testframework.GridStringLogger;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
+
+/**
+ * Tests of partitions distribution logging.
+ *
+ * Tests based on using of affinity function which provides an even distribution of partitions between nodes.
+ *
+ * @see EvenDistributionAffinityFunction
+ */
+public class AffinityDistributionLoggingTest extends GridCommonAbstractTest {
+ /** Pattern to test. */
+ private static final String LOG_MESSAGE_PREFIX = "Local node affinity assignment distribution is not ideal ";
+
+ /** Partitions number. */
+ private int parts = 0;
+
+ /** Nodes number. */
+ private int nodes = 0;
+
+ /** Backups number. */
+ private int backups = 0;
+
+ /** For storing original value of system property. */
+ private String tempProp;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ tempProp = System.getProperty(IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ if (tempProp != null)
+ System.setProperty(IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD, tempProp);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ System.clearProperty(IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD);
+
+ stopAllGrids();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+ cacheCfg.setCacheMode(CacheMode.PARTITIONED);
+ cacheCfg.setBackups(backups);
+ cacheCfg.setAffinity(new EvenDistributionAffinityFunction(parts));
+
+ cfg.setCacheConfiguration(cacheCfg);
+
+ return cfg;
+ }
+
+ /**
+ * @throws Exception In case of an error.
+ */
+ public void test2PartitionsIdealDistributionIsNotLogged() throws Exception {
+ System.setProperty(IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD, "0");
+
+ nodes = 2;
+ parts = 2;
+ backups = 1;
+
+ String testsLog = runAndGetExchangeLog();
+
+ assertFalse(testsLog.contains(LOG_MESSAGE_PREFIX));
+ }
+
+ /**
+ * @throws Exception In case of an error.
+ */
+ public void test120PartitionsIdeadDistributionIsNotLogged() throws Exception {
+ System.setProperty(IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD, "0.0");
+
+ nodes = 3;
+ parts = 120;
+ backups = 2;
+
+ String testsLog = runAndGetExchangeLog();
+
+ assertFalse(testsLog.contains(LOG_MESSAGE_PREFIX));
+ }
+
+ /**
+ * @throws Exception In case of an error.
+ */
+ public void test5PartitionsNotIdealDistributionIsLogged() throws Exception {
+ System.setProperty(IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD, "50.0");
+
+ nodes = 4;
+ parts = 5;
+ backups = 3;
+
+ String testsLog = runAndGetExchangeLog();
+
+ assertTrue(testsLog.contains(LOG_MESSAGE_PREFIX));
+ }
+
+ /**
+ * @throws Exception In case of an error.
+ */
+ public void test7PartitionsNotIdealDistributionSuppressedLogging() throws Exception {
+ System.setProperty(IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD, "50.0");
+
+ nodes = 3;
+ parts = 7;
+ backups = 0;
+
+ String testsLog = runAndGetExchangeLog();
+
+ assertFalse(testsLog.contains(LOG_MESSAGE_PREFIX));
+ }
+
+ /**
+ * @throws Exception In case of an error.
+ */
+ public void test5PartitionsNotIdealDistributionSuppressedLogging() throws Exception {
+ System.setProperty(IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD, "65");
+
+ nodes = 4;
+ parts = 5;
+ backups = 3;
+
+ String testsLog = runAndGetExchangeLog();
+
+ assertFalse(testsLog.contains(LOG_MESSAGE_PREFIX));
+ }
+
+ /**
+ * Starts a specified number of Ignite nodes and log partition node exchange during a last node's startup.
+ *
+ * @return Log of latest partition map exchange.
+ * @throws Exception In case of an error.
+ */
+ private String runAndGetExchangeLog() throws Exception {
+ assert nodes > 1;
+
+ IgniteEx ignite = (IgniteEx)startGrids(nodes - 1);
+
+ awaitPartitionMapExchange();
+
+ GridCacheProcessor proc = ignite.context().cache();
+
+ GridCacheContext cctx = proc.context().cacheContext(CU.cacheId(DEFAULT_CACHE_NAME));
+
+ final GridStringLogger log = new GridStringLogger(false, this.log);
+
+ GridAffinityAssignmentCache aff = GridTestUtils.getFieldValue(cctx.affinity(), "aff");
+
+ GridTestUtils.setFieldValue(aff, "log", log);
+
+ startGrid(nodes);
+
+ awaitPartitionMapExchange();
+
+ return log.toString();
+ }
+
+ /**
+ * Affinity function for a partitioned cache which provides even distribution partitions between nodes in cluster.
+ */
+ private static class EvenDistributionAffinityFunction implements AffinityFunction {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Partitions number. */
+ private int parts;
+
+ /**
+ * @param parts Number of partitions for one cache.
+ */
+ private EvenDistributionAffinityFunction(int parts) {
+ this.parts = parts;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int partitions() {
+ return parts;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int partition(Object key) {
+ return key.hashCode() % parts;
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext affCtx) {
+ List<ClusterNode> nodes = new ArrayList<>(affCtx.currentTopologySnapshot());
+
+ nodes.sort(Comparator.comparing(o -> o.<String>attribute(ATTR_IGNITE_INSTANCE_NAME)));
+
+ List<List<ClusterNode>> res = new ArrayList<>(parts);
+
+ for (int i = 0; i < parts; i++) {
+ Set<ClusterNode> n0 = new LinkedHashSet<>();
+
+ n0.add(nodes.get(i % nodes.size()));
+
+ for (int j = 1; j <= affCtx.backups(); j++)
+ n0.add(nodes.get((i + j) % nodes.size()));
+
+ res.add(new ArrayList<>(n0));
+ }
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void removeNode(UUID nodeId) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void reset() {
+ // No-op.
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a3eb1f5d/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 7c41e49..945a76c 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -21,6 +21,7 @@ import junit.framework.TestSuite;
import org.apache.ignite.GridCacheAffinityBackupsSelfTest;
import org.apache.ignite.IgniteCacheAffinitySelfTest;
import org.apache.ignite.cache.affinity.AffinityClientNodeSelfTest;
+import org.apache.ignite.cache.affinity.AffinityDistributionLoggingTest;
import org.apache.ignite.cache.affinity.AffinityHistoryCleanupTest;
import org.apache.ignite.cache.affinity.local.LocalAffinityFunctionTest;
import org.apache.ignite.internal.GridCachePartitionExchangeManagerHistSizeTest;
@@ -35,13 +36,7 @@ import org.apache.ignite.internal.processors.cache.EntryVersionConsistencyReadTh
import org.apache.ignite.internal.processors.cache.IgniteCachePutStackOverflowSelfTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheReadThroughEvictionsVariationsSuite;
import org.apache.ignite.internal.processors.cache.IgniteCacheStoreCollectionTest;
-import org.apache.ignite.internal.processors.cache.PartitionedAtomicCacheGetsDistributionTest;
-import org.apache.ignite.internal.processors.cache.PartitionedTransactionalOptimisticCacheGetsDistributionTest;
-import org.apache.ignite.internal.processors.cache.PartitionedTransactionalPessimisticCacheGetsDistributionTest;
import org.apache.ignite.internal.processors.cache.PartitionsExchangeOnDiscoveryHistoryOverflowTest;
-import org.apache.ignite.internal.processors.cache.ReplicatedAtomicCacheGetsDistributionTest;
-import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalOptimisticCacheGetsDistributionTest;
-import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalPessimisticCacheGetsDistributionTest;
import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentTest;
@@ -95,6 +90,8 @@ public class IgniteCacheTestSuite5 extends TestSuite {
suite.addTestSuite(LocalAffinityFunctionTest.class);
suite.addTestSuite(AffinityHistoryCleanupTest.class);
+ suite.addTestSuite(AffinityDistributionLoggingTest.class);
+
suite.addTestSuite(IgniteCacheAtomicProtocolTest.class);
suite.addTestSuite(PartitionsExchangeOnDiscoveryHistoryOverflowTest.class);
[10/54] [abbrv] ignite git commit: IGNITE-7222 Added ZooKeeper
discovery SPI
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 01a5a71..cebe4b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -56,6 +56,8 @@ import org.apache.ignite.internal.managers.deployment.GridDeployment;
import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
import org.apache.ignite.internal.managers.deployment.GridDeploymentInfoBean;
import org.apache.ignite.internal.managers.discovery.CustomEventListener;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.discovery.DiscoveryMessageResultsCollector;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.managers.eventstorage.HighPriorityListener;
import org.apache.ignite.internal.processors.GridProcessorAdapter;
@@ -63,6 +65,7 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler;
import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -147,6 +150,12 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
/** Query sequence number for message topic. */
private final AtomicLong seq = new AtomicLong();
+ /** */
+ private ContinuousRoutinesInfo routinesInfo;
+
+ /** */
+ private int discoProtoVer;
+
/**
* @param ctx Kernal context.
*/
@@ -156,6 +165,11 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
/** {@inheritDoc} */
@Override public void start() throws IgniteCheckedException {
+ discoProtoVer = ctx.discovery().mutableCustomMessages() ? 1 : 2;
+
+ if (discoProtoVer == 2)
+ routinesInfo = new ContinuousRoutinesInfo();
+
if (ctx.config().isDaemon())
return;
@@ -177,6 +191,8 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
@Override public void onCustomEvent(AffinityTopologyVersion topVer,
ClusterNode snd,
StartRoutineDiscoveryMessage msg) {
+ assert discoProtoVer == 1 : discoProtoVer;
+
if (ctx.isStopping())
return;
@@ -184,6 +200,20 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
}
});
+ ctx.discovery().setCustomEventListener(StartRoutineDiscoveryMessageV2.class,
+ new CustomEventListener<StartRoutineDiscoveryMessageV2>() {
+ @Override public void onCustomEvent(AffinityTopologyVersion topVer,
+ ClusterNode snd,
+ StartRoutineDiscoveryMessageV2 msg) {
+ assert discoProtoVer == 2 : discoProtoVer;
+
+ if (ctx.isStopping())
+ return;
+
+ processStartRequestV2(topVer, snd, msg);
+ }
+ });
+
ctx.discovery().setCustomEventListener(StartRoutineAckDiscoveryMessage.class,
new CustomEventListener<StartRoutineAckDiscoveryMessage>() {
@Override public void onCustomEvent(AffinityTopologyVersion topVer,
@@ -201,6 +231,9 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
@Override public void onCustomEvent(AffinityTopologyVersion topVer,
ClusterNode snd,
StopRoutineDiscoveryMessage msg) {
+ if (discoProtoVer == 2)
+ routinesInfo.removeRoutine(msg.routineId);
+
if (ctx.isStopping())
return;
@@ -222,32 +255,36 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
ctx.io().addMessageListener(TOPIC_CONTINUOUS, new GridMessageListener() {
@Override public void onMessage(UUID nodeId, Object obj, byte plc) {
- GridContinuousMessage msg = (GridContinuousMessage)obj;
+ if (obj instanceof ContinuousRoutineStartResultMessage)
+ processRoutineStartResultMessage(nodeId, (ContinuousRoutineStartResultMessage)obj);
+ else {
+ GridContinuousMessage msg = (GridContinuousMessage)obj;
- if (msg.data() == null && msg.dataBytes() != null) {
- try {
- msg.data(U.unmarshal(marsh, msg.dataBytes(), U.resolveClassLoader(ctx.config())));
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to process message (ignoring): " + msg, e);
+ if (msg.data() == null && msg.dataBytes() != null) {
+ try {
+ msg.data(U.unmarshal(marsh, msg.dataBytes(), U.resolveClassLoader(ctx.config())));
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to process message (ignoring): " + msg, e);
- return;
+ return;
+ }
}
- }
- switch (msg.type()) {
- case MSG_EVT_NOTIFICATION:
- processNotification(nodeId, msg);
+ switch (msg.type()) {
+ case MSG_EVT_NOTIFICATION:
+ processNotification(nodeId, msg);
- break;
+ break;
- case MSG_EVT_ACK:
- processMessageAck(msg);
+ case MSG_EVT_ACK:
+ processMessageAck(msg);
- break;
+ break;
- default:
- assert false : "Unexpected message received: " + msg.type();
+ default:
+ assert false : "Unexpected message received: " + msg.type();
+ }
}
}
});
@@ -341,6 +378,15 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
/** {@inheritDoc} */
@Override public void collectJoiningNodeData(DiscoveryDataBag dataBag) {
+ if (ctx.isDaemon())
+ return;
+
+ if (discoProtoVer == 2) {
+ routinesInfo.collectJoiningNodeData(dataBag);
+
+ return;
+ }
+
Serializable data = getDiscoveryData(dataBag.joiningNodeId());
if (data != null)
@@ -349,6 +395,15 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
/** {@inheritDoc} */
@Override public void collectGridNodeData(DiscoveryDataBag dataBag) {
+ if (ctx.isDaemon())
+ return;
+
+ if (discoProtoVer == 2) {
+ routinesInfo.collectGridNodeData(dataBag);
+
+ return;
+ }
+
Serializable data = getDiscoveryData(dataBag.joiningNodeId());
if (data != null)
@@ -393,6 +448,7 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
return data;
}
+
return null;
}
@@ -430,22 +486,118 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
@Override public void onJoiningNodeDataReceived(JoiningNodeDiscoveryData data) {
if (log.isDebugEnabled()) {
log.debug("onJoiningNodeDataReceived [joining=" + data.joiningNodeId() +
- ", loc=" + ctx.localNodeId() +
- ", data=" + data.joiningNodeData() +
- ']');
+ ", loc=" + ctx.localNodeId() +
+ ", data=" + data.joiningNodeData() +
+ ']');
}
- if (data.hasJoiningNodeData())
- onDiscoDataReceived((DiscoveryData) data.joiningNodeData());
+ if (discoProtoVer == 2) {
+ if (data.hasJoiningNodeData()) {
+ ContinuousRoutinesJoiningNodeDiscoveryData nodeData = (ContinuousRoutinesJoiningNodeDiscoveryData)
+ data.joiningNodeData();
+
+ for (ContinuousRoutineInfo routineInfo : nodeData.startedRoutines) {
+ routinesInfo.addRoutineInfo(routineInfo);
+
+ startDiscoveryDataRoutine(routineInfo);
+ }
+ }
+ }
+ else {
+ if (data.hasJoiningNodeData())
+ onDiscoDataReceived((DiscoveryData) data.joiningNodeData());
+ }
}
/** {@inheritDoc} */
@Override public void onGridDataReceived(GridDiscoveryData data) {
- Map<UUID, Serializable> nodeSpecData = data.nodeSpecificData();
+ if (discoProtoVer == 2) {
+ if (ctx.isDaemon())
+ return;
+
+ if (data.commonData() != null) {
+ ContinuousRoutinesCommonDiscoveryData commonData =
+ (ContinuousRoutinesCommonDiscoveryData)data.commonData();
+
+ for (ContinuousRoutineInfo routineInfo : commonData.startedRoutines) {
+ if (routinesInfo.routineExists(routineInfo.routineId))
+ continue;
+
+ routinesInfo.addRoutineInfo(routineInfo);
+
+ startDiscoveryDataRoutine(routineInfo);
+ }
+ }
+ }
+ else {
+ Map<UUID, Serializable> nodeSpecData = data.nodeSpecificData();
+
+ if (nodeSpecData != null) {
+ for (Map.Entry<UUID, Serializable> e : nodeSpecData.entrySet())
+ onDiscoDataReceived((DiscoveryData) e.getValue());
+ }
+ }
+ }
+
+ /**
+ * @param routineInfo Routine info.
+ */
+ private void startDiscoveryDataRoutine(ContinuousRoutineInfo routineInfo) {
+ IgnitePredicate<ClusterNode> nodeFilter = null;
+
+ try {
+ if (routineInfo.nodeFilter != null) {
+ nodeFilter = U.unmarshal(marsh, routineInfo.nodeFilter, U.resolveClassLoader(ctx.config()));
+
+ ctx.resource().injectGeneric(nodeFilter);
+ }
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to unmarshal continuous routine filter, ignore routine [" +
+ "routineId=" + routineInfo.routineId +
+ ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
+
+ return;
+ }
+
+ if (nodeFilter == null || nodeFilter.apply(ctx.discovery().localNode())) {
+ GridContinuousHandler hnd;
- if (nodeSpecData != null) {
- for (Map.Entry<UUID, Serializable> e : nodeSpecData.entrySet())
- onDiscoDataReceived((DiscoveryData) e.getValue());
+ try {
+ hnd = U.unmarshal(marsh, routineInfo.hnd, U.resolveClassLoader(ctx.config()));
+
+ if (ctx.config().isPeerClassLoadingEnabled())
+ hnd.p2pUnmarshal(routineInfo.srcNodeId, ctx);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to unmarshal continuous routine handler, ignore routine [" +
+ "routineId=" + routineInfo.routineId +
+ ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
+
+ return;
+ }
+
+ try {
+ registerHandler(routineInfo.srcNodeId,
+ routineInfo.routineId,
+ hnd,
+ routineInfo.bufSize,
+ routineInfo.interval,
+ routineInfo.autoUnsubscribe,
+ false);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to register continuous routine handler, ignore routine [" +
+ "routineId=" + routineInfo.routineId +
+ ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
+ }
+ }
+ else {
+ if (log.isDebugEnabled()) {
+ log.debug("Do not register continuous routine, rejected by node filter [" +
+ "routineId=" + routineInfo.routineId +
+ ", srcNodeId=" + routineInfo.srcNodeId + ']');
+ }
}
}
@@ -564,13 +716,14 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
* @param rmtFilter Remote filter.
* @param prjPred Projection predicate.
* @return Routine ID.
+ * @throws IgniteCheckedException If failed.
*/
@SuppressWarnings("unchecked")
public UUID registerStaticRoutine(
String cacheName,
CacheEntryUpdatedListener<?, ?> locLsnr,
CacheEntryEventSerializableFilter rmtFilter,
- @Nullable IgnitePredicate<ClusterNode> prjPred) {
+ @Nullable IgnitePredicate<ClusterNode> prjPred) throws IgniteCheckedException {
String topicPrefix = "CONTINUOUS_QUERY_STATIC" + "_" + cacheName;
CacheContinuousQueryHandler hnd = new CacheContinuousQueryHandler(
@@ -589,6 +742,17 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
LocalRoutineInfo routineInfo = new LocalRoutineInfo(prjPred, hnd, 1, 0, true);
+ if (discoProtoVer == 2) {
+ routinesInfo.addRoutineInfo(createRoutineInfo(
+ ctx.localNodeId(),
+ routineId,
+ hnd,
+ prjPred,
+ routineInfo.bufSize,
+ routineInfo.interval,
+ routineInfo.autoUnsubscribe));
+ }
+
locInfos.put(routineId, routineInfo);
registerMessageListener(hnd);
@@ -597,6 +761,40 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
}
/**
+ * @param srcNodeId Source node ID.
+ * @param routineId Routine ID.
+ * @param hnd Handler.
+ * @param nodeFilter Node filter.
+ * @param bufSize Handler buffer size.
+ * @param interval Time interval.
+ * @param autoUnsubscribe Auto unsubscribe flag.
+ * @return Routine info instance.
+ * @throws IgniteCheckedException If failed.
+ */
+ private ContinuousRoutineInfo createRoutineInfo(
+ UUID srcNodeId,
+ UUID routineId,
+ GridContinuousHandler hnd,
+ @Nullable IgnitePredicate<ClusterNode> nodeFilter,
+ int bufSize,
+ long interval,
+ boolean autoUnsubscribe)
+ throws IgniteCheckedException {
+ byte[] hndBytes = marsh.marshal(hnd);
+
+ byte[] filterBytes = nodeFilter != null ? marsh.marshal(nodeFilter) : null;
+
+ return new ContinuousRoutineInfo(
+ srcNodeId,
+ routineId,
+ hndBytes,
+ filterBytes,
+ bufSize,
+ interval,
+ autoUnsubscribe);
+ }
+
+ /**
* @param hnd Handler.
* @param bufSize Buffer size.
* @param interval Time interval.
@@ -638,30 +836,10 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
// Whether local node is included in routine.
boolean locIncluded = prjPred == null || prjPred.apply(ctx.discovery().localNode());
- StartRequestData reqData = new StartRequestData(prjPred, hnd.clone(), bufSize, interval, autoUnsubscribe);
+ AbstractContinuousMessage msg;
try {
- if (ctx.config().isPeerClassLoadingEnabled()) {
- // Handle peer deployment for projection predicate.
- if (prjPred != null && !U.isGrid(prjPred.getClass())) {
- Class cls = U.detectClass(prjPred);
-
- String clsName = cls.getName();
-
- GridDeployment dep = ctx.deploy().deploy(cls, U.detectClassLoader(cls));
-
- if (dep == null)
- throw new IgniteDeploymentCheckedException("Failed to deploy projection predicate: " + prjPred);
-
- reqData.className(clsName);
- reqData.deploymentInfo(new GridDeploymentInfoBean(dep));
-
- reqData.p2pMarshal(marsh);
- }
-
- // Handle peer deployment for other handler-specific objects.
- reqData.handler().p2pMarshal(ctx);
- }
+ msg = createStartMessage(routineId, hnd, bufSize, interval, autoUnsubscribe, prjPred);
}
catch (IgniteCheckedException e) {
return new GridFinishedFuture<>(e);
@@ -674,20 +852,26 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
return new GridFinishedFuture<>(new NodeStoppingException("Failed to start continuous query (node is stopping)"));
try {
- StartFuture fut = new StartFuture(ctx, routineId);
+ StartFuture fut = new StartFuture(routineId);
startFuts.put(routineId, fut);
try {
- if (locIncluded || hnd.isQuery())
- registerHandler(ctx.localNodeId(), routineId, hnd, bufSize, interval, autoUnsubscribe, true);
+ if (locIncluded || hnd.isQuery()) {
+ registerHandler(ctx.localNodeId(),
+ routineId,
+ hnd,
+ bufSize,
+ interval,
+ autoUnsubscribe,
+ true);
+ }
- ctx.discovery().sendCustomEvent(new StartRoutineDiscoveryMessage(routineId, reqData,
- reqData.handler().keepBinary()));
- }
- catch (IgniteCheckedException e) {
- startFuts.remove(routineId);
- locInfos.remove(routineId);
+ ctx.discovery().sendCustomEvent(msg);
+ }
+ catch (IgniteCheckedException e) {
+ startFuts.remove(routineId);
+ locInfos.remove(routineId);
unregisterHandler(routineId, hnd, true);
@@ -707,6 +891,92 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
}
/**
+ * @param routineId Routine ID.
+ * @param hnd Handler.
+ * @param bufSize Buffer size.
+ * @param interval Interval.
+ * @param autoUnsubscribe Auto unsubscribe flag.
+ * @param nodeFilter Node filter.
+ * @return Routine start message.
+ * @throws IgniteCheckedException If failed.
+ */
+ private AbstractContinuousMessage createStartMessage(UUID routineId,
+ GridContinuousHandler hnd,
+ int bufSize,
+ long interval,
+ boolean autoUnsubscribe,
+ @Nullable IgnitePredicate<ClusterNode> nodeFilter)
+ throws IgniteCheckedException
+ {
+ hnd = hnd.clone();
+
+ String clsName = null;
+ GridDeploymentInfoBean dep = null;
+
+ if (ctx.config().isPeerClassLoadingEnabled()) {
+ // Handle peer deployment for projection predicate.
+ if (nodeFilter != null && !U.isGrid(nodeFilter.getClass())) {
+ Class cls = U.detectClass(nodeFilter);
+
+ clsName = cls.getName();
+
+ GridDeployment dep0 = ctx.deploy().deploy(cls, U.detectClassLoader(cls));
+
+ if (dep0 == null)
+ throw new IgniteDeploymentCheckedException("Failed to deploy projection predicate: " + nodeFilter);
+
+ dep = new GridDeploymentInfoBean(dep0);
+ }
+
+ // Handle peer deployment for other handler-specific objects.
+ hnd.p2pMarshal(ctx);
+ }
+
+ if (discoProtoVer == 1) {
+ StartRequestData reqData = new StartRequestData(
+ nodeFilter,
+ hnd,
+ bufSize,
+ interval,
+ autoUnsubscribe);
+
+ if (clsName != null) {
+ reqData.className(clsName);
+ reqData.deploymentInfo(dep);
+
+ reqData.p2pMarshal(marsh);
+ }
+
+ return new StartRoutineDiscoveryMessage(
+ routineId,
+ reqData,
+ reqData.handler().keepBinary());
+ }
+ else {
+ assert discoProtoVer == 2 : discoProtoVer;
+
+ byte[] nodeFilterBytes = nodeFilter != null ? U.marshal(marsh, nodeFilter) : null;
+ byte[] hndBytes = U.marshal(marsh, hnd);
+
+ StartRequestDataV2 reqData = new StartRequestDataV2(nodeFilterBytes,
+ hndBytes,
+ bufSize,
+ interval,
+ autoUnsubscribe);
+
+ if (clsName != null) {
+ reqData.className(clsName);
+ reqData.deploymentInfo(dep);
+ }
+
+ return new StartRoutineDiscoveryMessageV2(
+ routineId,
+ reqData,
+ hnd.keepBinary());
+ }
+ }
+
+ /**
* @param hnd Handler.
*/
private void registerMessageListener(GridContinuousHandler hnd) {
@@ -760,29 +1030,38 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
doStop = true;
}
- if (doStop) {
- // Unregister routine locally.
- LocalRoutineInfo routine = locInfos.remove(routineId);
-
- // Finish if routine is not found (wrong ID is provided).
- if (routine == null) {
- stopFuts.remove(routineId);
+ if (doStop) {
+ boolean stop = false;
- fut.onDone();
+ // Unregister routine locally.
+ LocalRoutineInfo routine = locInfos.remove(routineId);
- return fut;
- }
+ if (routine != null) {
+ stop = true;
// Unregister handler locally.
unregisterHandler(routineId, routine.hnd, true);
+ }
- try {
- ctx.discovery().sendCustomEvent(new StopRoutineDiscoveryMessage(routineId));
- }
- catch (IgniteCheckedException e) {
- fut.onDone(e);
+ if (!stop && discoProtoVer == 2)
+ stop = routinesInfo.routineExists(routineId);
+
+ // Finish if routine is not found (wrong ID is provided).
+ if (!stop) {
+ stopFuts.remove(routineId);
+
+ fut.onDone();
+
+ return fut;
}
+ try {
+ ctx.discovery().sendCustomEvent(new StopRoutineDiscoveryMessage(routineId));
+ }
+ catch (IgniteCheckedException e) {
+ fut.onDone(e);
+ }
+
if (ctx.isStopping())
fut.onDone();
}
@@ -924,6 +1203,9 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
clientInfos.clear();
+ if (discoProtoVer == 2)
+ routinesInfo.onClientDisconnected(locInfos.keySet());
+
if (log.isDebugEnabled()) {
log.debug("after onDisconnected [rmtInfos=" + rmtInfos +
", locInfos=" + locInfos +
@@ -996,35 +1278,11 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
StartFuture fut = startFuts.remove(msg.routineId());
if (fut != null) {
- if (msg.errs().isEmpty()) {
- LocalRoutineInfo routine = locInfos.get(msg.routineId());
-
- // Update partition counters.
- if (routine != null && routine.handler().isQuery()) {
- Map<UUID, Map<Integer, T2<Long, Long>>> cntrsPerNode = msg.updateCountersPerNode();
- Map<Integer, T2<Long, Long>> cntrs = msg.updateCounters();
-
- GridCacheAdapter<Object, Object> interCache =
- ctx.cache().internalCache(routine.handler().cacheName());
-
- GridCacheContext cctx = interCache != null ? interCache.context() : null;
-
- if (cctx != null && cntrsPerNode != null && !cctx.isLocal() && cctx.affinityNode())
- cntrsPerNode.put(ctx.localNodeId(),
- toCountersMap(cctx.topology().localUpdateCounters(false)));
-
- routine.handler().updateCounters(topVer, cntrsPerNode, cntrs);
- }
-
- fut.onRemoteRegistered();
- }
- else {
- IgniteCheckedException firstEx = F.first(msg.errs().values());
-
- fut.onDone(firstEx);
-
- stopRoutine(msg.routineId());
- }
+ fut.onAllRemoteRegistered(
+ topVer,
+ msg.errs(),
+ msg.updateCountersPerNode(),
+ msg.updateCounters());
}
}
@@ -1138,6 +1396,199 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
}
/**
+ * @param sndId Sender node ID.
+ * @param msg Message.
+ */
+ private void processRoutineStartResultMessage(UUID sndId, ContinuousRoutineStartResultMessage msg) {
+ StartFuture fut = startFuts.get(msg.routineId());
+
+ if (fut != null)
+ fut.onResult(sndId, msg);
+ }
+
+ /**
+ * @param topVer Current topology version.
+ * @param snd Sender.
+ * @param msg Start request.
+ */
+ private void processStartRequestV2(final AffinityTopologyVersion topVer,
+ final ClusterNode snd,
+ final StartRoutineDiscoveryMessageV2 msg) {
+ StartRequestDataV2 reqData = msg.startRequestData();
+
+ ContinuousRoutineInfo routineInfo = new ContinuousRoutineInfo(snd.id(),
+ msg.routineId(),
+ reqData.handlerBytes(),
+ reqData.nodeFilterBytes(),
+ reqData.bufferSize(),
+ reqData.interval(),
+ reqData.autoUnsubscribe());
+
+ routinesInfo.addRoutineInfo(routineInfo);
+
+ final DiscoCache discoCache = ctx.discovery().discoCache(topVer);
+
+ // Should not use marshaller and send messages from discovery thread.
+ ctx.getSystemExecutorService().execute(new Runnable() {
+ @Override public void run() {
+ if (snd.id().equals(ctx.localNodeId())) {
+ StartFuture fut = startFuts.get(msg.routineId());
+
+ if (fut != null)
+ fut.initRemoteNodes(discoCache);
+
+ return;
+ }
+
+ StartRequestDataV2 reqData = msg.startRequestData();
+
+ Exception err = null;
+
+ IgnitePredicate<ClusterNode> nodeFilter = null;
+
+ byte[] cntrs = null;
+
+ if (reqData.nodeFilterBytes() != null) {
+ try {
+ if (ctx.config().isPeerClassLoadingEnabled() && reqData.className() != null) {
+ String clsName = reqData.className();
+ GridDeploymentInfo depInfo = reqData.deploymentInfo();
+
+ GridDeployment dep = ctx.deploy().getGlobalDeployment(depInfo.deployMode(),
+ clsName,
+ clsName,
+ depInfo.userVersion(),
+ snd.id(),
+ depInfo.classLoaderId(),
+ depInfo.participants(),
+ null);
+
+ if (dep == null) {
+ throw new IgniteDeploymentCheckedException("Failed to obtain deployment " +
+ "for class: " + clsName);
+ }
+
+ nodeFilter = U.unmarshal(marsh,
+ reqData.nodeFilterBytes(),
+ U.resolveClassLoader(dep.classLoader(), ctx.config()));
+ }
+ else {
+ nodeFilter = U.unmarshal(marsh,
+ reqData.nodeFilterBytes(),
+ U.resolveClassLoader(ctx.config()));
+ }
+
+ if (nodeFilter != null)
+ ctx.resource().injectGeneric(nodeFilter);
+ }
+ catch (Exception e) {
+ err = e;
+
+ U.error(log, "Failed to unmarshal continuous routine filter [" +
+ "routineId=" + msg.routineId +
+ ", srcNodeId=" + snd.id() + ']', e);
+ }
+ }
+
+ boolean register = err == null &&
+ (nodeFilter == null || nodeFilter.apply(ctx.discovery().localNode()));
+
+ if (register) {
+ try {
+ GridContinuousHandler hnd = U.unmarshal(marsh,
+ reqData.handlerBytes(),
+ U.resolveClassLoader(ctx.config()));
+
+ if (ctx.config().isPeerClassLoadingEnabled())
+ hnd.p2pUnmarshal(snd.id(), ctx);
+
+ if (msg.keepBinary()) {
+ assert hnd instanceof CacheContinuousQueryHandler : hnd;
+
+ ((CacheContinuousQueryHandler)hnd).keepBinary(true);
+ }
+
+ GridContinuousHandler hnd0 = hnd instanceof GridMessageListenHandler ?
+ new GridMessageListenHandler((GridMessageListenHandler)hnd) :
+ hnd;
+
+ registerHandler(snd.id(),
+ msg.routineId,
+ hnd0,
+ reqData.bufferSize(),
+ reqData.interval(),
+ reqData.autoUnsubscribe(),
+ false);
+
+ if (hnd0.isQuery()) {
+ GridCacheProcessor proc = ctx.cache();
+
+ if (proc != null) {
+ GridCacheAdapter cache = ctx.cache().internalCache(hnd0.cacheName());
+
+ if (cache != null && !cache.isLocal() && cache.context().userCache()) {
+ CachePartitionPartialCountersMap cntrsMap =
+ cache.context().topology().localUpdateCounters(false);
+
+ cntrs = U.marshal(marsh, cntrsMap);
+ }
+ }
+ }
+ }
+ catch (Exception e) {
+ err = e;
+
+ U.error(log, "Failed to register continuous routine handler [" +
+ "routineId=" + msg.routineId +
+ ", srcNodeId=" + snd.id() + ']', e);
+ }
+ }
+
+ sendMessageStartResult(snd, msg.routineId(), cntrs, err);
+ }
+ });
+ }
+
+ /**
+ * @param node Target node.
+ * @param routineId Routine ID.
+ * @param cntrsMapBytes Marshalled {@link CachePartitionPartialCountersMap}.
+ * @param err Start error if any.
+ */
+ private void sendMessageStartResult(final ClusterNode node,
+ final UUID routineId,
+ byte[] cntrsMapBytes,
+ final @Nullable Exception err)
+ {
+ byte[] errBytes = null;
+
+ if (err != null) {
+ try {
+ errBytes = U.marshal(marsh, err);
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to marshal routine start error: " + e, e);
+ }
+ }
+
+ ContinuousRoutineStartResultMessage msg = new ContinuousRoutineStartResultMessage(routineId,
+ cntrsMapBytes,
+ errBytes,
+ err != null);
+
+ try {
+ ctx.io().sendToGridTopic(node, TOPIC_CONTINUOUS, msg, SYSTEM_POOL, null);
+ }
+ catch (ClusterTopologyCheckedException e) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to send routine start result, node failed: " + e);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to send routine start result: " + e, e);
+ }
+ }
+
+ /**
* @param msg Message.
*/
private void processMessageAck(GridContinuousMessage msg) {
@@ -1455,6 +1906,13 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
UUID nodeId = ((DiscoveryEvent)evt).eventNode().id();
+ if (discoProtoVer == 2) {
+ routinesInfo.onNodeFail(nodeId);
+
+ for (StartFuture fut : startFuts.values())
+ fut.onNodeFail(nodeId);
+ }
+
clientInfos.remove(nodeId);
// Unregister handlers created by left node.
@@ -1894,10 +2352,7 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
/**
* Future for start routine.
*/
- private static class StartFuture extends GridFutureAdapter<UUID> {
- /** */
- private GridKernalContext ctx;
-
+ private class StartFuture extends GridFutureAdapter<UUID> {
/** Consume ID. */
private UUID routineId;
@@ -1907,56 +2362,170 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
/** All remote listeners are registered. */
private volatile boolean rmt;
- /** Timeout object. */
- private volatile GridTimeoutObject timeoutObj;
+ /** */
+ private final DiscoveryMessageResultsCollector<ContinuousRoutineStartResultMessage, RoutineRegisterResults>
+ resCollect;
/**
- * @param ctx Kernal context.
* @param routineId Consume ID.
*/
- StartFuture(GridKernalContext ctx, UUID routineId) {
- this.ctx = ctx;
-
+ StartFuture(UUID routineId) {
this.routineId = routineId;
+
+ resCollect = new DiscoveryMessageResultsCollector<ContinuousRoutineStartResultMessage, RoutineRegisterResults>(ctx) {
+ @Override protected RoutineRegisterResults createResult(Map<UUID, NodeMessage<ContinuousRoutineStartResultMessage>> rcvd) {
+ Map<UUID, Exception> errs = null;
+ Map<UUID, Map<Integer, T2<Long, Long>>> cntrsPerNode = null;
+
+ for (Map.Entry<UUID, NodeMessage<ContinuousRoutineStartResultMessage>> entry : rcvd.entrySet()) {
+ ContinuousRoutineStartResultMessage msg = entry.getValue().message();
+
+ if (msg == null)
+ continue;
+
+ if (msg.error()) {
+ byte[] errBytes = msg.errorBytes();
+
+ Exception err = null;
+
+ if (errBytes != null) {
+ try {
+ err = U.unmarshal(marsh, errBytes, U.resolveClassLoader(ctx.config()));
+ }
+ catch (Exception e) {
+ U.warn(log, "Failed to unmarhal continuous routine start error: " + e);
+ }
+ }
+
+ if (err == null) {
+ err = new IgniteCheckedException("Failed to start continuous " +
+ "routine on node: " + entry.getKey());
+ }
+
+ if (errs == null)
+ errs = new HashMap<>();
+
+ errs.put(entry.getKey(), err);
+ }
+ else {
+ byte[] cntrsMapBytes = msg.countersMapBytes();
+
+ if (cntrsMapBytes != null) {
+ try {
+ CachePartitionPartialCountersMap cntrsMap = U.unmarshal(
+ marsh,
+ cntrsMapBytes,
+ U.resolveClassLoader(ctx.config()));
+
+ if (cntrsPerNode == null)
+ cntrsPerNode = new HashMap<>();
+
+ cntrsPerNode.put(entry.getKey(), CachePartitionPartialCountersMap.toCountersMap(cntrsMap));
+ }
+ catch (Exception e) {
+ U.warn(log, "Failed to unmarhal continuous query update counters: " + e);
+ }
+ }
+ }
+ }
+
+ return new RoutineRegisterResults(discoCache.version(), errs, cntrsPerNode);
+ }
+
+ @Override protected void onResultsCollected(RoutineRegisterResults res0) {
+ onAllRemoteRegistered(res0.topVer, res0.errs, res0.cntrsPerNode, null);
+ }
+
+ @Override protected boolean waitForNode(DiscoCache discoCache, ClusterNode node) {
+ return !ctx.localNodeId().equals(node.id());
+ }
+ };
}
/**
- * Called when local listener is registered.
+ * @param topVer Topology version.
+ * @param errs Errors.
+ * @param cntrsPerNode Update counters.
+ * @param cntrs Update counters.
*/
- public void onLocalRegistered() {
- loc = true;
+ private void onAllRemoteRegistered(
+ AffinityTopologyVersion topVer,
+ @Nullable Map<UUID, ? extends Exception> errs,
+ Map<UUID, Map<Integer, T2<Long, Long>>> cntrsPerNode,
+ Map<Integer, T2<Long, Long>> cntrs) {
+ try {
+ if (errs == null || errs.isEmpty()) {
+ LocalRoutineInfo routine = locInfos.get(routineId);
- if (rmt && !isDone())
- onDone(routineId);
+ // Update partition counters.
+ if (routine != null && routine.handler().isQuery()) {
+ GridCacheAdapter<Object, Object> interCache =
+ ctx.cache().internalCache(routine.handler().cacheName());
+
+ GridCacheContext cctx = interCache != null ? interCache.context() : null;
+
+ if (cctx != null && cntrsPerNode != null && !cctx.isLocal() && cctx.affinityNode())
+ cntrsPerNode.put(ctx.localNodeId(),
+ toCountersMap(cctx.topology().localUpdateCounters(false)));
+
+ routine.handler().updateCounters(topVer, cntrsPerNode, cntrs);
+ }
+
+ onRemoteRegistered();
+ }
+ else {
+ Exception firstEx = F.first(errs.values());
+
+ onDone(firstEx);
+
+ stopRoutine(routineId);
+ }
+ }
+ finally {
+ startFuts.remove(routineId, this);
+ }
}
/**
- * Called when all remote listeners are registered.
+ * @param discoCache Discovery state.
*/
- public void onRemoteRegistered() {
- rmt = true;
+ void initRemoteNodes(DiscoCache discoCache) {
+ resCollect.init(discoCache);
+ }
- if (loc && !isDone())
- onDone(routineId);
+ /**
+ * @param nodeId Node ID.
+ * @param msg Message.
+ */
+ void onResult(UUID nodeId, ContinuousRoutineStartResultMessage msg) {
+ resCollect.onMessage(nodeId, msg);
}
/**
- * @param timeoutObj Timeout object.
+ * @param nodeId Failed node ID.
*/
- public void addTimeoutObject(GridTimeoutObject timeoutObj) {
- assert timeoutObj != null;
+ void onNodeFail(UUID nodeId) {
+ resCollect.onNodeFail(nodeId);
+ }
- this.timeoutObj = timeoutObj;
+ /**
+ * Called when local listener is registered.
+ */
+ void onLocalRegistered() {
+ loc = true;
- ctx.timeout().addTimeoutObject(timeoutObj);
+ if (rmt && !isDone())
+ onDone(routineId);
}
- /** {@inheritDoc} */
- @Override public boolean onDone(@Nullable UUID res, @Nullable Throwable err) {
- if (timeoutObj != null)
- ctx.timeout().removeTimeoutObject(timeoutObj);
+ /**
+ * Called when all remote listeners are registered.
+ */
+ void onRemoteRegistered() {
+ rmt = true;
- return super.onDone(res, err);
+ if (loc && !isDone())
+ onDone(routineId);
}
/** {@inheritDoc} */
@@ -1966,6 +2535,33 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
}
/**
+ *
+ */
+ private static class RoutineRegisterResults {
+ /** */
+ private final AffinityTopologyVersion topVer;
+
+ /** */
+ private final Map<UUID, ? extends Exception> errs;
+
+ /** */
+ private final Map<UUID, Map<Integer, T2<Long, Long>>> cntrsPerNode;
+
+ /**
+ * @param topVer Topology version.
+ * @param errs Errors.
+ * @param cntrsPerNode Update counters.
+ */
+ RoutineRegisterResults(AffinityTopologyVersion topVer,
+ Map<UUID, ? extends Exception> errs,
+ Map<UUID, Map<Integer, T2<Long, Long>>> cntrsPerNode) {
+ this.topVer = topVer;
+ this.errs = errs;
+ this.cntrsPerNode = cntrsPerNode;
+ }
+ }
+
+ /**
* Future for stop routine.
*/
private static class StopFuture extends GridFutureAdapter<Object> {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRequestDataV2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRequestDataV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRequestDataV2.java
new file mode 100644
index 0000000..c001616
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRequestDataV2.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Start request data.
+ */
+class StartRequestDataV2 implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Serialized node filter. */
+ private byte[] nodeFilterBytes;
+
+ /** Deployment class name. */
+ private String clsName;
+
+ /** Deployment info. */
+ private GridDeploymentInfo depInfo;
+
+ /** Serialized handler. */
+ private byte[] hndBytes;
+
+ /** Buffer size. */
+ private int bufSize;
+
+ /** Time interval. */
+ private long interval;
+
+ /** Automatic unsubscribe flag. */
+ private boolean autoUnsubscribe;
+
+ /**
+ * @param nodeFilterBytes Serialized node filter.
+ * @param hndBytes Serialized handler.
+ * @param bufSize Buffer size.
+ * @param interval Time interval.
+ * @param autoUnsubscribe Automatic unsubscribe flag.
+ */
+ StartRequestDataV2(
+ byte[] nodeFilterBytes,
+ byte[] hndBytes,
+ int bufSize,
+ long interval,
+ boolean autoUnsubscribe) {
+ assert hndBytes != null;
+ assert bufSize > 0;
+ assert interval >= 0;
+
+ this.nodeFilterBytes = nodeFilterBytes;
+ this.hndBytes = hndBytes;
+ this.bufSize = bufSize;
+ this.interval = interval;
+ this.autoUnsubscribe = autoUnsubscribe;
+ }
+
+ /**
+ * @return Serialized node filter.
+ */
+ public byte[] nodeFilterBytes() {
+ return nodeFilterBytes;
+ }
+
+ /**
+ * @return Deployment class name.
+ */
+ public String className() {
+ return clsName;
+ }
+
+ /**
+ * @param clsName New deployment class name.
+ */
+ public void className(String clsName) {
+ this.clsName = clsName;
+ }
+
+ /**
+ * @return Deployment info.
+ */
+ public GridDeploymentInfo deploymentInfo() {
+ return depInfo;
+ }
+
+ /**
+ * @param depInfo New deployment info.
+ */
+ public void deploymentInfo(GridDeploymentInfo depInfo) {
+ this.depInfo = depInfo;
+ }
+
+ /**
+ * @return Handler.
+ */
+ public byte[] handlerBytes() {
+ return hndBytes;
+ }
+
+ /**
+ * @return Buffer size.
+ */
+ public int bufferSize() {
+ return bufSize;
+ }
+
+ /**
+ * @param bufSize New buffer size.
+ */
+ public void bufferSize(int bufSize) {
+ this.bufSize = bufSize;
+ }
+
+ /**
+ * @return Time interval.
+ */
+ public long interval() {
+ return interval;
+ }
+
+ /**
+ * @param interval New time interval.
+ */
+ public void interval(long interval) {
+ this.interval = interval;
+ }
+
+ /**
+ * @return Automatic unsubscribe flag.
+ */
+ public boolean autoUnsubscribe() {
+ return autoUnsubscribe;
+ }
+
+ /**
+ * @param autoUnsubscribe New automatic unsubscribe flag.
+ */
+ public void autoUnsubscribe(boolean autoUnsubscribe) {
+ this.autoUnsubscribe = autoUnsubscribe;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(StartRequestDataV2.class, this);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRoutineDiscoveryMessageV2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRoutineDiscoveryMessageV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRoutineDiscoveryMessageV2.java
new file mode 100644
index 0000000..275765d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRoutineDiscoveryMessageV2.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.util.UUID;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+public class StartRoutineDiscoveryMessageV2 extends AbstractContinuousMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private static final int KEEP_BINARY_FLAG = 0x01;
+
+ /** */
+ private final StartRequestDataV2 startReqData;
+
+ /** Flags. */
+ private int flags;
+
+ /**
+ * @param routineId Routine id.
+ * @param startReqData Start request data.
+ * @param keepBinary Keep binary flag.
+ */
+ StartRoutineDiscoveryMessageV2(UUID routineId, StartRequestDataV2 startReqData, boolean keepBinary) {
+ super(routineId);
+
+ this.startReqData = startReqData;
+
+ if (keepBinary)
+ flags |= KEEP_BINARY_FLAG;
+ }
+
+ /**
+ * @return Start request data.
+ */
+ public StartRequestDataV2 startRequestData() {
+ return startReqData;
+ }
+
+ /**
+ * @return {@code True} if keep binary flag was set on continuous handler.
+ */
+ public boolean keepBinary() {
+ return (flags & KEEP_BINARY_FLAG) != 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public DiscoveryCustomMessage ackMessage() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(StartRoutineDiscoveryMessageV2.class, this, "routineId", routineId());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StopRoutineAckDiscoveryMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StopRoutineAckDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StopRoutineAckDiscoveryMessage.java
index 79d8b29..dfba0e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StopRoutineAckDiscoveryMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StopRoutineAckDiscoveryMessage.java
@@ -42,6 +42,11 @@ public class StopRoutineAckDiscoveryMessage extends AbstractContinuousMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return S.toString(StopRoutineAckDiscoveryMessage.class, this, "routineId", routineId());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index 4a893f4..8cad342 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -47,6 +47,7 @@ import javax.cache.expiry.ExpiryPolicy;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteDataStreamerTimeoutException;
import org.apache.ignite.IgniteException;
@@ -100,6 +101,7 @@ import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.GPC;
@@ -1059,6 +1061,9 @@ public class DataStreamerImpl<K, V> implements IgniteDataStreamer<K, V>, Delayed
return;
while (true) {
+ if (disconnectErr != null)
+ throw disconnectErr;
+
Queue<IgniteInternalFuture<?>> q = null;
for (Buffer buf : bufMappings.values()) {
@@ -1826,15 +1831,19 @@ public class DataStreamerImpl<K, V> implements IgniteDataStreamer<K, V>, Delayed
catch (IgniteCheckedException e) {
GridFutureAdapter<Object> fut0 = ((GridFutureAdapter<Object>)fut);
- try {
- if (ctx.discovery().alive(node) && ctx.discovery().pingNode(node.id()))
- fut0.onDone(e);
- else
- fut0.onDone(new ClusterTopologyCheckedException("Failed to send request (node has left): "
- + node.id()));
- }
- catch (IgniteClientDisconnectedCheckedException e0) {
- fut0.onDone(e0);
+ if (X.hasCause(e, IgniteClientDisconnectedCheckedException.class, IgniteClientDisconnectedException.class))
+ fut0.onDone(e);
+ else {
+ try {
+ if (ctx.discovery().alive(node) && ctx.discovery().pingNode(node.id()))
+ fut0.onDone(e);
+ else
+ fut0.onDone(new ClusterTopologyCheckedException("Failed to send request (node has left): "
+ + node.id()));
+ }
+ catch (IgniteClientDisconnectedCheckedException e0) {
+ fut0.onDone(e0);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingAcceptedMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingAcceptedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingAcceptedMessage.java
index 7af0559..80e3f7d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingAcceptedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingAcceptedMessage.java
@@ -63,6 +63,11 @@ public class MappingAcceptedMessage implements DiscoveryCustomMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
AffinityTopologyVersion topVer, DiscoCache discoCache) {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingProposedMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingProposedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingProposedMessage.java
index b4e13fb..9358585 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingProposedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingProposedMessage.java
@@ -98,6 +98,11 @@ public class MappingProposedMessage implements DiscoveryCustomMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
AffinityTopologyVersion topVer, DiscoCache discoCache) {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaFinishDiscoveryMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaFinishDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaFinishDiscoveryMessage.java
index 2245b24..f802e09 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaFinishDiscoveryMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaFinishDiscoveryMessage.java
@@ -59,6 +59,11 @@ public class SchemaFinishDiscoveryMessage extends SchemaAbstractDiscoveryMessage
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public boolean exchange() {
return false;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaProposeDiscoveryMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaProposeDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaProposeDiscoveryMessage.java
index 0e1270b..62b6d6a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaProposeDiscoveryMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaProposeDiscoveryMessage.java
@@ -60,6 +60,11 @@ public class SchemaProposeDiscoveryMessage extends SchemaAbstractDiscoveryMessag
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public boolean exchange() {
return exchange;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index e0ec8d1..0fcde0e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -491,6 +491,17 @@ public class GridNioServer<T> {
/**
* @param ses Session.
+ */
+ public void closeFromWorkerThread(GridNioSession ses) {
+ assert ses instanceof GridSelectorNioSessionImpl : ses;
+
+ GridSelectorNioSessionImpl ses0 = (GridSelectorNioSessionImpl)ses;
+
+ ((AbstractNioClientWorker)ses0.worker()).close((GridSelectorNioSessionImpl)ses, null);
+ }
+
+ /**
+ * @param ses Session.
* @param msg Message.
* @param createFut {@code True} if future should be created.
* @param ackC Closure invoked when message ACK is received.
@@ -2170,7 +2181,12 @@ public class GridNioServer<T> {
dumpSelectorInfo(sb, keys);
for (SelectionKey key : keys) {
- GridSelectorNioSessionImpl ses = (GridSelectorNioSessionImpl)key.attachment();
+ GridNioKeyAttachment attach = (GridNioKeyAttachment)key.attachment();
+
+ if (!attach.hasSession())
+ continue;
+
+ GridSelectorNioSessionImpl ses = attach.session();
boolean sesInfo = p == null || p.apply(ses);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
index 1754cc8..e8c27d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
@@ -963,5 +963,15 @@ public abstract class IgniteSpiAdapter implements IgniteSpi {
@Override public Map<String, Object> nodeAttributes() {
return Collections.emptyMap();
}
+
+ /** {@inheritDoc} */
+ @Override public boolean communicationFailureResolveSupported() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resolveCommunicationFailure(ClusterNode node, Exception err) {
+ throw new UnsupportedOperationException();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
index 108c4d4..d4402f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
@@ -365,4 +365,15 @@ public interface IgniteSpiContext {
* @return Current node attributes.
*/
public Map<String, Object> nodeAttributes();
+
+ /**
+ * @return {@code True} if cluster supports communication error resolving.
+ */
+ public boolean communicationFailureResolveSupported();
+
+ /**
+ * @param node Problem node.
+ * @param err Error.
+ */
+ public void resolveCommunicationFailure(ClusterNode node, Exception err);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index e1addd8..4a0710e 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -32,6 +32,7 @@ import java.nio.channels.SocketChannel;
import java.nio.channels.spi.AbstractInterruptibleChannel;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.BitSet;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -66,12 +67,14 @@ import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.managers.eventstorage.HighPriorityListener;
import org.apache.ignite.internal.util.GridConcurrentFactory;
import org.apache.ignite.internal.util.GridSpinReadWriteLock;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.future.IgniteFutureImpl;
import org.apache.ignite.internal.util.ipc.IpcEndpoint;
import org.apache.ignite.internal.util.ipc.IpcToNioAdapter;
import org.apache.ignite.internal.util.ipc.shmem.IpcOutOfSystemResourcesException;
@@ -134,6 +137,9 @@ import org.apache.ignite.spi.IgniteSpiThread;
import org.apache.ignite.spi.IgniteSpiTimeoutObject;
import org.apache.ignite.spi.communication.CommunicationListener;
import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.internal.ConnectionKey;
+import org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture;
+import org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationNodeConnectionCheckFuture;
import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage;
import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage2;
import org.apache.ignite.spi.communication.tcp.messages.NodeIdMessage;
@@ -146,6 +152,7 @@ import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META;
+import static org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture.SES_FUT_META;
import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.ALREADY_CONNECTED;
import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.NEED_WAIT;
import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.NODE_STOPPING;
@@ -310,7 +317,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
private static final IgniteProductVersion VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT = IgniteProductVersion.fromString("2.1.4");
/** Connection index meta for session. */
- private static final int CONN_IDX_META = GridNioSessionMetaKey.nextUniqueKey();
+ public static final int CONN_IDX_META = GridNioSessionMetaKey.nextUniqueKey();
/** Message tracker meta for session. */
private static final int TRACKER_META = GridNioSessionMetaKey.nextUniqueKey();
@@ -408,6 +415,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
ConnectionKey connId = ses.meta(CONN_IDX_META);
if (connId != null) {
+ if (connId.dummy())
+ return;
+
UUID id = connId.nodeId();
GridCommunicationClient[] nodeClients = clients.get(id);
@@ -481,20 +491,22 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
if (rmtNode == null) {
DiscoverySpi discoverySpi = ignite().configuration().getDiscoverySpi();
- assert discoverySpi instanceof TcpDiscoverySpi;
-
- TcpDiscoverySpi tcpDiscoverySpi = (TcpDiscoverySpi) discoverySpi;
+ boolean unknownNode = true;
- ClusterNode node0 = tcpDiscoverySpi.getNode0(sndId);
+ if (discoverySpi instanceof TcpDiscoverySpi) {
+ TcpDiscoverySpi tcpDiscoverySpi = (TcpDiscoverySpi) discoverySpi;
- boolean unknownNode = true;
+ ClusterNode node0 = tcpDiscoverySpi.getNode0(sndId);
- if (node0 != null) {
- assert node0.isClient() : node0;
+ if (node0 != null) {
+ assert node0.isClient() : node0;
- if (node0.version().compareTo(VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT) >= 0)
- unknownNode = false;
+ if (node0.version().compareTo(VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT) >= 0)
+ unknownNode = false;
+ }
}
+ else if (discoverySpi instanceof IgniteDiscoverySpi)
+ unknownNode = !((IgniteDiscoverySpi) discoverySpi).knownNode(sndId);
if (unknownNode) {
U.warn(log, "Close incoming connection, unknown node [nodeId=" + sndId + ", ses=" + ses + ']');
@@ -709,9 +721,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
}
else {
- metricsLsnr.onMessageReceived(msg, connKey.nodeId());
-
if (msg instanceof RecoveryLastReceivedMessage) {
+ metricsLsnr.onMessageReceived(msg, connKey.nodeId());
+
GridNioRecoveryDescriptor recovery = ses.outRecoveryDescriptor();
if (recovery != null) {
@@ -724,9 +736,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
recovery.ackReceived(msg0.received());
-
- return;
}
+
+ return;
}
else {
GridNioRecoveryDescriptor recovery = ses.inRecoveryDescriptor();
@@ -746,8 +758,23 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
recovery.lastAcknowledged(rcvCnt);
}
}
+ else if (connKey.dummy()) {
+ assert msg instanceof NodeIdMessage : msg;
+
+ TcpCommunicationNodeConnectionCheckFuture fut = ses.meta(SES_FUT_META);
+
+ assert fut != null : msg;
+
+ fut.onConnected(U.bytesToUuid(((NodeIdMessage)msg).nodeIdBytes(), 0));
+
+ nioSrvr.closeFromWorkerThread(ses);
+
+ return;
+ }
}
+ metricsLsnr.onMessageReceived(msg, connKey.nodeId());
+
IgniteRunnable c;
if (msgQueueLimit > 0) {
@@ -2112,6 +2139,13 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
}
+ /**
+ * @return Bound TCP server port.
+ */
+ public int boundPort() {
+ return boundTcpPort;
+ }
+
/** {@inheritDoc} */
@Override public void spiStart(String igniteInstanceName) throws IgniteSpiException {
assert locHost != null;
@@ -2570,6 +2604,27 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
/**
+ * @param nodes Nodes to check connection with.
+ * @return Result future (each bit in result BitSet contains connection status to corresponding node).
+ */
+ public IgniteFuture<BitSet> checkConnection(List<ClusterNode> nodes) {
+ TcpCommunicationConnectionCheckFuture fut = new TcpCommunicationConnectionCheckFuture(
+ this,
+ log.getLogger(TcpCommunicationConnectionCheckFuture.class),
+ nioSrvr,
+ nodes);
+
+ long timeout = failureDetectionTimeoutEnabled() ? failureDetectionTimeout() : connTimeout;
+
+ if (log.isInfoEnabled())
+ log.info("Start check connection process [nodeCnt=" + nodes.size() + ", timeout=" + timeout + ']');
+
+ fut.init(timeout);
+
+ return new IgniteFutureImpl<>(fut);
+ }
+
+ /**
* Sends given message to destination node. Note that characteristics of the
* exchange such as durability, guaranteed delivery or error notification is
* dependant on SPI implementation.
@@ -3010,7 +3065,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
ConnectionKey id = ses.meta(CONN_IDX_META);
if (id != null) {
- ClusterNode node = getSpiContext().node(id.nodeId);
+ ClusterNode node = getSpiContext().node(id.nodeId());
if (node != null && node.isClient()) {
String msg = "Client node outbound message queue size exceeded slowClientQueueLimit, " +
@@ -3031,9 +3086,20 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
/**
* @param node Node.
* @return Node addresses.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Collection<InetSocketAddress> nodeAddresses(ClusterNode node) throws IgniteCheckedException {
+ return nodeAddresses(node, filterReachableAddresses);
+ }
+
+ /**
+ * @param node Node.
+ * @param filterReachableAddresses Filter addresses flag.
+ * @return Node addresses.
* @throws IgniteCheckedException If node does not have addresses.
*/
- private LinkedHashSet<InetSocketAddress> nodeAddresses(ClusterNode node) throws IgniteCheckedException {
+ public Collection<InetSocketAddress> nodeAddresses(ClusterNode node, boolean filterReachableAddresses)
+ throws IgniteCheckedException {
Collection<String> rmtAddrs0 = node.attribute(createSpiAttributeName(ATTR_ADDRS));
Collection<String> rmtHostNames0 = node.attribute(createSpiAttributeName(ATTR_HOST_NAMES));
Integer boundPort = node.attribute(createSpiAttributeName(ATTR_PORT));
@@ -3114,7 +3180,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
* @throws IgniteCheckedException If failed.
*/
protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
- LinkedHashSet<InetSocketAddress> addrs = nodeAddresses(node);
+ Collection<InetSocketAddress> addrs = nodeAddresses(node);
GridCommunicationClient client = null;
IgniteCheckedException errs = null;
@@ -3132,6 +3198,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
int lastWaitingTimeout = 1;
while (client == null) { // Reconnection on handshake timeout.
+ if (stopping)
+ throw new IgniteSpiException("Node is stopping.");
+
if (addr.getAddress().isLoopbackAddress() && addr.getPort() == boundTcpPort) {
if (log.isDebugEnabled())
log.debug("Skipping local address [addr=" + addr +
@@ -3372,8 +3441,18 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
"operating system firewall is disabled on local and remote hosts) " +
"[addrs=" + addrs + ']');
- if (enableForcibleNodeKill) {
- if (getSpiContext().node(node.id()) != null
+ boolean commErrResolve = false;
+
+ IgniteSpiContext ctx = getSpiContext();
+
+ if (connectionError(errs) && ctx.communicationFailureResolveSupported()) {
+ commErrResolve = true;
+
+ ctx.resolveCommunicationFailure(node, errs);
+ }
+
+ if (!commErrResolve && enableForcibleNodeKill) {
+ if (ctx.node(node.id()) != null
&& (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
connectionError(errs)) {
String msg = "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
@@ -3384,7 +3463,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
else
U.warn(log, msg);
- getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
+ ctx.failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
"rmtNode=" + node +
", errs=" + errs +
", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
@@ -4590,77 +4669,6 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
/**
*
*/
- private static class ConnectionKey {
- /** */
- private final UUID nodeId;
-
- /** */
- private final int idx;
-
- /** */
- private final long connCnt;
-
- /**
- * @param nodeId Node ID.
- * @param idx Connection index.
- * @param connCnt Connection counter (set only for incoming connections).
- */
- ConnectionKey(UUID nodeId, int idx, long connCnt) {
- this.nodeId = nodeId;
- this.idx = idx;
- this.connCnt = connCnt;
- }
-
- /**
- * @return Connection counter.
- */
- long connectCount() {
- return connCnt;
- }
-
- /**
- * @return Node ID.
- */
- UUID nodeId() {
- return nodeId;
- }
-
- /**
- * @return Connection index.
- */
- int connectionIndex() {
- return idx;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- ConnectionKey key = (ConnectionKey) o;
-
- return idx == key.idx && nodeId.equals(key.nodeId);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = nodeId.hashCode();
- res = 31 * res + idx;
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(ConnectionKey.class, this);
- }
- }
-
- /**
- *
- */
interface ConnectionPolicy {
/**
* @return Thread connection index.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionKey.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionKey.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionKey.java
new file mode 100644
index 0000000..0559df7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionKey.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.internal;
+
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Connection Key.
+ */
+public class ConnectionKey {
+ /** */
+ private final UUID nodeId;
+
+ /** */
+ private final int idx;
+
+ /** */
+ private final long connCnt;
+
+ /** */
+ private final boolean dummy;
+
+ /**
+ * Creates ConnectionKey with false value of dummy flag.
+ *
+ * @param nodeId Node ID. Should be not null.
+ * @param idx Connection index.
+ * @param connCnt Connection counter (set only for incoming connections).
+ */
+ public ConnectionKey(@NotNull UUID nodeId, int idx, long connCnt) {
+ this(nodeId, idx, connCnt, false);
+ }
+
+ /**
+ * @param nodeId Node ID. Should be not null.
+ * @param idx Connection index.
+ * @param connCnt Connection counter (set only for incoming connections).
+ * @param dummy Indicates that session with this ConnectionKey is temporary
+ * (for now dummy sessions are used only for Communication Failure Resolving process).
+ */
+ public ConnectionKey(@NotNull UUID nodeId, int idx, long connCnt, boolean dummy) {
+ this.nodeId = nodeId;
+ this.idx = idx;
+ this.connCnt = connCnt;
+ this.dummy = dummy;
+ }
+
+ /**
+ * @return Connection counter.
+ */
+ public long connectCount() {
+ return connCnt;
+ }
+
+ /**
+ * @return Node ID.
+ */
+ public UUID nodeId() {
+ return nodeId;
+ }
+
+ /**
+ * @return Connection index.
+ */
+ public int connectionIndex() {
+ return idx;
+ }
+
+ /**
+ * @return {@code True} if this ConnectionKey is dummy and serves temporary session.
+ */
+ public boolean dummy() {
+ return dummy;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ ConnectionKey key = (ConnectionKey) o;
+
+ return idx == key.idx && nodeId.equals(key.nodeId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = nodeId.hashCode();
+ res = 31 * res + idx;
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ConnectionKey.class, this);
+ }
+}
[12/54] [abbrv] ignite git commit: IGNITE-7222 Added ZooKeeper
discovery SPI
Posted by ag...@apache.org.
IGNITE-7222 Added ZooKeeper discovery SPI
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a64b941d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a64b941d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a64b941d
Branch: refs/heads/ignite-6083
Commit: a64b941df0a4d3bfc3a2dab32f85c371c1a509be
Parents: a0a187b
Author: Semyon Boikov <sb...@apache.org>
Authored: Tue Apr 10 11:37:39 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Apr 10 11:57:40 2018 +0300
----------------------------------------------------------------------
.../jdbc2/JdbcAbstractDmlStatementSelfTest.java | 6 +-
.../CommunicationFailureContext.java | 62 +
.../CommunicationFailureResolver.java | 28 +
.../DefaultCommunicationFailureResolver.java | 305 ++
.../configuration/IgniteConfiguration.java | 22 +
.../org/apache/ignite/internal/GridTopic.java | 3 +
.../apache/ignite/internal/IgniteKernal.java | 5 +-
.../org/apache/ignite/internal/IgnitionEx.java | 3 +
.../internal/managers/GridManagerAdapter.java | 8 +
.../managers/communication/GridIoManager.java | 4 +-
.../communication/GridIoMessageFactory.java | 12 +
.../discovery/CustomMessageWrapper.java | 5 +
.../internal/managers/discovery/DiscoCache.java | 8 +
.../discovery/DiscoveryCustomMessage.java | 10 +-
.../DiscoveryMessageResultsCollector.java | 222 +
.../discovery/GridDiscoveryManager.java | 128 +-
.../managers/discovery/IgniteClusterNode.java | 69 +
.../managers/discovery/IgniteDiscoverySpi.java | 67 +
.../IgniteDiscoverySpiInternalListener.java | 42 +
.../authentication/UserAcceptedMessage.java | 5 +
.../authentication/UserProposedMessage.java | 5 +
.../cache/CacheAffinityChangeMessage.java | 5 +
.../cache/CacheAffinitySharedManager.java | 26 +-
.../cache/CacheStatisticsModeChangeMessage.java | 5 +
.../ClientCacheChangeDiscoveryMessage.java | 5 +
.../ClientCacheChangeDummyDiscoveryMessage.java | 5 +
.../cache/DynamicCacheChangeBatch.java | 5 +
.../processors/cache/GridCacheAdapter.java | 3 +-
.../GridCachePartitionExchangeManager.java | 16 +-
.../processors/cache/GridCacheProcessor.java | 4 +-
.../processors/cache/GridCacheUtils.java | 6 +-
.../processors/cache/WalStateFinishMessage.java | 5 +
.../cache/WalStateProposeMessage.java | 5 +
.../cache/binary/BinaryMetadataTransport.java | 24 +-
.../binary/MetadataUpdateAcceptedMessage.java | 5 +
.../binary/MetadataUpdateProposedMessage.java | 5 +
.../dht/GridClientPartitionTopology.java | 39 +
.../distributed/dht/GridDhtCacheAdapter.java | 4 +
.../dht/GridDhtPartitionTopology.java | 6 +
.../dht/GridDhtPartitionTopologyImpl.java | 39 +
.../GridDhtPartitionsExchangeFuture.java | 26 +-
.../cluster/ChangeGlobalStateFinishMessage.java | 5 +
.../cluster/ChangeGlobalStateMessage.java | 5 +
.../cluster/ClusterMetricsUpdateMessage.java | 158 +
.../processors/cluster/ClusterNodeMetrics.java | 62 +
.../processors/cluster/ClusterProcessor.java | 249 +-
.../continuous/AbstractContinuousMessage.java | 5 +
.../continuous/ContinuousRoutineInfo.java | 100 +
.../ContinuousRoutineStartResultMessage.java | 206 +
.../ContinuousRoutinesCommonDiscoveryData.java | 45 +
.../continuous/ContinuousRoutinesInfo.java | 132 +
...tinuousRoutinesJoiningNodeDiscoveryData.java | 45 +
.../continuous/GridContinuousProcessor.java | 862 +++-
.../continuous/StartRequestDataV2.java | 164 +
.../StartRoutineDiscoveryMessageV2.java | 77 +
.../StopRoutineAckDiscoveryMessage.java | 5 +
.../datastreamer/DataStreamerImpl.java | 27 +-
.../marshaller/MappingAcceptedMessage.java | 5 +
.../marshaller/MappingProposedMessage.java | 5 +
.../message/SchemaFinishDiscoveryMessage.java | 5 +
.../message/SchemaProposeDiscoveryMessage.java | 5 +
.../ignite/internal/util/nio/GridNioServer.java | 18 +-
.../org/apache/ignite/spi/IgniteSpiAdapter.java | 10 +
.../org/apache/ignite/spi/IgniteSpiContext.java | 11 +
.../communication/tcp/TcpCommunicationSpi.java | 190 +-
.../tcp/internal/ConnectionKey.java | 117 +
.../TcpCommunicationConnectionCheckFuture.java | 519 ++
...pCommunicationNodeConnectionCheckFuture.java | 30 +
.../discovery/DiscoverySpiCustomMessage.java | 15 +-
...DiscoverySpiMutableCustomMessageSupport.java | 40 +
.../spi/discovery/tcp/TcpDiscoverySpi.java | 55 +-
.../tcp/internal/TcpDiscoveryNode.java | 32 +-
.../resources/META-INF/classnames.properties | 2 +
...unctionExcludeNeighborsAbstractSelfTest.java | 8 +-
.../failure/FailureHandlerTriggeredTest.java | 4 +
.../internal/ClusterGroupHostsSelfTest.java | 3 +
.../ignite/internal/ClusterGroupSelfTest.java | 2 +
.../internal/ClusterNodeMetricsUpdateTest.java | 173 +
.../internal/DiscoverySpiTestListener.java | 162 +
.../ignite/internal/GridDiscoverySelfTest.java | 14 +-
.../GridJobMasterLeaveAwareSelfTest.java | 2 +
.../internal/GridJobStealingSelfTest.java | 2 +
.../internal/GridSameVmStartupSelfTest.java | 19 +-
.../apache/ignite/internal/GridSelfTest.java | 2 +
.../IgniteClientReconnectAbstractTest.java | 53 +-
.../IgniteClientReconnectApiExceptionTest.java | 21 +-
.../IgniteClientReconnectAtomicsTest.java | 30 +-
.../IgniteClientReconnectCacheTest.java | 49 +-
.../IgniteClientReconnectCollectionsTest.java | 14 +-
.../IgniteClientReconnectComputeTest.java | 6 +-
...eClientReconnectContinuousProcessorTest.java | 13 +-
...IgniteClientReconnectDiscoveryStateTest.java | 22 +-
...niteClientReconnectFailoverAbstractTest.java | 12 +-
.../IgniteClientReconnectServicesTest.java | 8 +-
.../internal/IgniteClientReconnectStopTest.java | 12 +-
.../IgniteClientReconnectStreamerTest.java | 4 +-
.../ignite/internal/IgniteClientRejoinTest.java | 3 +
.../GridDiscoveryManagerAliveCacheSelfTest.java | 16 +-
.../GridAffinityProcessorAbstractSelfTest.java | 4 +-
.../CacheMetricsForClusterGroupSelfTest.java | 12 +-
.../cache/GridCacheAbstractSelfTest.java | 2 +
.../cache/IgniteCacheNearLockValueSelfTest.java | 4 +-
.../IgniteCacheP2pUnmarshallingErrorTest.java | 11 +
.../IgniteClusterActivateDeactivateTest.java | 65 +
.../IgniteDaemonNodeMarshallerCacheTest.java | 3 +-
.../binary/BinaryMetadataUpdatesFlowTest.java | 12 +-
...ntNodeBinaryObjectMetadataMultinodeTest.java | 2 +-
.../GridCacheQueueClientDisconnectTest.java | 10 +
.../IgniteClientDataStructuresAbstractTest.java | 3 +-
.../CacheLateAffinityAssignmentTest.java | 127 +-
.../GridCacheNodeFailureAbstractTest.java | 5 +-
.../distributed/IgniteCache150ClientsTest.java | 2 +
.../distributed/IgniteCacheManyClientsTest.java | 44 +-
.../IgniteOptimisticTxSuspendResumeTest.java | 2 +
...ridCacheDhtPreloadMultiThreadedSelfTest.java | 4 +
.../dht/GridCacheDhtPreloadSelfTest.java | 2 +
.../dht/TxRecoveryStoreEnabledTest.java | 15 +-
...titionedExplicitLockNodeFailureSelfTest.java | 3 +-
.../ClientReconnectContinuousQueryTest.java | 19 +-
...yRemoteFilterMissingInClassPathSelfTest.java | 23 +-
...CacheContinuousQueryClientReconnectTest.java | 3 +
.../CacheVersionedEntryAbstractTest.java | 33 +-
.../continuous/GridEventConsumeSelfTest.java | 34 +-
.../service/ClosureServiceClientsNodesTest.java | 19 +-
.../internal/util/GridTestClockTimer.java | 9 +
.../GridMarshallerMappingConsistencyTest.java | 4 +
.../ignite/messaging/GridMessagingSelfTest.java | 126 +-
.../GridTcpCommunicationSpiAbstractTest.java | 71 +
.../FilterDataForClientNodeDiscoveryTest.java | 5 +
.../testframework/GridSpiTestContext.java | 10 +
.../config/GridTestProperties.java | 9 +
.../testframework/junits/GridAbstractTest.java | 129 +-
.../junits/multijvm/IgniteNodeRunner.java | 2 +
.../testsuites/IgniteComputeGridTestSuite.java | 2 +
...niteCacheDistributedQueryCancelSelfTest.java | 2 +-
.../DynamicIndexAbstractBasicSelfTest.java | 5 +-
.../GridJtaTransactionManagerSelfTest.java | 21 +-
.../GridPartitionedCacheJtaFactorySelfTest.java | 19 +-
.../org/apache/ignite/spark/IgniteRDD.scala | 9 +-
.../ignite/internal/GridFactorySelfTest.java | 3 +-
.../p2p/GridP2PUserVersionChangeSelfTest.java | 5 +-
modules/yardstick/pom-standalone.xml | 6 +
modules/yardstick/pom.xml | 6 +
modules/zookeeper/pom.xml | 40 +
.../spi/discovery/zk/ZookeeperDiscoverySpi.java | 557 ++
.../zk/internal/ZkAbstractCallabck.java | 83 +
.../zk/internal/ZkAbstractChildrenCallback.java | 61 +
.../zk/internal/ZkAbstractWatcher.java | 55 +
.../discovery/zk/internal/ZkAliveNodeData.java | 40 +
.../zk/internal/ZkBulkJoinContext.java | 50 +
.../discovery/zk/internal/ZkClusterNodes.java | 103 +
.../internal/ZkCommunicationErrorNodeState.java | 46 +
.../ZkCommunicationErrorProcessFuture.java | 411 ++
...kCommunicationErrorResolveFinishMessage.java | 69 +
.../ZkCommunicationErrorResolveResult.java | 45 +
...ZkCommunicationErrorResolveStartMessage.java | 61 +
.../internal/ZkCommunicationFailureContext.java | 188 +
.../zk/internal/ZkDiscoveryCustomEventData.java | 89 +
.../zk/internal/ZkDiscoveryEventData.java | 165 +
.../zk/internal/ZkDiscoveryEventsData.java | 121 +
.../internal/ZkDiscoveryNodeFailEventData.java | 55 +
.../internal/ZkDiscoveryNodeJoinEventData.java | 60 +
.../ZkDistributedCollectDataFuture.java | 250 +
.../zk/internal/ZkForceNodeFailMessage.java | 65 +
.../discovery/zk/internal/ZkIgnitePaths.java | 307 ++
.../zk/internal/ZkInternalJoinErrorMessage.java | 44 +
.../zk/internal/ZkInternalMessage.java | 27 +
.../zk/internal/ZkJoinEventDataForJoined.java | 83 +
.../zk/internal/ZkJoinedNodeEvtData.java | 79 +
.../zk/internal/ZkJoiningNodeData.java | 87 +
.../zk/internal/ZkNoServersMessage.java | 50 +
.../zk/internal/ZkNodeValidateResult.java | 43 +
.../spi/discovery/zk/internal/ZkRunnable.java | 51 +
.../discovery/zk/internal/ZkRuntimeState.java | 135 +
.../discovery/zk/internal/ZkTimeoutObject.java | 54 +
.../discovery/zk/internal/ZookeeperClient.java | 1219 +++++
.../ZookeeperClientFailedException.java | 40 +
.../zk/internal/ZookeeperClusterNode.java | 362 ++
.../zk/internal/ZookeeperDiscoveryImpl.java | 4464 ++++++++++++++++
.../java/org/apache/ZookeeperNodeStart.java | 46 +
...CacheEntryListenerWithZkDiscoAtomicTest.java | 32 +
.../ZookeeperDiscoverySpiAbstractTestSuite.java | 118 +
.../zk/ZookeeperDiscoverySpiTestSuite1.java | 44 +
.../zk/ZookeeperDiscoverySpiTestSuite2.java | 94 +
...ZookeeperDiscoverySuitePreprocessorTest.java | 101 +
.../zk/internal/ZookeeperClientTest.java | 495 ++
...okeeperDiscoverySpiSaslAuthAbstractTest.java | 247 +
...ZookeeperDiscoverySpiSaslFailedAuthTest.java | 44 +
...eeperDiscoverySpiSaslSuccessfulAuthTest.java | 48 +
.../zk/internal/ZookeeperDiscoverySpiTest.java | 4847 ++++++++++++++++++
.../zookeeper/ZkTestClientCnxnSocketNIO.java | 137 +
191 files changed, 21158 insertions(+), 777 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
index f4c0ca3..0a055a9 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
@@ -138,8 +138,10 @@ public abstract class JdbcAbstractDmlStatementSelfTest extends GridCommonAbstrac
@Override protected void afterTest() throws Exception {
((IgniteEx)ignite(0)).context().cache().dynamicDestroyCache(DEFAULT_CACHE_NAME, true, true, false);
- conn.close();
- assertTrue(conn.isClosed());
+ if (conn != null) {
+ conn.close();
+ assertTrue(conn.isClosed());
+ }
cleanUpWorkingDir();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureContext.java b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureContext.java
new file mode 100644
index 0000000..a32d38c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureContext.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.spi.communication.CommunicationSpi;
+
+/**
+ * Communication Failure Context.
+ */
+public interface CommunicationFailureContext {
+ /**
+ * @return Current topology snapshot.
+ */
+ public List<ClusterNode> topologySnapshot();
+
+ /**
+ * @param node1 First node.
+ * @param node2 Second node.
+ * @return {@code True} if {@link CommunicationSpi} is able to establish connection from first node to second node.
+ */
+ public boolean connectionAvailable(ClusterNode node1, ClusterNode node2);
+
+ /**
+ * @return Currently started caches.
+ */
+ public Map<String, CacheConfiguration<?, ?>> startedCaches();
+
+ /**
+ * @param cacheName Cache name.
+ * @return Cache partitions affinity assignment.
+ */
+ public List<List<ClusterNode>> cacheAffinity(String cacheName);
+
+ /**
+ * @param cacheName Cache name.
+ * @return Cache partitions owners.
+ */
+ public List<List<ClusterNode>> cachePartitionOwners(String cacheName);
+
+ /**
+ * @param node Node to kill.
+ */
+ public void killNode(ClusterNode node);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureResolver.java b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureResolver.java
new file mode 100644
index 0000000..a4d92f3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureResolver.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+/**
+ * Communication Failure Resolver.
+ */
+public interface CommunicationFailureResolver {
+ /**
+ * @param ctx Context.
+ */
+ public void resolve(CommunicationFailureContext ctx);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java b/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java
new file mode 100644
index 0000000..a4c6da9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import java.util.BitSet;
+import java.util.List;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.LoggerResource;
+
+/**
+ * Default Communication Failure Resolver.
+ */
+public class DefaultCommunicationFailureResolver implements CommunicationFailureResolver {
+ /** */
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** {@inheritDoc} */
+ @Override public void resolve(CommunicationFailureContext ctx) {
+ ClusterGraph graph = new ClusterGraph(log, ctx);
+
+ ClusterSearch cluster = graph.findLargestIndependentCluster();
+
+ List<ClusterNode> nodes = ctx.topologySnapshot();
+
+ assert nodes.size() > 0;
+ assert cluster != null;
+
+ if (graph.checkFullyConnected(cluster.nodesBitSet)) {
+ assert cluster.nodeCnt <= nodes.size();
+
+ if (cluster.nodeCnt < nodes.size()) {
+ if (log.isInfoEnabled()) {
+ log.info("Communication problem resolver found fully connected independent cluster [" +
+ "clusterSrvCnt=" + cluster.srvCnt +
+ ", clusterTotalNodes=" + cluster.nodeCnt +
+ ", totalAliveNodes=" + nodes.size() + "]");
+ }
+
+ for (int i = 0; i < nodes.size(); i++) {
+ if (!cluster.nodesBitSet.get(i))
+ ctx.killNode(nodes.get(i));
+ }
+ }
+ else
+ U.warn(log, "All alive nodes are fully connected, this should be resolved automatically.");
+ }
+ else {
+ if (log.isInfoEnabled()) {
+ log.info("Communication problem resolver failed to find fully connected independent cluster.");
+ }
+ }
+ }
+
+ /**
+ * @param cluster Cluster nodes mask.
+ * @param nodes Nodes.
+ * @param limit IDs limit.
+ * @return Cluster node IDs string.
+ */
+ private static String clusterNodeIds(BitSet cluster, List<ClusterNode> nodes, int limit) {
+ int startIdx = 0;
+
+ StringBuilder builder = new StringBuilder();
+
+ int cnt = 0;
+
+ for (;;) {
+ int idx = cluster.nextSetBit(startIdx);
+
+ if (idx == -1)
+ break;
+
+ startIdx = idx + 1;
+
+ if (builder.length() == 0) {
+ builder.append('[');
+ }
+ else
+ builder.append(", ");
+
+ builder.append(nodes.get(idx).id());
+
+ if (cnt++ > limit)
+ builder.append(", ...");
+ }
+
+ builder.append(']');
+
+ return builder.toString();
+ }
+
+ /**
+ *
+ */
+ private static class ClusterSearch {
+ /** */
+ int srvCnt;
+
+ /** */
+ int nodeCnt;
+
+ /** */
+ final BitSet nodesBitSet;
+
+ /**
+ * @param nodes Total nodes.
+ */
+ ClusterSearch(int nodes) {
+ nodesBitSet = new BitSet(nodes);
+ }
+ }
+
+ /**
+ *
+ */
+ private static class ClusterGraph {
+ /** */
+ private final static int WORD_IDX_SHIFT = 6;
+
+ /** */
+ private final IgniteLogger log;
+
+ /** */
+ private final int nodeCnt;
+
+ /** */
+ private final long[] visitBitSet;
+
+ /** */
+ private final CommunicationFailureContext ctx;
+
+ /** */
+ private final List<ClusterNode> nodes;
+
+ /**
+ * @param log Logger.
+ * @param ctx Context.
+ */
+ ClusterGraph(IgniteLogger log, CommunicationFailureContext ctx) {
+ this.log = log;
+ this.ctx = ctx;
+
+ nodes = ctx.topologySnapshot();
+
+ nodeCnt = nodes.size();
+
+ assert nodeCnt > 0;
+
+ visitBitSet = initBitSet(nodeCnt);
+ }
+
+ /**
+ * @param bitIndex Bit index.
+ * @return Word index containing bit with given index.
+ */
+ private static int wordIndex(int bitIndex) {
+ return bitIndex >> WORD_IDX_SHIFT;
+ }
+
+ /**
+ * @param bitCnt Number of bits.
+ * @return Bit set words.
+ */
+ static long[] initBitSet(int bitCnt) {
+ return new long[wordIndex(bitCnt - 1) + 1];
+ }
+
+ /**
+ * @return Cluster nodes bit set.
+ */
+ ClusterSearch findLargestIndependentCluster() {
+ ClusterSearch maxCluster = null;
+
+ for (int i = 0; i < nodeCnt; i++) {
+ if (getBit(visitBitSet, i))
+ continue;
+
+ ClusterSearch cluster = new ClusterSearch(nodeCnt);
+
+ search(cluster, i);
+
+ if (log.isInfoEnabled()) {
+ log.info("Communication problem resolver found cluster [srvCnt=" + cluster.srvCnt +
+ ", totalNodeCnt=" + cluster.nodeCnt +
+ ", nodeIds=" + clusterNodeIds(cluster.nodesBitSet, nodes, 1000) + "]");
+ }
+
+ if (maxCluster == null || cluster.srvCnt > maxCluster.srvCnt)
+ maxCluster = cluster;
+ }
+
+ return maxCluster;
+ }
+
+ /**
+ * @param cluster Cluster nodes bit set.
+ * @return {@code True} if all cluster nodes are able to connect to each other.
+ */
+ boolean checkFullyConnected(BitSet cluster) {
+ int startIdx = 0;
+
+ int clusterNodes = cluster.cardinality();
+
+ for (;;) {
+ int idx = cluster.nextSetBit(startIdx);
+
+ if (idx == -1)
+ break;
+
+ ClusterNode node1 = nodes.get(idx);
+
+ for (int i = 0; i < clusterNodes; i++) {
+ if (!cluster.get(i) || i == idx)
+ continue;
+
+ ClusterNode node2 = nodes.get(i);
+
+ if (cluster.get(i) && !ctx.connectionAvailable(node1, node2))
+ return false;
+ }
+
+ startIdx = idx + 1;
+ }
+
+ return true;
+ }
+
+ /**
+ * @param cluster Current cluster bit set.
+ * @param idx Node index.
+ */
+ void search(ClusterSearch cluster, int idx) {
+ assert !getBit(visitBitSet, idx);
+
+ setBit(visitBitSet, idx);
+
+ cluster.nodesBitSet.set(idx);
+ cluster.nodeCnt++;
+
+ ClusterNode node1 = nodes.get(idx);
+
+ if (!CU.clientNode(node1))
+ cluster.srvCnt++;
+
+ for (int i = 0; i < nodeCnt; i++) {
+ if (i == idx || getBit(visitBitSet, i))
+ continue;
+
+ ClusterNode node2 = nodes.get(i);
+
+ boolean connected = ctx.connectionAvailable(node1, node2) ||
+ ctx.connectionAvailable(node2, node1);
+
+ if (connected)
+ search(cluster, i);
+ }
+ }
+
+ /**
+ * @param words Bit set words.
+ * @param bitIndex Bit index.
+ */
+ static void setBit(long words[], int bitIndex) {
+ int wordIndex = wordIndex(bitIndex);
+
+ words[wordIndex] |= (1L << bitIndex);
+ }
+
+ /**
+ * @param words Bit set words.
+ * @param bitIndex Bit index.
+ * @return Bit value.
+ */
+ static boolean getBit(long[] words, int bitIndex) {
+ int wordIndex = wordIndex(bitIndex);
+
+ return (words[wordIndex] & (1L << bitIndex)) != 0;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DefaultCommunicationFailureResolver.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index add3880..cc3ea10 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -493,6 +493,9 @@ public class IgniteConfiguration {
/** Failure handler. */
private FailureHandler failureHnd;
+ /** Communication failure resolver */
+ private CommunicationFailureResolver commFailureRslvr;
+
/**
* Creates valid grid configuration with all default values.
*/
@@ -520,6 +523,8 @@ public class IgniteConfiguration {
loadBalancingSpi = cfg.getLoadBalancingSpi();
indexingSpi = cfg.getIndexingSpi();
+ commFailureRslvr = cfg.getCommunicationFailureResolver();
+
/*
* Order alphabetically for maintenance purposes.
*/
@@ -607,6 +612,23 @@ public class IgniteConfiguration {
}
/**
+ * @return Communication failure resovler.
+ */
+ public CommunicationFailureResolver getCommunicationFailureResolver() {
+ return commFailureRslvr;
+ }
+
+ /**
+ * @param commFailureRslvr Communication failure resovler.
+ * @return {@code this} instance.
+ */
+ public IgniteConfiguration setCommunicationFailureResolver(CommunicationFailureResolver commFailureRslvr) {
+ this.commFailureRslvr = commFailureRslvr;
+
+ return this;
+ }
+
+ /**
* Gets optional grid name. Returns {@code null} if non-default grid name was not
* provided.
* <p>The name only works locally and has no effect on topology</p>
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index 4932e67..1227e8c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -121,6 +121,9 @@ public enum GridTopic {
TOPIC_WAL,
/** */
+ TOPIC_METRICS,
+
+ /** */
TOPIC_AUTH;
/** Enum values. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 8bc46fd..0b102e5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1298,7 +1298,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
ackStart(rtBean);
if (!isDaemon())
- ctx.discovery().ackTopology(localNode().order());
+ ctx.discovery().ackTopology(ctx.discovery().localJoin().joinTopologyVersion().topologyVersion());
}
/**
@@ -2623,6 +2623,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
objs.add(cfg.getGridLogger());
objs.add(cfg.getMBeanServer());
+ if (cfg.getCommunicationFailureResolver() != null)
+ objs.add(cfg.getCommunicationFailureResolver());
+
return objs;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 4708dd3..417ba1e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -75,6 +75,7 @@ import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.binary.BinaryMarshaller;
import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
import org.apache.ignite.internal.processors.igfs.IgfsThreadFactory;
import org.apache.ignite.internal.processors.igfs.IgfsUtils;
@@ -2243,6 +2244,8 @@ public class IgnitionEx {
initializeDefaultSpi(myCfg);
+ GridDiscoveryManager.initCommunicationErrorResolveConfiguration(myCfg);
+
initializeDefaultCacheConfiguration(myCfg);
ExecutorConfiguration[] execCfgs = myCfg.getExecutorConfiguration();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
index 74f5a10..b0756cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
@@ -618,6 +618,14 @@ public abstract class GridManagerAdapter<T extends IgniteSpi> implements GridMan
return ctx.nodeAttributes();
}
+ @Override public boolean communicationFailureResolveSupported() {
+ return ctx.discovery().communicationErrorResolveSupported();
+ }
+
+ @Override public void resolveCommunicationFailure(ClusterNode node, Exception err) {
+ ctx.discovery().resolveCommunicationError(node, err);
+ }
+
/**
* @param e Exception to handle.
* @return GridSpiException Converted exception.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index d5cdd2d..8d9a700 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -298,9 +298,9 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
@Override public MessageReader reader(UUID rmtNodeId, MessageFactory msgFactory)
throws IgniteCheckedException {
- assert rmtNodeId != null;
- return new DirectMessageReader(msgFactory, U.directProtocolVersion(ctx, rmtNodeId));
+ return new DirectMessageReader(msgFactory,
+ rmtNodeId != null ? U.directProtocolVersion(ctx, rmtNodeId) : GridIoManager.DIRECT_PROTO_VER);
}
};
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index a0fc2f8..5616fd0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -123,6 +123,8 @@ import org.apache.ignite.internal.processors.cache.transactions.TxLocksResponse;
import org.apache.ignite.internal.processors.cache.version.GridCacheRawVersionedEntry;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx;
+import org.apache.ignite.internal.processors.cluster.ClusterMetricsUpdateMessage;
+import org.apache.ignite.internal.processors.continuous.ContinuousRoutineStartResultMessage;
import org.apache.ignite.internal.processors.continuous.GridContinuousMessage;
import org.apache.ignite.internal.processors.datastreamer.DataStreamerEntry;
import org.apache.ignite.internal.processors.datastreamer.DataStreamerRequest;
@@ -909,6 +911,16 @@ public class GridIoMessageFactory implements MessageFactory {
break;
+ case 133:
+ msg = new ClusterMetricsUpdateMessage();
+
+ break;
+
+ case 134:
+ msg = new ContinuousRoutineStartResultMessage();
+
+ break;
+
// [-3..119] [124..129] [-23..-27] [-36..-55]- this
// [120..123] - DR
// [-4..-22, -30..-35] - SQL
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
index 4268886..4b6b7a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
@@ -49,6 +49,11 @@ public class CustomMessageWrapper implements DiscoverySpiCustomMessage {
return delegate.isMutable();
}
+ /** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return delegate.stopProcess();
+ }
+
/**
* @return Delegate.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
index c21698f..fef44fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -312,6 +312,14 @@ public class DiscoCache {
}
/**
+ * @param nodeId Node ID.
+ * @return {@code True} if node is in alives list.
+ */
+ public boolean alive(UUID nodeId) {
+ return alives.contains(nodeId);
+ }
+
+ /**
* Gets all nodes that have cache with given name.
*
* @param cacheName Cache name.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryCustomMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryCustomMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryCustomMessage.java
index c708c62..6ed2096 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryCustomMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryCustomMessage.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.managers.discovery;
import java.io.Serializable;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddedMessage;
import org.jetbrains.annotations.Nullable;
@@ -87,11 +88,18 @@ public interface DiscoveryCustomMessage extends Serializable {
@Nullable public DiscoveryCustomMessage ackMessage();
/**
- * @return {@code true} if message can be modified during listener notification. Changes will be send to next nodes.
+ * @return {@code True} if message can be modified during listener notification. Changes will be sent to next nodes.
*/
public boolean isMutable();
/**
+ * See {@link DiscoverySpiCustomMessage#stopProcess()}.
+ *
+ * @return {@code True} if message should not be sent to others nodes after it was processed on coordinator.
+ */
+ public boolean stopProcess();
+
+ /**
* Creates new discovery cache if message caused topology version change.
*
* @param mgr Discovery manager.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryMessageResultsCollector.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryMessageResultsCollector.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryMessageResultsCollector.java
new file mode 100644
index 0000000..43be952
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryMessageResultsCollector.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.discovery;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public abstract class DiscoveryMessageResultsCollector<M, R> {
+ /** */
+ private final Map<UUID, NodeMessage<M>> rcvd = new HashMap<>();
+
+ /** */
+ private int leftMsgs;
+
+ /** */
+ protected DiscoCache discoCache;
+
+ /** */
+ protected final GridKernalContext ctx;
+
+ /**
+ * @param ctx Context.
+ */
+ protected DiscoveryMessageResultsCollector(GridKernalContext ctx) {
+ this.ctx = ctx;
+ }
+
+ /**
+ * @param rcvd Received messages.
+ * @return Result.
+ */
+ protected abstract R createResult(Map<UUID, NodeMessage<M>> rcvd);
+
+ /**
+ * @param r Result.
+ */
+ protected abstract void onResultsCollected(R r);
+
+ /**
+ * @param discoCache Discovery state when discovery message was received.
+ * @param node Node.
+ * @return {@code True} if need wait for result from given node.
+ */
+ protected abstract boolean waitForNode(DiscoCache discoCache, ClusterNode node);
+
+ /**
+ * @param discoCache Discovery state.
+ */
+ public final void init(DiscoCache discoCache) {
+ assert discoCache != null;
+
+ R res = null;
+
+ synchronized (this) {
+ assert this.discoCache == null;
+ assert leftMsgs == 0 : leftMsgs;
+
+ this.discoCache = discoCache;
+
+ for (ClusterNode node : discoCache.allNodes()) {
+ if (ctx.discovery().alive(node) && waitForNode(discoCache, node) && !rcvd.containsKey(node.id())) {
+ rcvd.put(node.id(), new NodeMessage<>((M)null));
+
+ leftMsgs++;
+ }
+ }
+
+ if (leftMsgs == 0)
+ res = createResult(rcvd);
+ }
+
+ if (res != null)
+ onResultsCollected(res);
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @param msg Message.
+ */
+ public final void onMessage(UUID nodeId, M msg) {
+ R res = null;
+
+ synchronized (this) {
+ if (allReceived())
+ return;
+
+ NodeMessage<M> expMsg = rcvd.get(nodeId);
+
+ if (expMsg == null)
+ rcvd.put(nodeId, new NodeMessage<>(msg));
+ else if (expMsg.set(msg)) {
+ assert leftMsgs > 0;
+
+ leftMsgs--;
+
+ if (allReceived())
+ res = createResult(rcvd);
+ }
+ }
+
+ if (res != null)
+ onResultsCollected(res);
+ }
+
+ /**
+ * @param nodeId Failed node ID.
+ */
+ public final void onNodeFail(UUID nodeId) {
+ R res = null;
+
+ synchronized (this) {
+ if (allReceived())
+ return;
+
+ NodeMessage expMsg = rcvd.get(nodeId);
+
+ if (expMsg != null && expMsg.onNodeFailed()) {
+ assert leftMsgs > 0 : leftMsgs;
+
+ leftMsgs--;
+
+ if (allReceived())
+ res = createResult(rcvd);
+ }
+ }
+
+ if (res != null)
+ onResultsCollected(res);
+ }
+
+ /**
+ * @return {@code True} if expected messages are initialized and all message are received.
+ */
+ private boolean allReceived() {
+ return discoCache != null && leftMsgs == 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DiscoveryMessageResultsCollector.class, this);
+ }
+
+ /**
+ *
+ */
+ protected static class NodeMessage<M> {
+ /** */
+ boolean nodeFailed;
+
+ /** */
+ M msg;
+
+ /**
+ * @param msg Message.
+ */
+ NodeMessage(M msg) {
+ this.msg = msg;
+ }
+
+ /**
+ * @return Message or {@code null} if node failed.
+ */
+ @Nullable public M message() {
+ return msg;
+ }
+
+ /**
+ * @return {@code True} if node result was not set before.
+ */
+ boolean onNodeFailed() {
+ if (nodeFailed || msg != null)
+ return false;
+
+ nodeFailed = true;
+
+ return true;
+ }
+
+ /**
+ * @param msg Received message.
+ * @return {@code True} if node result was not set before.
+ */
+ boolean set(M msg) {
+ assert msg != null;
+
+ if (this.msg != null)
+ return false;
+
+ this.msg = msg;
+
+ return !nodeFailed;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(NodeMessage.class, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 2e814d4..4c5690e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -54,8 +54,11 @@ import org.apache.ignite.cluster.BaselineNode;
import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.CommunicationFailureResolver;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.DefaultCommunicationFailureResolver;
+import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.failure.FailureContext;
@@ -112,6 +115,8 @@ import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.plugin.security.SecurityCredentials;
import org.apache.ignite.plugin.segmentation.SegmentationPolicy;
import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
import org.apache.ignite.spi.discovery.DiscoveryDataBag;
import org.apache.ignite.spi.discovery.DiscoveryDataBag.JoiningNodeDiscoveryData;
import org.apache.ignite.spi.discovery.DiscoveryMetricsProvider;
@@ -120,10 +125,10 @@ import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange;
import org.apache.ignite.spi.discovery.DiscoverySpiHistorySupport;
import org.apache.ignite.spi.discovery.DiscoverySpiListener;
+import org.apache.ignite.spi.discovery.DiscoverySpiMutableCustomMessageSupport;
import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
import org.apache.ignite.thread.IgniteThread;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -478,7 +483,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
/** {@inheritDoc} */
@Override protected void onKernalStart0() throws IgniteCheckedException {
- if (Boolean.TRUE.equals(ctx.config().isClientMode()) && !getSpi().isClientMode())
+ if ((getSpi() instanceof TcpDiscoverySpi) && Boolean.TRUE.equals(ctx.config().isClientMode()) && !getSpi().isClientMode())
ctx.performance().add("Enable client mode for TcpDiscoverySpi " +
"(set TcpDiscoverySpi.forceServerMode to false)");
}
@@ -551,6 +556,9 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
});
}
+ if (ctx.config().getCommunicationFailureResolver() != null)
+ ctx.resource().injectGeneric(ctx.config().getCommunicationFailureResolver());
+
spi.setListener(new DiscoverySpiListener() {
private long gridStartTime;
@@ -559,8 +567,8 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
for (IgniteInClosure<ClusterNode> lsnr : locNodeInitLsnrs)
lsnr.apply(locNode);
- if (locNode instanceof TcpDiscoveryNode) {
- final TcpDiscoveryNode node = (TcpDiscoveryNode)locNode;
+ if (locNode instanceof IgniteClusterNode) {
+ final IgniteClusterNode node = (IgniteClusterNode)locNode;
if (consistentId != null)
node.setConsistentId(consistentId);
@@ -1052,7 +1060,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
/**
* @return Metrics provider.
*/
- private DiscoveryMetricsProvider createMetricsProvider() {
+ public DiscoveryMetricsProvider createMetricsProvider() {
return new DiscoveryMetricsProvider() {
/** */
private final long startTime = U.currentTimeMillis();
@@ -1679,13 +1687,15 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
return getSpi().pingNode(nodeId);
}
catch (IgniteException e) {
- if (e.hasCause(IgniteClientDisconnectedCheckedException.class)) {
+ if (e.hasCause(IgniteClientDisconnectedCheckedException.class, IgniteClientDisconnectedException.class)) {
IgniteFuture<?> reconnectFut = ctx.cluster().clientReconnectFuture();
throw new IgniteClientDisconnectedCheckedException(reconnectFut, e.getMessage());
}
- throw e;
+ LT.warn(log, "Ping failed with error [node=" + nodeId + ", err=" + e + ']');
+
+ return true;
}
finally {
busyLock.leaveBusy();
@@ -2025,7 +2035,16 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
Map<Long, Collection<ClusterNode>> snapshots = topHist;
- return snapshots.get(topVer);
+ Collection<ClusterNode> nodes = snapshots.get(topVer);
+
+ if (nodes == null) {
+ DiscoCache cache = discoCacheHist.get(new AffinityTopologyVersion(topVer, 0));
+
+ if (cache != null)
+ nodes = cache.allNodes();
+ }
+
+ return nodes;
}
/**
@@ -2158,6 +2177,19 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
}
/**
+ * @param discoCache
+ * @param node
+ */
+ public void metricsUpdateEvent(DiscoCache discoCache, ClusterNode node) {
+ discoWrk.addEvent(EVT_NODE_METRICS_UPDATED,
+ discoCache.version(),
+ node,
+ discoCache,
+ discoCache.nodeMap.values(),
+ null);
+ }
+
+ /**
* Gets first grid node start time, see {@link DiscoverySpi#getGridStartTime()}.
*
* @return Start time of the first grid node.
@@ -2211,8 +2243,9 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
public boolean reconnectSupported() {
DiscoverySpi spi = getSpi();
- return ctx.discovery().localNode().isClient() && (spi instanceof TcpDiscoverySpi) &&
- !(((TcpDiscoverySpi) spi).isClientReconnectDisabled());
+ return ctx.discovery().localNode().isClient() &&
+ (spi instanceof IgniteDiscoverySpi) &&
+ ((IgniteDiscoverySpi)spi).clientReconnectSupported();
}
/**
@@ -2225,7 +2258,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
DiscoverySpi discoverySpi = getSpi();
- ((TcpDiscoverySpi)discoverySpi).reconnect();
+ ((IgniteDiscoverySpi)discoverySpi).clientReconnect();
}
/**
@@ -2379,6 +2412,76 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
cacheNodes.add(rich);
}
+ /**
+ * @param cfg Configuration.
+ * @throws IgniteCheckedException If configuration is not valid.
+ */
+ public static void initCommunicationErrorResolveConfiguration(IgniteConfiguration cfg) throws IgniteCheckedException {
+ CommunicationFailureResolver rslvr = cfg.getCommunicationFailureResolver();
+ CommunicationSpi commSpi = cfg.getCommunicationSpi();
+ DiscoverySpi discoverySpi = cfg.getDiscoverySpi();
+
+ if (rslvr != null) {
+ if (!supportsCommunicationErrorResolve(commSpi))
+ throw new IgniteCheckedException("CommunicationFailureResolver is configured, but CommunicationSpi does not support communication" +
+ "problem resolve: " + commSpi.getClass().getName());
+
+ if (!supportsCommunicationErrorResolve(discoverySpi))
+ throw new IgniteCheckedException("CommunicationFailureResolver is configured, but DiscoverySpi does not support communication" +
+ "problem resolve: " + discoverySpi.getClass().getName());
+ }
+ else {
+ if (supportsCommunicationErrorResolve(commSpi) && supportsCommunicationErrorResolve(discoverySpi))
+ cfg.setCommunicationFailureResolver(new DefaultCommunicationFailureResolver());
+ }
+ }
+
+ /**
+ * @param spi Discovery SPI.
+ * @return {@code True} if SPI supports communication error resolve.
+ */
+ private static boolean supportsCommunicationErrorResolve(DiscoverySpi spi) {
+ return spi instanceof IgniteDiscoverySpi && ((IgniteDiscoverySpi)spi).supportsCommunicationFailureResolve();
+ }
+
+ /**
+ * @param spi Discovery SPI.
+ * @return {@code True} if SPI supports communication error resolve.
+ */
+ private static boolean supportsCommunicationErrorResolve(CommunicationSpi spi) {
+ return spi instanceof TcpCommunicationSpi;
+ }
+
+ /**
+ * @return {@code True} if communication error resolve is supported.
+ */
+ public boolean communicationErrorResolveSupported() {
+ return ctx.config().getCommunicationFailureResolver() != null;
+ }
+
+ /**
+ * @return {@code True} if configured {@link DiscoverySpi} supports mutable custom messages.
+ */
+ public boolean mutableCustomMessages() {
+ DiscoverySpiMutableCustomMessageSupport ann = U.getAnnotation(ctx.config().getDiscoverySpi().getClass(),
+ DiscoverySpiMutableCustomMessageSupport.class);
+
+ return ann != null && ann.value();
+ }
+
+ /**
+ * @param node Problem node.
+ * @param err Error.
+ */
+ public void resolveCommunicationError(ClusterNode node, Exception err) {
+ DiscoverySpi spi = getSpi();
+
+ if (!supportsCommunicationErrorResolve(spi) || !supportsCommunicationErrorResolve(ctx.config().getCommunicationSpi()))
+ throw new UnsupportedOperationException();
+
+ ((IgniteDiscoverySpi)spi).resolveCommunicationFailure(node, err);
+ }
+
/** Worker for network segment checks. */
private class SegmentCheckWorker extends GridWorker {
/** */
@@ -2587,6 +2690,9 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
AffinityTopologyVersion topVer = evt.get2();
+ if (type == EVT_NODE_METRICS_UPDATED && topVer.compareTo(discoCache.version()) < 0)
+ return;
+
ClusterNode node = evt.get3();
boolean isDaemon = node.isDaemon();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteClusterNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteClusterNode.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteClusterNode.java
new file mode 100644
index 0000000..cbc706a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteClusterNode.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.discovery;
+
+import java.io.Serializable;
+import java.util.Map;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public interface IgniteClusterNode extends ClusterNode {
+ /**
+ * Sets consistent globally unique node ID which survives node restarts.
+ *
+ * @param consistentId Consistent globally unique node ID.
+ */
+ public void setConsistentId(Serializable consistentId);
+
+ /**
+ * Sets node metrics.
+ *
+ * @param metrics Node metrics.
+ */
+ public void setMetrics(ClusterMetrics metrics);
+
+ /**
+ * Gets collections of cache metrics for this node. Note that node cache metrics are constantly updated
+ * and provide up to date information about caches.
+ * <p>
+ * Cache metrics are updated with some delay which is directly related to metrics update
+ * frequency. For example, by default the update will happen every {@code 2} seconds.
+ *
+ * @return Runtime metrics snapshots for this node.
+ */
+ public Map<Integer, CacheMetrics> cacheMetrics();
+
+ /**
+ * Sets node cache metrics.
+ *
+ * @param cacheMetrics Cache metrics.
+ */
+ public void setCacheMetrics(Map<Integer, CacheMetrics> cacheMetrics);
+
+ /**
+ * Whether this node is cache client (see {@link IgniteConfiguration#isClientMode()}).
+ *
+ * @return {@code True if client}.
+ */
+ public boolean isCacheClient();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java
new file mode 100644
index 0000000..9aa5d14
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.discovery;
+
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+
+/**
+ *
+ */
+public interface IgniteDiscoverySpi extends DiscoverySpi {
+ /**
+ * @param nodeId Node ID.
+ * @return {@code True} if node joining or already joined topology.
+ */
+ public boolean knownNode(UUID nodeId);
+
+ /**
+ *
+ * @return {@code True} if SPI supports client reconnect.
+ */
+ public boolean clientReconnectSupported();
+
+ /**
+ *
+ */
+ public void clientReconnect();
+
+ /**
+ * For TESTING only.
+ */
+ public void simulateNodeFailure();
+
+ /**
+ * For TESTING only.
+ *
+ * @param lsnr Listener.
+ */
+ public void setInternalListener(IgniteDiscoverySpiInternalListener lsnr);
+
+ /**
+ * @return {@code True} if supports communication error resolve.
+ */
+ public boolean supportsCommunicationFailureResolve();
+
+ /**
+ * @param node Problem node.
+ * @param err Connection error.
+ */
+ public void resolveCommunicationFailure(ClusterNode node, Exception err);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpiInternalListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpiInternalListener.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpiInternalListener.java
new file mode 100644
index 0000000..24405f8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpiInternalListener.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.discovery;
+
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+
+/**
+ * For TESTING only.
+ */
+public interface IgniteDiscoverySpiInternalListener {
+ /**
+ * @param locNode Local node.
+ * @param log Log.
+ */
+ public void beforeJoin(ClusterNode locNode, IgniteLogger log);
+
+ /**
+ * @param spi SPI instance.
+ * @param log Logger.
+ * @param msg Custom message.
+ * @return {@code False} to cancel event send.
+ */
+ public boolean beforeSendCustomEvent(DiscoverySpi spi, IgniteLogger log, DiscoverySpiCustomMessage msg);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAcceptedMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAcceptedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAcceptedMessage.java
index ef87a44..2e2aed9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAcceptedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAcceptedMessage.java
@@ -72,6 +72,11 @@ public class UserAcceptedMessage implements DiscoveryCustomMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
AffinityTopologyVersion topVer, DiscoCache discoCache) {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserProposedMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserProposedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserProposedMessage.java
index 1a0be8e..19f9e82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserProposedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserProposedMessage.java
@@ -72,6 +72,11 @@ public class UserProposedMessage implements DiscoveryServerOnlyCustomMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
AffinityTopologyVersion topVer, DiscoCache discoCache) {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinityChangeMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinityChangeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinityChangeMessage.java
index fe1014c..937a889 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinityChangeMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinityChangeMessage.java
@@ -156,6 +156,11 @@ public class CacheAffinityChangeMessage implements DiscoveryCustomMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
AffinityTopologyVersion topVer, DiscoCache discoCache) {
return discoCache.copy(topVer, null);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 6691b13..92b8d3e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -1310,20 +1310,17 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
* @param fut Current exchange future.
* @param msg Message finish message.
* @param resTopVer Result topology version.
- * @throws IgniteCheckedException If failed.
*/
public void onLocalJoin(final GridDhtPartitionsExchangeFuture fut,
GridDhtPartitionsFullMessage msg,
- final AffinityTopologyVersion resTopVer)
- throws IgniteCheckedException {
+ final AffinityTopologyVersion resTopVer) {
final Set<Integer> affReq = fut.context().groupsAffinityRequestOnJoin();
final Map<Long, ClusterNode> nodesByOrder = new HashMap<>();
final Map<Integer, CacheGroupAffinityMessage> joinedNodeAff = msg.joinedNodeAffinity();
- assert !F.isEmpty(joinedNodeAff) : msg;
- assert joinedNodeAff.size() >= affReq.size();
+ assert F.isEmpty(affReq) || (!F.isEmpty(joinedNodeAff) && joinedNodeAff.size() >= affReq.size()) : msg;
forAllCacheGroups(false, new IgniteInClosureX<GridAffinityAssignmentCache>() {
@Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException {
@@ -1333,7 +1330,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
assert grp != null;
- if (affReq.contains(aff.groupId())) {
+ if (affReq != null && affReq.contains(aff.groupId())) {
assert AffinityTopologyVersion.NONE.equals(aff.lastVersion());
CacheGroupAffinityMessage affMsg = joinedNodeAff.get(aff.groupId());
@@ -2282,6 +2279,23 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
}
/**
+ * @return All registered cache groups.
+ */
+ public Map<Integer, DynamicCacheDescriptor> caches() {
+ return caches.registeredCaches;
+ }
+
+ /**
+ * @param grpId Cache group ID
+ * @return Cache affinity cache.
+ */
+ @Nullable public GridAffinityAssignmentCache groupAffinity(int grpId) {
+ CacheGroupHolder grpHolder = grpHolders.get(grpId);
+
+ return grpHolder != null ? grpHolder.affinity() : null;
+ }
+
+ /**
*
*/
public void dumpDebugInfo() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStatisticsModeChangeMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStatisticsModeChangeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStatisticsModeChangeMessage.java
index 40bcfaf..e33256f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStatisticsModeChangeMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStatisticsModeChangeMessage.java
@@ -101,6 +101,11 @@ public class CacheStatisticsModeChangeMessage implements DiscoveryCustomMessage
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr, AffinityTopologyVersion topVer,
DiscoCache discoCache) {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDiscoveryMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDiscoveryMessage.java
index e35d80e..ae76c95 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDiscoveryMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDiscoveryMessage.java
@@ -173,6 +173,11 @@ public class ClientCacheChangeDiscoveryMessage implements DiscoveryCustomMessage
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
AffinityTopologyVersion topVer, DiscoCache discoCache) {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDummyDiscoveryMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDummyDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDummyDiscoveryMessage.java
index 6ed3ecc..4ce0c87 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDummyDiscoveryMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDummyDiscoveryMessage.java
@@ -105,6 +105,11 @@ public class ClientCacheChangeDummyDiscoveryMessage implements DiscoveryCustomMe
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
AffinityTopologyVersion topVer, DiscoCache discoCache) {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
index 83459a5..d85e29b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
@@ -77,6 +77,11 @@ public class DynamicCacheChangeBatch implements DiscoveryCustomMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr, AffinityTopologyVersion topVer,
DiscoCache discoCache) {
return mgr.createDiscoCacheOnCacheChange(topVer, discoCache);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 55357ff..c2d0f42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -82,6 +82,7 @@ import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.affinity.GridCacheAffinityImpl;
import org.apache.ignite.internal.processors.cache.distributed.IgniteExternalizableExpiryPolicy;
@@ -3234,7 +3235,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
List<CacheMetrics> metrics = new ArrayList<>(grp.nodes().size());
for (ClusterNode node : grp.nodes()) {
- Map<Integer, CacheMetrics> nodeCacheMetrics = ((TcpDiscoveryNode)node).cacheMetrics();
+ Map<Integer, CacheMetrics> nodeCacheMetrics = ((IgniteClusterNode)node).cacheMetrics();
if (nodeCacheMetrics != null) {
CacheMetrics e = nodeCacheMetrics.get(context().cacheId());
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index a30a24a..77ffce3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -427,7 +427,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
if (evt.type() != EVT_DISCOVERY_CUSTOM_EVT) {
assert evt.type() != EVT_NODE_JOINED || n.isLocal() || n.order() > loc.order() :
"Node joined with smaller-than-local " +
- "order [newOrder=" + n.order() + ", locOrder=" + loc.order() + ']';
+ "order [newOrder=" + n.order() + ", locOrder=" + loc.order() + ", evt=" + evt + ']';
exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt);
@@ -570,12 +570,6 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
for (ClusterNode n : cctx.discovery().remoteNodes())
cctx.versions().onReceived(n.id(), n.metrics().getLastDataVersion());
- ClusterNode loc = cctx.localNode();
-
- long startTime = loc.metrics().getStartTime();
-
- assert startTime > 0;
-
DiscoveryLocalJoinData locJoin = cctx.discovery().localJoin();
GridDhtPartitionsExchangeFuture fut = null;
@@ -758,6 +752,14 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
/**
* @param grpId Cache group ID.
+ * @return Topology.
+ */
+ @Nullable public GridDhtPartitionTopology clientTopologyIfExists(int grpId) {
+ return clientTops.get(grpId);
+ }
+
+ /**
+ * @param grpId Cache group ID.
* @param discoCache Discovery data cache.
* @return Topology.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index a3f7c94..7edac73 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3363,7 +3363,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
* @return Validation result or {@code null} in case of success.
*/
@Nullable private IgniteNodeValidationResult validateHashIdResolvers(ClusterNode node) {
- if (!node.isClient()) {
+ if (!CU.clientNode(node)) {
for (DynamicCacheDescriptor desc : cacheDescriptors().values()) {
CacheConfiguration cfg = desc.cacheConfiguration();
@@ -3372,7 +3372,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
Object nodeHashObj = aff.resolveNodeHash(node);
- for (ClusterNode topNode : ctx.discovery().allNodes()) {
+ for (ClusterNode topNode : ctx.discovery().aliveServerNodes()) {
Object topNodeHashObj = aff.resolveNodeHash(topNode);
if (nodeHashObj.hashCode() == topNodeHashObj.hashCode()) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 83ce2ba..a5169d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -65,6 +65,7 @@ import org.apache.ignite.internal.IgniteNodeAttributes;
import org.apache.ignite.internal.cluster.ClusterGroupEmptyCheckedException;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
+import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockCancelledException;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
@@ -97,7 +98,6 @@ import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.lang.IgniteReducer;
import org.apache.ignite.lifecycle.LifecycleAware;
import org.apache.ignite.plugin.CachePluginConfiguration;
-import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
import org.apache.ignite.transactions.Transaction;
import org.apache.ignite.transactions.TransactionConcurrency;
import org.apache.ignite.transactions.TransactionIsolation;
@@ -1348,8 +1348,8 @@ public class GridCacheUtils {
* @return {@code True} if given node is client node (has flag {@link IgniteConfiguration#isClientMode()} set).
*/
public static boolean clientNode(ClusterNode node) {
- if (node instanceof TcpDiscoveryNode)
- return ((TcpDiscoveryNode)node).isCacheClient();
+ if (node instanceof IgniteClusterNode)
+ return ((IgniteClusterNode)node).isCacheClient();
else
return clientNodeDirect(node);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
index 57f25d0..4afa403 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
@@ -67,6 +67,11 @@ public class WalStateFinishMessage extends WalStateAbstractMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return S.toString(WalStateFinishMessage.class, this, "super", super.toString());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
index 747fd6a..b9d96fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
@@ -98,6 +98,11 @@ public class WalStateProposeMessage extends WalStateAbstractMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return S.toString(WalStateProposeMessage.class, this, "super", super.toString());
}
[52/54] [abbrv] ignite git commit: IGNITE-7691: Provide info about
DECIMAL column precision and scale. - Fixes #3691.
Posted by ag...@apache.org.
IGNITE-7691: Provide info about DECIMAL column precision and scale. - Fixes #3691.
Signed-off-by: Nikolay Izhikov <ni...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6df5f99b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6df5f99b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6df5f99b
Branch: refs/heads/ignite-6083
Commit: 6df5f99bec46d9b0eabd2e623f0d26b32d031e4c
Parents: 08a700d
Author: Nikolay Izhikov <ni...@apache.org>
Authored: Fri Apr 13 11:53:50 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Fri Apr 13 11:53:50 2018 +0300
----------------------------------------------------------------------
.../jdbc/thin/JdbcThinMetadataSelfTest.java | 22 +-
.../org/apache/ignite/cache/QueryEntity.java | 41 ++-
.../cache/query/annotations/QuerySqlField.java | 16 +-
.../internal/client/thin/ClientUtils.java | 38 ++-
.../jdbc/thin/JdbcThinDatabaseMetadata.java | 6 +-
.../cache/query/QueryEntityTypeDescriptor.java | 21 ++
.../processors/odbc/jdbc/JdbcColumnMeta.java | 14 +
.../processors/odbc/jdbc/JdbcColumnMetaV4.java | 90 +++++++
.../odbc/jdbc/JdbcConnectionContext.java | 4 +-
.../odbc/jdbc/JdbcMetaColumnsResultV4.java | 50 ++++
.../odbc/jdbc/JdbcRequestHandler.java | 14 +-
.../processors/odbc/jdbc/JdbcResult.java | 8 +
.../utils/PlatformConfigurationUtils.java | 20 ++
.../processors/query/GridQueryProcessor.java | 2 +-
.../processors/query/GridQueryProperty.java | 14 +
.../internal/processors/query/QueryField.java | 26 +-
.../internal/processors/query/QueryUtils.java | 28 +-
.../query/property/QueryBinaryProperty.java | 23 +-
.../query/property/QueryClassProperty.java | 10 +
.../query/h2/ddl/DdlStatementsProcessor.java | 13 +-
.../processors/query/h2/sql/GridSqlColumn.java | 17 +-
.../cache/index/IgniteDecimalSelfTest.java | 265 +++++++++++++++++++
.../h2/GridIndexingSpiAbstractSelfTest.java | 12 +-
.../IgniteCacheQuerySelfTestSuite.java | 2 +
.../QueryEntityConfigurationParityTest.cs | 5 +-
.../Cache/CacheConfigurationTest.cs | 2 +
.../Cache/Configuration/QueryEntity.cs | 4 +-
.../Cache/Configuration/QueryField.cs | 16 +-
.../Configuration/QuerySqlFieldAttribute.cs | 10 +
.../IgniteConfigurationSection.xsd | 10 +
30 files changed, 773 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java
index 4a34be6..cc6790a 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.jdbc.thin;
import java.io.Serializable;
+import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
@@ -143,6 +144,7 @@ public class JdbcThinMetadataSelfTest extends JdbcThinAbstractSelfTest {
stmt.execute("CREATE TABLE \"Quoted\" (\"Id\" INT primary key, \"Name\" VARCHAR(50)) WITH WRAP_KEY");
stmt.execute("CREATE INDEX \"MyTestIndex quoted\" on \"Quoted\" (\"Id\" DESC)");
stmt.execute("CREATE INDEX IDX ON TEST (ID ASC)");
+ stmt.execute("CREATE TABLE TEST_DECIMAL_COLUMN (ID INT primary key, DEC_COL DECIMAL(8, 3))");
}
}
@@ -244,7 +246,8 @@ public class JdbcThinMetadataSelfTest extends JdbcThinAbstractSelfTest {
"org.ORGANIZATION",
"pers.PERSON",
"PUBLIC.TEST",
- "PUBLIC.Quoted"));
+ "PUBLIC.Quoted",
+ "PUBLIC.TEST_DECIMAL_COLUMN"));
Set<String> actualTbls = new HashSet<>(expectedTbls.size());
@@ -390,15 +393,25 @@ public class JdbcThinMetadataSelfTest extends JdbcThinAbstractSelfTest {
"PUBLIC.TEST.VAL.null",
"PUBLIC.TEST.AGE.21",
"PUBLIC.Quoted.Id.null",
- "PUBLIC.Quoted.Name.null"));
+ "PUBLIC.Quoted.Name.null",
+ "PUBLIC.TEST_DECIMAL_COLUMN.ID.null",
+ "PUBLIC.TEST_DECIMAL_COLUMN.DEC_COL.null.8.3"
+ ));
Set<String> actualCols = new HashSet<>(expectedCols.size());
while(rs.next()) {
+ int precision = rs.getInt("COLUMN_SIZE");
+
+ int scale = rs.getInt("DECIMAL_DIGITS");
+
actualCols.add(rs.getString("TABLE_SCHEM") + '.'
+ rs.getString("TABLE_NAME") + "."
+ rs.getString("COLUMN_NAME") + "."
- + rs.getString("COLUMN_DEF"));
+ + rs.getString("COLUMN_DEF")
+ + (precision == 0 ? "" : ("." + precision))
+ + (scale == 0 ? "" : ("." + scale))
+ );
}
assert expectedCols.equals(actualCols) : "expectedCols=" + expectedCols +
@@ -532,7 +545,8 @@ public class JdbcThinMetadataSelfTest extends JdbcThinAbstractSelfTest {
"pers.PERSON.PK_pers_PERSON._KEY",
"PUBLIC.TEST.PK_PUBLIC_TEST.ID",
"PUBLIC.TEST.PK_PUBLIC_TEST.NAME",
- "PUBLIC.Quoted.PK_PUBLIC_Quoted.Id"));
+ "PUBLIC.Quoted.PK_PUBLIC_Quoted.Id",
+ "PUBLIC.TEST_DECIMAL_COLUMN.ID._KEY"));
Set<String> actualPks = new HashSet<>(expectedPks.size());
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java b/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java
index 0065bae..aff3461 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java
@@ -19,6 +19,7 @@ package org.apache.ignite.cache;
import java.io.Serializable;
import java.lang.reflect.Field;
+import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -42,8 +43,11 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
import org.jetbrains.annotations.Nullable;
+import static java.util.Collections.unmodifiableMap;
+
/**
* Query entity is a description of {@link org.apache.ignite.IgniteCache cache} entry (composed of key and value)
* in a way of how it must be indexed and can be queried.
@@ -89,6 +93,9 @@ public class QueryEntity implements Serializable {
/** Fields default values. */
private Map<String, Object> defaultFieldValues = new HashMap<>();
+ /** Decimal fields information. */
+ private Map<String, IgniteBiTuple<Integer, Integer>> decimalInfo = new HashMap<>();
+
/**
* Creates an empty query entity.
*/
@@ -120,6 +127,8 @@ public class QueryEntity implements Serializable {
defaultFieldValues = other.defaultFieldValues != null ? new HashMap<>(other.defaultFieldValues)
: new HashMap<String, Object>();
+
+ decimalInfo = other.decimalInfo != null ? new HashMap<>(other.decimalInfo) : new HashMap<>();
}
/**
@@ -391,6 +400,27 @@ public class QueryEntity implements Serializable {
}
/**
+ * Gets set of field name to precision and scale.
+ *
+ * @return Set of names of fields that must have non-null values.
+ */
+ public Map<String, IgniteBiTuple<Integer, Integer>> getDecimalInfo() {
+ return unmodifiableMap(decimalInfo);
+ }
+
+ /**
+ * Sets decimal fields info.
+ *
+ * @param decimalInfo Set of name to precision and scale for decimal fields.
+ * @return {@code this} for chaining.
+ */
+ public QueryEntity setDecimalInfo(Map<String, IgniteBiTuple<Integer, Integer>> decimalInfo) {
+ this.decimalInfo = decimalInfo;
+
+ return this;
+ }
+
+ /**
* Gets fields default values.
*
* @return Field's name to default value map.
@@ -504,6 +534,9 @@ public class QueryEntity implements Serializable {
if (!F.isEmpty(desc.notNullFields()))
entity.setNotNullFields(desc.notNullFields());
+ if (!F.isEmpty(desc.decimalInfo()))
+ entity.setDecimalInfo(desc.decimalInfo());
+
return entity;
}
@@ -629,6 +662,9 @@ public class QueryEntity implements Serializable {
if (sqlAnn.notNull())
desc.addNotNullField(prop.fullName());
+ if (BigDecimal.class == fldCls && sqlAnn.precision() != -1 && sqlAnn.scale() != -1)
+ desc.addDecimalInfo(prop.fullName(), F.t(sqlAnn.precision(), sqlAnn.scale()));
+
if ((!F.isEmpty(sqlAnn.groups()) || !F.isEmpty(sqlAnn.orderedGroups()))
&& sqlAnn.inlineSize() != QueryIndex.DFLT_INLINE_SIZE) {
throw new CacheException("Inline size cannot be set on a field with group index [" +
@@ -670,13 +706,14 @@ public class QueryEntity implements Serializable {
F.eqNotOrdered(idxs, entity.idxs) &&
F.eq(tableName, entity.tableName) &&
F.eq(_notNullFields, entity._notNullFields) &&
- F.eq(defaultFieldValues, entity.defaultFieldValues);
+ F.eq(defaultFieldValues, entity.defaultFieldValues) &&
+ F.eq(decimalInfo, entity.decimalInfo);
}
/** {@inheritDoc} */
@Override public int hashCode() {
return Objects.hash(keyType, valType, keyFieldName, valueFieldName, fields, keyFields, aliases, idxs,
- tableName, _notNullFields, defaultFieldValues);
+ tableName, _notNullFields, defaultFieldValues, decimalInfo);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlField.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlField.java b/modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlField.java
index 0343474..35b7575 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlField.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlField.java
@@ -64,6 +64,20 @@ public @interface QuerySqlField {
boolean notNull() default false;
/**
+ * Specifies precision for a decimal field.
+ *
+ * @return precision for a decimal field.
+ */
+ int precision() default -1;
+
+ /**
+ * Specifies scale for a decimal field.
+ *
+ * @return scale for a decimal field.
+ */
+ int scale() default -1;
+
+ /**
* Array of index groups this field belongs to. Groups are used for compound indexes,
* whenever index should be created on more than one field. All fields within the same
* group will belong to the same index.
@@ -155,4 +169,4 @@ public @interface QuerySqlField {
*/
boolean descending() default false;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java
index e21cc4e..a50d78e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java
@@ -55,6 +55,7 @@ import org.apache.ignite.internal.binary.BinarySchema;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
import org.apache.ignite.internal.binary.streams.BinaryInputStream;
import org.apache.ignite.internal.binary.streams.BinaryOutputStream;
+import org.apache.ignite.lang.IgniteBiTuple;
/**
* Shared serialization/deserialization utils.
@@ -309,6 +310,8 @@ final class ClientUtils {
w.writeBoolean(qf.isKey());
w.writeBoolean(qf.isNotNull());
w.writeObject(qf.getDefaultValue());
+ w.writeInt(qf.getPrecision());
+ w.writeInt(qf.getScale());
}
);
ClientUtils.collection(
@@ -392,7 +395,9 @@ final class ClientUtils {
reader.readString(),
reader.readBoolean(),
reader.readBoolean(),
- reader.readObject()
+ reader.readObject(),
+ reader.readInt(),
+ reader.readInt()
)
);
@@ -494,6 +499,12 @@ final class ClientUtils {
/** Default value. */
private final Object dfltVal;
+ /** Precision. */
+ private final int precision;
+
+ /** Scale. */
+ private final int scale;
+
/** Serialization constructor. */
QueryField(QueryEntity e, Map.Entry<String, String> nameAndTypeName) {
name = nameAndTypeName.getKey();
@@ -502,19 +513,28 @@ final class ClientUtils {
Set<String> keys = e.getKeyFields();
Set<String> notNulls = e.getNotNullFields();
Map<String, Object> dflts = e.getDefaultFieldValues();
+ Map<String, IgniteBiTuple<Integer, Integer>> decimalInfo = e.getDecimalInfo();
isKey = keys != null && keys.contains(name);
isNotNull = notNulls != null && notNulls.contains(name);
dfltVal = dflts == null ? null : dflts.get(name);
+
+ IgniteBiTuple<Integer, Integer> precisionAndScale = decimalInfo == null ? null : decimalInfo.get(name);
+
+ precision = precisionAndScale == null? -1 : precisionAndScale.get1();
+ scale = precisionAndScale == null? -1 : precisionAndScale.get2();
}
/** Deserialization constructor. */
- public QueryField(String name, String typeName, boolean isKey, boolean isNotNull, Object dfltVal) {
+ public QueryField(String name, String typeName, boolean isKey, boolean isNotNull, Object dfltVal,
+ int precision, int scale) {
this.name = name;
this.typeName = typeName;
this.isKey = isKey;
this.isNotNull = isNotNull;
this.dfltVal = dfltVal;
+ this.precision = precision;
+ this.scale = scale;
}
/**
@@ -551,6 +571,20 @@ final class ClientUtils {
Object getDefaultValue() {
return dfltVal;
}
+
+ /**
+ * @return Precision.
+ */
+ public int getPrecision() {
+ return precision;
+ }
+
+ /**
+ * @return Scale.
+ */
+ public int getScale() {
+ return scale;
+ }
}
/** Thin client protocol cache configuration item codes. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinDatabaseMetadata.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinDatabaseMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinDatabaseMetadata.java
index dd8b733..7adc301 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinDatabaseMetadata.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinDatabaseMetadata.java
@@ -846,9 +846,9 @@ public class JdbcThinDatabaseMetadata implements DatabaseMetaData {
row.add(colMeta.columnName()); // 4. COLUMN_NAME
row.add(colMeta.dataType()); // 5. DATA_TYPE
row.add(colMeta.dataTypeName()); // 6. TYPE_NAME
- row.add((Integer)null); // 7. COLUMN_SIZE
+ row.add(colMeta.precision() == -1 ? null : colMeta.precision()); // 7. COLUMN_SIZE
row.add((Integer)null); // 8. BUFFER_LENGTH
- row.add((Integer)null); // 9. DECIMAL_DIGITS
+ row.add(colMeta.scale() == -1 ? null : colMeta.scale()); // 9. DECIMAL_DIGITS
row.add(10); // 10. NUM_PREC_RADIX
row.add(colMeta.isNullable() ? columnNullable : columnNoNulls); // 11. NULLABLE
row.add((String)null); // 12. REMARKS
@@ -1542,4 +1542,4 @@ public class JdbcThinDatabaseMetadata implements DatabaseMetaData {
@Override public boolean generatedKeyAlwaysReturned() throws SQLException {
return false;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryEntityTypeDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryEntityTypeDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryEntityTypeDescriptor.java
index fd0ef2b..9f8abc1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryEntityTypeDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryEntityTypeDescriptor.java
@@ -30,6 +30,7 @@ import org.apache.ignite.internal.processors.query.GridQueryIndexDescriptor;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteBiTuple;
/**
* Descriptor of type.
@@ -54,6 +55,9 @@ public class QueryEntityTypeDescriptor {
/** */
private Set<String> notNullFields = new HashSet<>();
+ /** Decimal fields information. */
+ private Map<String, IgniteBiTuple<Integer, Integer>> decimalInfo = new HashMap<>();
+
/** */
private QueryEntityIndexDescriptor fullTextIdx;
@@ -187,6 +191,16 @@ public class QueryEntityTypeDescriptor {
}
/**
+ * Adds decimal info.
+ *
+ * @param field Field.
+ * @param info Decimal column info.
+ */
+ public void addDecimalInfo(String field, IgniteBiTuple<Integer, Integer> info) {
+ decimalInfo.put(field, info);
+ }
+
+ /**
* @return notNull fields.
*/
public Set<String> notNullFields() {
@@ -194,6 +208,13 @@ public class QueryEntityTypeDescriptor {
}
/**
+ * @return Decimal info for fields.
+ */
+ public Map<String, IgniteBiTuple<Integer, Integer>> decimalInfo() {
+ return decimalInfo;
+ }
+
+ /**
* @return Class properties.
*/
public Map<String, QueryEntityClassProperty> properties() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMeta.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMeta.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMeta.java
index c0ac322..5b6304d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMeta.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMeta.java
@@ -135,6 +135,20 @@ public class JdbcColumnMeta implements JdbcRawBinarylizable {
}
/**
+ * @return Column's precision.
+ */
+ public int precision() {
+ return -1;
+ }
+
+ /**
+ * @return Column's scale.
+ */
+ public int scale() {
+ return -1;
+ }
+
+ /**
* Return 'nullable' flag in compatibility mode (according with column name and column type).
*
* @return {@code true} in case the column allows null values. Otherwise returns {@code false}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV4.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV4.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV4.java
new file mode 100644
index 0000000..ec76983
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV4.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.odbc.jdbc;
+
+import org.apache.ignite.internal.binary.BinaryReaderExImpl;
+import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * JDBC column metadata V4.
+ */
+public class JdbcColumnMetaV4 extends JdbcColumnMetaV3 {
+ /** Decimal field precision. */
+ private int precision;
+
+ /** Decimal field scale. */
+ private int scale;
+
+ /**
+ * Default constructor is used for serialization.
+ */
+ JdbcColumnMetaV4() {
+ // No-op.
+ }
+
+ /**
+ * @param schemaName Schema.
+ * @param tblName Table.
+ * @param colName Column.
+ * @param cls Type.
+ * @param nullable Allow nulls.
+ * @param dfltVal Default value.
+ * @param precision Decimal column precision.
+ * @param scale Decimal column scale.
+ */
+ public JdbcColumnMetaV4(String schemaName, String tblName, String colName, Class<?> cls, boolean nullable,
+ Object dfltVal, int precision, int scale) {
+ super(schemaName, tblName, colName, cls, nullable, dfltVal);
+
+ this.precision = precision;
+
+ this.scale = scale;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int precision() {
+ return precision;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int scale() {
+ return scale;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary(BinaryWriterExImpl writer) {
+ super.writeBinary(writer);
+
+ writer.writeInt(precision);
+ writer.writeInt(scale);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary(BinaryReaderExImpl reader) {
+ super.readBinary(reader);
+
+ precision = reader.readInt();
+ scale = reader.readInt();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(JdbcColumnMetaV4.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
index 2fe3b9c..ed37e0b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
@@ -47,8 +47,8 @@ public class JdbcConnectionContext implements ClientListenerConnectionContext {
/** Version 2.4.0: adds default values for columns feature. */
static final ClientListenerProtocolVersion VER_2_4_0 = ClientListenerProtocolVersion.create(2, 4, 0);
- /** Version 2.5.0. */
- private static final ClientListenerProtocolVersion VER_2_5_0 = ClientListenerProtocolVersion.create(2, 5, 0);
+ /** Version 2.5.0: adds precision and scale for columns feature. */
+ static final ClientListenerProtocolVersion VER_2_5_0 = ClientListenerProtocolVersion.create(2, 5, 0);
/** Current version. */
private static final ClientListenerProtocolVersion CURRENT_VER = VER_2_5_0;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsResultV4.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsResultV4.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsResultV4.java
new file mode 100644
index 0000000..9c8e2b7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsResultV4.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.odbc.jdbc;
+
+import java.util.Collection;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * JDBC columns metadata result.
+ */
+public class JdbcMetaColumnsResultV4 extends JdbcMetaColumnsResult {
+ /**
+ * Default constructor is used for deserialization.
+ */
+ JdbcMetaColumnsResultV4() {
+ super(META_COLUMNS_V4);
+ }
+
+ /**
+ * @param meta Columns metadata.
+ */
+ JdbcMetaColumnsResultV4(Collection<JdbcColumnMeta> meta) {
+ super(META_COLUMNS_V4, meta);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected JdbcColumnMeta createMetaColumn() {
+ return new JdbcColumnMetaV4();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(JdbcMetaColumnsResultV4.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
index cf0e98b..b9c9cdd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
@@ -67,6 +67,7 @@ import static org.apache.ignite.internal.processors.odbc.jdbc.JdbcBulkLoadBatchR
import static org.apache.ignite.internal.processors.odbc.jdbc.JdbcBulkLoadBatchRequest.CMD_FINISHED_ERROR;
import static org.apache.ignite.internal.processors.odbc.jdbc.JdbcConnectionContext.VER_2_3_0;
import static org.apache.ignite.internal.processors.odbc.jdbc.JdbcConnectionContext.VER_2_4_0;
+import static org.apache.ignite.internal.processors.odbc.jdbc.JdbcConnectionContext.VER_2_5_0;
import static org.apache.ignite.internal.processors.odbc.jdbc.JdbcRequest.BATCH_EXEC;
import static org.apache.ignite.internal.processors.odbc.jdbc.JdbcRequest.BULK_LOAD_BATCH;
import static org.apache.ignite.internal.processors.odbc.jdbc.JdbcRequest.META_COLUMNS;
@@ -744,7 +745,14 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
JdbcColumnMeta columnMeta;
- if (protocolVer.compareTo(VER_2_4_0) >= 0) {
+ if (protocolVer.compareTo(VER_2_5_0) >= 0) {
+ GridQueryProperty prop = table.property(colName);
+
+ columnMeta = new JdbcColumnMetaV4(table.schemaName(), table.tableName(),
+ field.getKey(), field.getValue(), !prop.notNull(), prop.defaultValue(),
+ prop.precision(), prop.scale());
+ }
+ else if (protocolVer.compareTo(VER_2_4_0) >= 0) {
GridQueryProperty prop = table.property(colName);
columnMeta = new JdbcColumnMetaV3(table.schemaName(), table.tableName(),
@@ -768,7 +776,9 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
JdbcMetaColumnsResult res;
- if (protocolVer.compareTo(VER_2_4_0) >= 0)
+ if (protocolVer.compareTo(VER_2_5_0) >= 0)
+ res = new JdbcMetaColumnsResultV4(meta);
+ else if (protocolVer.compareTo(VER_2_4_0) >= 0)
res = new JdbcMetaColumnsResultV3(meta);
else if (protocolVer.compareTo(VER_2_3_0) >= 0)
res = new JdbcMetaColumnsResultV2(meta);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResult.java
index 43631e9..4fea207 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResult.java
@@ -68,6 +68,9 @@ public class JdbcResult implements JdbcRawBinarylizable {
/** A request to send file from client to server. */
static final byte BULK_LOAD_ACK = 16;
+ /** Columns metadata result V4. */
+ static final byte META_COLUMNS_V4 = 17;
+
/** Success status. */
private byte type;
@@ -171,6 +174,11 @@ public class JdbcResult implements JdbcRawBinarylizable {
break;
+ case META_COLUMNS_V4:
+ res = new JdbcMetaColumnsResultV4();
+
+ break;
+
default:
throw new IgniteException("Unknown SQL listener request ID: [request ID=" + resId + ']');
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
index 8ebf09e..fa8e509 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
@@ -76,6 +76,8 @@ import org.apache.ignite.internal.processors.platform.cache.affinity.PlatformAff
import org.apache.ignite.internal.processors.platform.cache.expiry.PlatformExpiryPolicyFactory;
import org.apache.ignite.internal.processors.platform.events.PlatformLocalEventListener;
import org.apache.ignite.internal.processors.platform.plugin.cache.PlatformCachePluginConfiguration;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.platform.dotnet.PlatformDotNetAffinityFunction;
import org.apache.ignite.platform.dotnet.PlatformDotNetBinaryConfiguration;
@@ -492,6 +494,7 @@ public class PlatformConfigurationUtils {
Set<String> keyFields = new HashSet<>(cnt);
Set<String> notNullFields = new HashSet<>(cnt);
Map<String, Object> defVals = new HashMap<>(cnt);
+ Map<String, IgniteBiTuple<Integer, Integer>> decimalInfo = new HashMap<>(cnt);
if (cnt > 0) {
LinkedHashMap<String, String> fields = new LinkedHashMap<>(cnt);
@@ -511,6 +514,13 @@ public class PlatformConfigurationUtils {
Object defVal = in.readObject();
if (defVal != null)
defVals.put(fieldName, defVal);
+
+ int precision = in.readInt();
+
+ int scale = in.readInt();
+
+ if (precision != -1 || scale != -1)
+ decimalInfo.put(fieldName, F.t(precision, scale));
}
res.setFields(fields);
@@ -523,6 +533,9 @@ public class PlatformConfigurationUtils {
if (!defVals.isEmpty())
res.setDefaultFieldValues(defVals);
+
+ if (!decimalInfo.isEmpty())
+ res.setDecimalInfo(decimalInfo);
}
// Aliases
@@ -1012,6 +1025,7 @@ public class PlatformConfigurationUtils {
Set<String> keyFields = qryEntity.getKeyFields();
Set<String> notNullFields = qryEntity.getNotNullFields();
Map<String, Object> defVals = qryEntity.getDefaultFieldValues();
+ Map<String, IgniteBiTuple<Integer, Integer>> decimalInfo = qryEntity.getDecimalInfo();
writer.writeInt(fields.size());
@@ -1021,6 +1035,12 @@ public class PlatformConfigurationUtils {
writer.writeBoolean(keyFields != null && keyFields.contains(field.getKey()));
writer.writeBoolean(notNullFields != null && notNullFields.contains(field.getKey()));
writer.writeObject(defVals != null ? defVals.get(field.getKey()) : null);
+
+ IgniteBiTuple<Integer, Integer> precisionAndScale =
+ decimalInfo == null ? null : decimalInfo.get(field.getKey());
+
+ writer.writeInt(precisionAndScale == null ? -1 : precisionAndScale.get1());
+ writer.writeInt(precisionAndScale == null ? -1 : precisionAndScale.get2());
}
}
else
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 03e5254..a74548a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -2400,7 +2400,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
for (QueryField col : cols) {
try {
props.add(new QueryBinaryProperty(ctx, col.name(), null, Class.forName(col.typeName()),
- false, null, !col.isNullable(), null));
+ false, null, !col.isNullable(), null, -1, -1));
}
catch (ClassNotFoundException e) {
throw new SchemaOperationException("Class not found for new property: " + col.typeName());
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProperty.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProperty.java
index b258b7c..448c844 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProperty.java
@@ -77,4 +77,18 @@ public interface GridQueryProperty {
* @return {@code null} if a default value is not set for the property.
*/
public Object defaultValue();
+
+ /**
+ * Gets precision for this property.
+ *
+ * @return Precision for a decimal property or -1.
+ */
+ public int precision();
+
+ /**
+ * Gets scale for this property.
+ *
+ * @return Scale for a decimal property or -1.
+ */
+ public int scale();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryField.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryField.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryField.java
index 1a75ef1..882d816 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryField.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryField.java
@@ -40,13 +40,19 @@ public class QueryField implements Serializable {
/** Default value. */
private final Object dfltValue;
+ /** Precision. */
+ private final int precision;
+
+ /** Scale. */
+ private final int scale;
+
/**
* @param name Field name.
* @param typeName Class name for this field's values.
* @param nullable Nullable flag.
*/
public QueryField(String name, String typeName, boolean nullable) {
- this(name, typeName, nullable, null);
+ this(name, typeName, nullable, null, -1, -1);
}
/**
@@ -55,11 +61,13 @@ public class QueryField implements Serializable {
* @param nullable Nullable flag.
* @param dfltValue Default value.
*/
- public QueryField(String name, String typeName, boolean nullable, Object dfltValue) {
+ public QueryField(String name, String typeName, boolean nullable, Object dfltValue, int precision, int scale) {
this.name = name;
this.typeName = typeName;
this.nullable = nullable;
this.dfltValue = dfltValue;
+ this.precision = precision;
+ this.scale = scale;
}
/**
@@ -90,6 +98,20 @@ public class QueryField implements Serializable {
return dfltValue;
}
+ /**
+ * @return Precision.
+ */
+ public int precision() {
+ return precision;
+ }
+
+ /**
+ * @return Scale.
+ */
+ public int scale() {
+ return scale;
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(QueryField.class, this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
index 2ee37a1..12eacef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
@@ -61,6 +61,7 @@ import org.apache.ignite.internal.util.Jsr310Java8DateTimeApiUtils;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -264,6 +265,7 @@ public class QueryUtils {
normalEntity.setValueFieldName(entity.getValueFieldName());
normalEntity.setNotNullFields(entity.getNotNullFields());
normalEntity.setDefaultFieldValues(entity.getDefaultFieldValues());
+ normalEntity.setDecimalInfo(entity.getDecimalInfo());
// Normalize table name.
String normalTblName = entity.getTableName();
@@ -541,6 +543,7 @@ public class QueryUtils {
Set<String> keyFields = qryEntity.getKeyFields();
Set<String> notNulls = qryEntity.getNotNullFields();
Map<String, Object> dlftVals = qryEntity.getDefaultFieldValues();
+ Map<String, IgniteBiTuple<Integer, Integer>> decimalInfo = qryEntity.getDecimalInfo();
// We have to distinguish between empty and null keyFields when the key is not of SQL type -
// when a key is not of SQL type, absence of a field in nonnull keyFields tell us that this field
@@ -571,9 +574,14 @@ public class QueryUtils {
Object dfltVal = dlftVals != null ? dlftVals.get(entry.getKey()) : null;
+ IgniteBiTuple<Integer, Integer> precisionAndScale =
+ decimalInfo != null ? decimalInfo.get(entry.getKey()) : null;
+
QueryBinaryProperty prop = buildBinaryProperty(ctx, entry.getKey(),
U.classForName(entry.getValue(), Object.class, true),
- d.aliases(), isKeyField, notNull, dfltVal);
+ d.aliases(), isKeyField, notNull, dfltVal,
+ precisionAndScale != null ? precisionAndScale.get1() : -1,
+ precisionAndScale != null ? precisionAndScale.get2() : -1);
d.addProperty(prop, false);
}
@@ -717,11 +725,14 @@ public class QueryUtils {
* to key, {@code false} if it belongs to value, {@code null} if QueryEntity#keyFields is null.
* @param notNull {@code true} if {@code null} value is not allowed.
* @param dlftVal Default value.
+ * @param precision Precision.
+ * @param scale Scale.
* @return Binary property.
* @throws IgniteCheckedException On error.
*/
public static QueryBinaryProperty buildBinaryProperty(GridKernalContext ctx, String pathStr, Class<?> resType,
- Map<String, String> aliases, @Nullable Boolean isKeyField, boolean notNull, Object dlftVal) throws IgniteCheckedException {
+ Map<String, String> aliases, @Nullable Boolean isKeyField, boolean notNull, Object dlftVal,
+ int precision, int scale) throws IgniteCheckedException {
String[] path = pathStr.split("\\.");
QueryBinaryProperty res = null;
@@ -737,7 +748,8 @@ public class QueryUtils {
String alias = aliases.get(fullName.toString());
// The key flag that we've found out is valid for the whole path.
- res = new QueryBinaryProperty(ctx, prop, res, resType, isKeyField, alias, notNull, dlftVal);
+ res = new QueryBinaryProperty(ctx, prop, res, resType, isKeyField, alias, notNull, dlftVal,
+ precision, scale);
}
return res;
@@ -1400,5 +1412,15 @@ public class QueryUtils {
@Override public Object defaultValue() {
return null;
}
+
+ /** {@inheritDoc} */
+ @Override public int precision() {
+ return -1;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int scale() {
+ return -1;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryBinaryProperty.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryBinaryProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryBinaryProperty.java
index f440d12..7a47c2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryBinaryProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryBinaryProperty.java
@@ -71,6 +71,12 @@ public class QueryBinaryProperty implements GridQueryProperty {
/** */
private final Object defaultValue;
+ /** */
+ private final int precision;
+
+ /** */
+ private final int scale;
+
/**
* Constructor.
*
@@ -82,9 +88,12 @@ public class QueryBinaryProperty implements GridQueryProperty {
* @param alias Field alias.
* @param notNull {@code true} if null value is not allowed.
* @param defaultValue Default value.
+ * @param precision Precision.
+ * @param scale Scale.
*/
public QueryBinaryProperty(GridKernalContext ctx, String propName, QueryBinaryProperty parent,
- Class<?> type, @Nullable Boolean key, String alias, boolean notNull, Object defaultValue) {
+ Class<?> type, @Nullable Boolean key, String alias, boolean notNull, Object defaultValue,
+ int precision, int scale) {
this.ctx = ctx;
log = ctx.log(QueryBinaryProperty.class);
@@ -99,6 +108,8 @@ public class QueryBinaryProperty implements GridQueryProperty {
this.isKeyProp = key ? 1 : -1;
this.defaultValue = defaultValue;
+ this.precision = precision;
+ this.scale = scale;
}
/** {@inheritDoc} */
@@ -286,4 +297,14 @@ public class QueryBinaryProperty implements GridQueryProperty {
@Override public Object defaultValue() {
return defaultValue;
}
+
+ /** {@inheritDoc} */
+ @Override public int precision() {
+ return precision;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int scale() {
+ return scale;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryClassProperty.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryClassProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryClassProperty.java
index 575fe17..487ee5b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryClassProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryClassProperty.java
@@ -146,4 +146,14 @@ public class QueryClassProperty implements GridQueryProperty {
@Override public Object defaultValue() {
return null;
}
+
+ /** {@inheritDoc} */
+ @Override public int precision() {
+ return -1;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int scale() {
+ return -1;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
index bc5c1e0..f907138 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
@@ -71,6 +71,7 @@ import org.apache.ignite.internal.sql.command.SqlIndexColumn;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.plugin.security.SecurityPermission;
import org.h2.command.Prepared;
import org.h2.command.ddl.AlterTableAlterColumn;
@@ -80,6 +81,7 @@ import org.h2.command.ddl.DropIndex;
import org.h2.command.ddl.DropTable;
import org.h2.table.Column;
import org.h2.value.DataType;
+import org.h2.value.Value;
import static org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.UPDATE_RESULT_META;
import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser.PARAM_WRAP_VALUE;
@@ -420,7 +422,8 @@ public class DdlStatementsProcessor {
QueryField field = new QueryField(col.columnName(),
DataType.getTypeClassName(col.column().getType()),
- col.column().isNullable(), col.defaultValue());
+ col.column().isNullable(), col.defaultValue(),
+ col.precision(), col.scale());
cols.add(field);
@@ -595,6 +598,8 @@ public class DdlStatementsProcessor {
HashMap<String, Object> dfltValues = new HashMap<>();
+ Map<String, IgniteBiTuple<Integer, Integer>> decimalInfo = new HashMap<>();
+
for (Map.Entry<String, GridSqlColumn> e : createTbl.columns().entrySet()) {
GridSqlColumn gridCol = e.getValue();
@@ -613,11 +618,17 @@ public class DdlStatementsProcessor {
if (dfltVal != null)
dfltValues.put(e.getKey(), dfltVal);
+
+ if (col.getType() == Value.DECIMAL)
+ decimalInfo.put(e.getKey(), F.t((int)col.getPrecision(), col.getScale()));
}
if (!F.isEmpty(dfltValues))
res.setDefaultFieldValues(dfltValues);
+ if (!F.isEmpty(decimalInfo))
+ res.setDecimalInfo(decimalInfo);
+
String valTypeName = QueryUtils.createTableValueTypeName(createTbl.schemaName(), createTbl.tableName());
String keyTypeName = QueryUtils.createTableKeyTypeName(valTypeName);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java
index bc14ae2..0efbd4c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java
@@ -22,6 +22,7 @@ import org.apache.ignite.internal.util.typedef.F;
import org.h2.command.Parser;
import org.h2.expression.Expression;
import org.h2.table.Column;
+import org.h2.value.Value;
/**
* Column.
@@ -129,9 +130,23 @@ public class GridSqlColumn extends GridSqlElement {
}
/**
+ * @return Precision.
+ */
+ public int precision() {
+ return (int) col.getPrecision();
+ }
+
+ /**
+ * @return Scale.
+ */
+ public int scale() {
+ return col.getScale();
+ }
+
+ /**
* @return H2 Column.
*/
public Column column() {
return col;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/IgniteDecimalSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/IgniteDecimalSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/IgniteDecimalSelfTest.java
new file mode 100644
index 0000000..9e65276
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/IgniteDecimalSelfTest.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.index;
+
+import java.math.BigDecimal;
+import java.math.MathContext;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.jetbrains.annotations.NotNull;
+
+import static java.math.RoundingMode.HALF_UP;
+import static java.util.Arrays.asList;
+
+/**
+ * Test to check decimal columns.
+ */
+public class IgniteDecimalSelfTest extends AbstractSchemaSelfTest {
+ /** */
+ private static final int PRECISION = 9;
+
+ /** */
+ private static final int SCALE = 8;
+
+ /** */
+ private static final String DEC_TAB_NAME = "DECIMAL_TABLE";
+
+ /** */
+ private static final String VALUE = "VALUE";
+
+ /** */
+ private static final String SALARY_TAB_NAME = "SALARY";
+
+ /** */
+ private static final MathContext MATH_CTX = new MathContext(PRECISION);
+
+ /** */
+ private static final BigDecimal VAL_1 = new BigDecimal("123456789", MATH_CTX).setScale(SCALE, HALF_UP);
+
+ /** */
+ private static final BigDecimal VAL_2 = new BigDecimal("12345678.12345678", MATH_CTX).setScale(SCALE, HALF_UP);
+
+ /** */
+ private static final BigDecimal VAL_3 = new BigDecimal(".123456789", MATH_CTX).setScale(SCALE, HALF_UP);
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ IgniteEx grid = startGrid(0);
+
+ execute(grid, "CREATE TABLE " + DEC_TAB_NAME +
+ "(id LONG PRIMARY KEY, " + VALUE + " DECIMAL(" + PRECISION + ", " + SCALE + "))");
+
+ String insertQry = "INSERT INTO " + DEC_TAB_NAME + " VALUES (?, ?)";
+
+ execute(grid, insertQry, 1, VAL_1);
+ execute(grid, insertQry, 2, VAL_2);
+ execute(grid, insertQry, 3, VAL_3);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ CacheConfiguration<Integer, Salary> ccfg = cacheCfg(SALARY_TAB_NAME, "salary_cache");
+
+ cfg.setCacheConfiguration(ccfg);
+
+ return cfg;
+ }
+
+ /** */
+ @NotNull private CacheConfiguration<Integer, Salary> cacheCfg(String tabName, String cacheName) {
+ CacheConfiguration<Integer, Salary> ccfg = new CacheConfiguration<>(cacheName);
+
+ QueryEntity queryEntity = new QueryEntity(Integer.class.getName(), Salary.class.getName());
+
+ queryEntity.setTableName(tabName);
+
+ queryEntity.addQueryField("id", Integer.class.getName(), null);
+ queryEntity.addQueryField("amount", BigDecimal.class.getName(), null);
+
+ Map<String, IgniteBiTuple<Integer, Integer>> decimalInfo = new HashMap<>();
+
+ decimalInfo.put("amount", F.t(PRECISION, SCALE));
+
+ queryEntity.setDecimalInfo(decimalInfo);
+
+ ccfg.setQueryEntities(Collections.singletonList(queryEntity));
+
+ return ccfg;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConfiguredFromDdl() throws Exception {
+ checkDecimalInfo(DEC_TAB_NAME, VALUE, PRECISION, SCALE);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConfiguredFromQueryEntity() throws Exception {
+ checkDecimalInfo(SALARY_TAB_NAME, "amount", PRECISION, SCALE);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConfiguredFromQueryEntityInDynamicallyCreatedCache() throws Exception {
+ IgniteEx grid = grid(0);
+
+ String tabName = SALARY_TAB_NAME + "2";
+
+ CacheConfiguration<Integer, Salary> ccfg = cacheCfg(tabName, "SalaryCache-2");
+
+ IgniteCache<Integer, Salary> cache = grid.createCache(ccfg);
+
+ checkDecimalInfo(tabName, "amount", PRECISION, SCALE);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConfiguredFromAnnotations() throws Exception {
+ IgniteEx grid = grid(0);
+
+ CacheConfiguration<Integer, Salary> ccfg = new CacheConfiguration<>("SalaryCache-3");
+
+ ccfg.setIndexedTypes(Integer.class, SalaryWithAnnotations.class);
+
+ grid.createCache(ccfg);
+
+ checkDecimalInfo(SalaryWithAnnotations.class.getSimpleName().toUpperCase(), "amount", PRECISION, SCALE);
+ }
+
+ /** */
+ public void testSelectDecimal() throws Exception {
+ IgniteEx grid = grid(0);
+
+ List rows = execute(grid, "SELECT id, value FROM " + DEC_TAB_NAME + " order by id");
+
+ assertEquals(rows.size(), 3);
+
+ assertEquals(asList(1L, VAL_1), rows.get(0));
+ assertEquals(asList(2L, VAL_2), rows.get(1));
+ assertEquals(asList(3L, VAL_3), rows.get(2));
+ }
+
+ /** */
+ private void checkDecimalInfo(String tabName, String colName, Integer precision, Integer scale) {
+ QueryEntity queryEntity = findTableInfo(tabName);
+
+ assertNotNull(queryEntity);
+
+ Map<String, IgniteBiTuple<Integer, Integer>> decimalInfo = queryEntity.getDecimalInfo();
+
+ assertNotNull(decimalInfo);
+
+ IgniteBiTuple<Integer, Integer> columnInfo = decimalInfo.get(colName);
+
+ assertNotNull(columnInfo);
+
+ assertEquals(columnInfo.get1(), precision);
+ assertEquals(columnInfo.get2(), scale);
+ }
+
+ /**
+ * @param tabName Table name.
+ * @return QueryEntity of table.
+ */
+ private QueryEntity findTableInfo(String tabName) {
+ IgniteEx ignite = grid(0);
+
+ Collection<String> cacheNames = ignite.cacheNames();
+
+ for (String cacheName : cacheNames) {
+ CacheConfiguration ccfg = ignite.cache(cacheName).getConfiguration(CacheConfiguration.class);
+
+ Collection<QueryEntity> entities = ccfg.getQueryEntities();
+
+ for (QueryEntity entity : entities)
+ if (entity.getTableName().equalsIgnoreCase(tabName))
+ return entity;
+ }
+
+ return null;
+ }
+
+ /**
+ * Execute DDL statement on given node.
+ *
+ * @param node Node.
+ * @param sql Statement.
+ */
+ private List<List<?>> execute(Ignite node, String sql, Object... args) {
+ SqlFieldsQuery qry = new SqlFieldsQuery(sql)
+ .setArgs(args)
+ .setSchema("PUBLIC");
+
+ return queryProcessor(node).querySqlFields(qry, true).getAll();
+ }
+
+ /** */
+ private static class Salary {
+ /** */
+ private BigDecimal amount;
+
+ /** */
+ public BigDecimal getAmount() {
+ return amount;
+ }
+
+ /** */
+ public void setAmount(BigDecimal amount) {
+ this.amount = amount;
+ }
+ }
+
+ /** */
+ private static class SalaryWithAnnotations {
+ /** */
+ @QuerySqlField(index = true, precision = PRECISION, scale = SCALE)
+ private BigDecimal amount;
+
+ /** */
+ public BigDecimal getAmount() {
+ return amount;
+ }
+
+ /** */
+ public void setAmount(BigDecimal amount) {
+ this.amount = amount;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
index 1da695b..235b28b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
@@ -565,6 +565,16 @@ public abstract class GridIndexingSpiAbstractSelfTest extends GridCommonAbstract
@Override public Object defaultValue() {
return null;
}
+
+ /** */
+ @Override public int precision() {
+ return -1;
+ }
+
+ /** */
+ @Override public int scale() {
+ return -1;
+ }
};
}
@@ -783,4 +793,4 @@ public abstract class GridIndexingSpiAbstractSelfTest extends GridCommonAbstract
return false;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index e10fff1..619e7cf 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -108,6 +108,7 @@ import org.apache.ignite.internal.processors.cache.index.DynamicIndexServerCoord
import org.apache.ignite.internal.processors.cache.index.DynamicIndexServerNodeFIlterBasicSelfTest;
import org.apache.ignite.internal.processors.cache.index.DynamicIndexServerNodeFilterCoordinatorBasicSelfTest;
import org.apache.ignite.internal.processors.cache.index.H2ConnectionLeaksSelfTest;
+import org.apache.ignite.internal.processors.cache.index.IgniteDecimalSelfTest;
import org.apache.ignite.internal.processors.cache.index.H2DynamicColumnsClientBasicSelfTest;
import org.apache.ignite.internal.processors.cache.index.H2DynamicColumnsServerBasicSelfTest;
import org.apache.ignite.internal.processors.cache.index.H2DynamicColumnsServerCoordinatorBasicSelfTest;
@@ -397,6 +398,7 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
suite.addTestSuite(OptimizedMarshallerIndexNameTest.class);
suite.addTestSuite(IgniteSqlDefaultValueTest.class);
+ suite.addTestSuite(IgniteDecimalSelfTest.class);
// H2 Rows on-heap cache
suite.addTestSuite(H2RowCacheSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/QueryEntityConfigurationParityTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/QueryEntityConfigurationParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/QueryEntityConfigurationParityTest.cs
index a9830d8..ba10cda 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/QueryEntityConfigurationParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/QueryEntityConfigurationParityTest.cs
@@ -32,7 +32,8 @@ namespace Apache.Ignite.Core.Tests.ApiParity
"findValueType",
"KeyFields",
"NotNullFields",
- "DefaultFieldValues"
+ "DefaultFieldValues",
+ "DecimalInfo"
};
/// <summary>
@@ -47,4 +48,4 @@ namespace Apache.Ignite.Core.Tests.ApiParity
UnneededProperties);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs
index 6834d5d..9d59a50 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs
@@ -551,6 +551,8 @@ namespace Apache.Ignite.Core.Tests.Cache
Assert.AreEqual(x.IsKeyField, y.IsKeyField);
Assert.AreEqual(x.NotNull, y.NotNull);
Assert.AreEqual(x.DefaultValue, y.DefaultValue);
+ Assert.AreEqual(x.Precision, y.Precision);
+ Assert.AreEqual(x.Scale, y.Scale);
}
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QueryEntity.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QueryEntity.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QueryEntity.cs
index 8855535..32173ba 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QueryEntity.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QueryEntity.cs
@@ -458,7 +458,9 @@ namespace Apache.Ignite.Core.Cache.Configuration
{
IsKeyField = isKey,
NotNull = attr.NotNull,
- DefaultValue = attr.DefaultValue
+ DefaultValue = attr.DefaultValue,
+ Precision = attr.Precision,
+ Scale = attr.Scale
});
ScanAttributes(memberInfo.Value, fields, indexes, columnName, visitedTypes, isKey);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QueryField.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QueryField.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QueryField.cs
index a96f1bf..869ce7d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QueryField.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QueryField.cs
@@ -85,6 +85,8 @@ namespace Apache.Ignite.Core.Cache.Configuration
IsKeyField = reader.ReadBoolean();
NotNull = reader.ReadBoolean();
DefaultValue = reader.ReadObject<object>();
+ Precision = reader.ReadInt();
+ Scale = reader.ReadInt();
}
/// <summary>
@@ -99,6 +101,8 @@ namespace Apache.Ignite.Core.Cache.Configuration
writer.WriteBoolean(IsKeyField);
writer.WriteBoolean(NotNull);
writer.WriteObject(DefaultValue);
+ writer.WriteInt(Precision);
+ writer.WriteInt(Scale);
}
/// <summary>
@@ -154,6 +158,16 @@ namespace Apache.Ignite.Core.Cache.Configuration
public object DefaultValue { get; set; }
/// <summary>
+ /// Gets or sets the precision for the field.
+ /// </summary>
+ public int Precision { get; set; }
+
+ /// <summary>
+ /// Gets or sets the scale for the field.
+ /// </summary>
+ public int Scale { get; set; }
+
+ /// <summary>
/// Validates this instance and outputs information to the log, if necessary.
/// </summary>
internal void Validate(ILogger log, string logInfo)
@@ -179,4 +193,4 @@ namespace Apache.Ignite.Core.Cache.Configuration
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QuerySqlFieldAttribute.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QuerySqlFieldAttribute.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QuerySqlFieldAttribute.cs
index dd131f9..bfd3575 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QuerySqlFieldAttribute.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/QuerySqlFieldAttribute.cs
@@ -81,5 +81,15 @@ namespace Apache.Ignite.Core.Cache.Configuration
/// Gets or sets the default value for the field (has effect when inserting with DML).
/// </summary>
public object DefaultValue { get; set; }
+
+ /// <summary>
+ /// Gets or sets the precision for the field.
+ /// </summary>
+ public int Precision { get; set; }
+
+ /// <summary>
+ /// Gets or sets the scale for the field.
+ /// </summary>
+ public int Scale { get; set; }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6df5f99b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd
index a97136f..8707272 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd
@@ -340,6 +340,16 @@
<xs:documentation>Indicates whether null value is allowed for the field.</xs:documentation>
</xs:annotation>
</xs:attribute>
+ <xs:attribute name="precision" type="xs:int">
+ <xs:annotation>
+ <xs:documentation>Precision of field</xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute name="scale" type="xs:int">
+ <xs:annotation>
+ <xs:documentation>Scale of field</xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
</xs:complexType>
</xs:element>
</xs:sequence>
[30/54] [abbrv] ignite git commit: IGNITE-7830: Knn Lin Reg with new
datasets
Posted by ag...@apache.org.
IGNITE-7830: Knn Lin Reg with new datasets
this closes #3583
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a4653b7c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a4653b7c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a4653b7c
Branch: refs/heads/ignite-6083
Commit: a4653b7c1287a039206bf22e9d85125bb15bc412
Parents: 6557fe6
Author: zaleslaw <za...@gmail.com>
Authored: Wed Apr 11 12:31:48 2018 +0300
Committer: YuriBabak <y....@gmail.com>
Committed: Wed Apr 11 12:31:48 2018 +0300
----------------------------------------------------------------------
.../java/org/apache/ignite/ml/knn/KNNUtils.java | 59 ++++++++
.../KNNClassificationTrainer.java | 23 +--
.../ml/knn/regression/KNNRegressionModel.java | 87 +++++++++++
.../ml/knn/regression/KNNRegressionTrainer.java | 40 ++++++
.../ignite/ml/knn/regression/package-info.java | 22 +++
.../apache/ignite/ml/knn/KNNRegressionTest.java | 143 +++++++++++++++++++
.../org/apache/ignite/ml/knn/KNNTestSuite.java | 1 +
7 files changed, 354 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4653b7c/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNUtils.java
new file mode 100644
index 0000000..88fa70f
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNUtils.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.knn;
+
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.PartitionDataBuilder;
+import org.apache.ignite.ml.knn.partitions.KNNPartitionContext;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.structures.LabeledDataset;
+import org.apache.ignite.ml.structures.LabeledVector;
+import org.apache.ignite.ml.structures.partition.LabeledDatasetPartitionDataBuilderOnHeap;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Helper class for KNNRegression.
+ */
+public class KNNUtils {
+ /**
+ * Builds dataset.
+ *
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @return Dataset.
+ */
+ @Nullable public static <K, V> Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> buildDataset(DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+ PartitionDataBuilder<K, V, KNNPartitionContext, LabeledDataset<Double, LabeledVector>> partDataBuilder
+ = new LabeledDatasetPartitionDataBuilderOnHeap<>(
+ featureExtractor,
+ lbExtractor
+ );
+
+ Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> dataset = null;
+
+ if (datasetBuilder != null) {
+ dataset = datasetBuilder.build(
+ (upstream, upstreamSize) -> new KNNPartitionContext(),
+ partDataBuilder
+ );
+ }
+ return dataset;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4653b7c/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java
index 357047f..c0c8e65 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java
@@ -17,14 +17,9 @@
package org.apache.ignite.ml.knn.classification;
-import org.apache.ignite.ml.dataset.Dataset;
import org.apache.ignite.ml.dataset.DatasetBuilder;
-import org.apache.ignite.ml.dataset.PartitionDataBuilder;
-import org.apache.ignite.ml.knn.partitions.KNNPartitionContext;
+import org.apache.ignite.ml.knn.KNNUtils;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.structures.LabeledDataset;
-import org.apache.ignite.ml.structures.partition.LabeledDatasetPartitionDataBuilderOnHeap;
-import org.apache.ignite.ml.structures.LabeledVector;
import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
/**
@@ -41,20 +36,6 @@ public class KNNClassificationTrainer implements SingleLabelDatasetTrainer<KNNCl
*/
@Override public <K, V> KNNClassificationModel fit(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
- PartitionDataBuilder<K, V, KNNPartitionContext, LabeledDataset<Double, LabeledVector>> partDataBuilder
- = new LabeledDatasetPartitionDataBuilderOnHeap<>(
- featureExtractor,
- lbExtractor
- );
-
- Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> dataset = null;
-
- if (datasetBuilder != null) {
- dataset = datasetBuilder.build(
- (upstream, upstreamSize) -> new KNNPartitionContext(),
- partDataBuilder
- );
- }
- return new KNNClassificationModel<>(dataset);
+ return new KNNClassificationModel<>(KNNUtils.buildDataset(datasetBuilder, featureExtractor, lbExtractor));
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4653b7c/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionModel.java
new file mode 100644
index 0000000..cabc143
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionModel.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.knn.regression;
+
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
+import org.apache.ignite.ml.knn.partitions.KNNPartitionContext;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
+import org.apache.ignite.ml.structures.LabeledDataset;
+import org.apache.ignite.ml.structures.LabeledVector;
+
+import java.util.List;
+
+/**
+ * This class provides kNN Multiple Linear Regression or Locally [weighted] regression (Simple and Weighted versions).
+ *
+ * <p> This is an instance-based learning method. </p>
+ *
+ * <ul>
+ * <li>Local means using nearby points (i.e. a nearest neighbors approach).</li>
+ * <li>Weighted means we value points based upon how far away they are.</li>
+ * <li>Regression means approximating a function.</li>
+ * </ul>
+ */
+public class KNNRegressionModel<K,V> extends KNNClassificationModel<K,V> {
+ /**
+ * Builds the model via prepared dataset.
+ * @param dataset Specially prepared object to run algorithm over it.
+ */
+ public KNNRegressionModel(Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> dataset) {
+ super(dataset);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double apply(Vector v) {
+ List<LabeledVector> neighbors = findKNearestNeighbors(v);
+
+ return predictYBasedOn(neighbors, v);
+ }
+
+ /** */
+ private double predictYBasedOn(List<LabeledVector> neighbors, Vector v) {
+ switch (stgy) {
+ case SIMPLE:
+ return simpleRegression(neighbors);
+ case WEIGHTED:
+ return weightedRegression(neighbors, v);
+ default:
+ throw new UnsupportedOperationException("Strategy " + stgy.name() + " is not supported");
+ }
+ }
+
+ /** */
+ private double weightedRegression(List<LabeledVector> neighbors, Vector v) {
+ double sum = 0.0;
+ double div = 0.0;
+ for (LabeledVector<Vector, Double> neighbor : neighbors) {
+ double distance = distanceMeasure.compute(v, neighbor.features());
+ sum += neighbor.label() * distance;
+ div += distance;
+ }
+ return sum / div;
+ }
+
+ /** */
+ private double simpleRegression(List<LabeledVector> neighbors) {
+ double sum = 0.0;
+ for (LabeledVector<Vector, Double> neighbor : neighbors)
+ sum += neighbor.label();
+ return sum / (double)k;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4653b7c/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java
new file mode 100644
index 0000000..2d13cd5
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.knn.regression;
+
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.knn.KNNUtils;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+
+/**
+ * kNN algorithm trainer to solve regression task.
+ */
+public class KNNRegressionTrainer{
+ /**
+ * Trains model based on the specified data.
+ *
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @return Model.
+ */
+ public <K, V> KNNRegressionModel fit(DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+ return new KNNRegressionModel<>(KNNUtils.buildDataset(datasetBuilder, featureExtractor, lbExtractor));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4653b7c/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/package-info.java
new file mode 100644
index 0000000..82e7192
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains helper classes for kNN regression algorithms.
+ */
+package org.apache.ignite.ml.knn.regression;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4653b7c/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
new file mode 100644
index 0000000..66dbca9
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.knn;
+
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.knn.classification.KNNStrategy;
+import org.apache.ignite.ml.knn.regression.KNNRegressionModel;
+import org.apache.ignite.ml.knn.regression.KNNRegressionTrainer;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.distances.EuclideanDistance;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.junit.Assert;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Tests for {@link KNNRegressionTrainer}.
+ */
+public class KNNRegressionTest extends BaseKNNTest {
+ /** */
+ private double[] y;
+
+ /** */
+ private double[][] x;
+
+ /** */
+ public void testSimpleRegressionWithOneNeighbour() {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ Map<Integer, double[]> data = new HashMap<>();
+ data.put(0, new double[] {11.0, 0, 0, 0, 0, 0});
+ data.put(1, new double[] {12.0, 2.0, 0, 0, 0, 0});
+ data.put(2, new double[] {13.0, 0, 3.0, 0, 0, 0});
+ data.put(3, new double[] {14.0, 0, 0, 4.0, 0, 0});
+ data.put(4, new double[] {15.0, 0, 0, 0, 5.0, 0});
+ data.put(5, new double[] {16.0, 0, 0, 0, 0, 6.0});
+
+ KNNRegressionTrainer trainer = new KNNRegressionTrainer();
+
+ KNNRegressionModel knnMdl = (KNNRegressionModel) trainer.fit(
+ new LocalDatasetBuilder<>(data, 2),
+ (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+ (k, v) -> v[0]
+ ).withK(1)
+ .withDistanceMeasure(new EuclideanDistance())
+ .withStrategy(KNNStrategy.SIMPLE);
+
+ Vector vector = new DenseLocalOnHeapVector(new double[] {0, 0, 0, 5.0, 0.0});
+ System.out.println(knnMdl.apply(vector));
+ Assert.assertEquals(15, knnMdl.apply(vector), 1E-12);
+ }
+
+ /** */
+ public void testLongly() {
+
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ Map<Integer, double[]> data = new HashMap<>();
+ data.put(0, new double[] {60323, 83.0, 234289, 2356, 1590, 107608, 1947});
+ data.put(1, new double[] {61122, 88.5, 259426, 2325, 1456, 108632, 1948});
+ data.put(2, new double[] {60171, 88.2, 258054, 3682, 1616, 109773, 1949});
+ data.put(3, new double[] {61187, 89.5, 284599, 3351, 1650, 110929, 1950});
+ data.put(4, new double[] {63221, 96.2, 328975, 2099, 3099, 112075, 1951});
+ data.put(5, new double[] {63639, 98.1, 346999, 1932, 3594, 113270, 1952});
+ data.put(6, new double[] {64989, 99.0, 365385, 1870, 3547, 115094, 1953});
+ data.put(7, new double[] {63761, 100.0, 363112, 3578, 3350, 116219, 1954});
+ data.put(8, new double[] {66019, 101.2, 397469, 2904, 3048, 117388, 1955});
+ data.put(9, new double[] {68169, 108.4, 442769, 2936, 2798, 120445, 1957});
+ data.put(10, new double[] {66513, 110.8, 444546, 4681, 2637, 121950, 1958});
+ data.put(11, new double[] {68655, 112.6, 482704, 3813, 2552, 123366, 1959});
+ data.put(12, new double[] {69564, 114.2, 502601, 3931, 2514, 125368, 1960});
+ data.put(13, new double[] {69331, 115.7, 518173, 4806, 2572, 127852, 1961});
+ data.put(14, new double[] {70551, 116.9, 554894, 4007, 2827, 130081, 1962});
+
+ KNNRegressionTrainer trainer = new KNNRegressionTrainer();
+
+ KNNRegressionModel knnMdl = (KNNRegressionModel) trainer.fit(
+ new LocalDatasetBuilder<>(data, 2),
+ (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+ (k, v) -> v[0]
+ ).withK(3)
+ .withDistanceMeasure(new EuclideanDistance())
+ .withStrategy(KNNStrategy.SIMPLE);
+
+ Vector vector = new DenseLocalOnHeapVector(new double[] {104.6, 419180, 2822, 2857, 118734, 1956});
+ System.out.println(knnMdl.apply(vector));
+ Assert.assertEquals(67857, knnMdl.apply(vector), 2000);
+ }
+
+ /** */
+ public void testLonglyWithWeightedStrategy() {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ Map<Integer, double[]> data = new HashMap<>();
+ data.put(0, new double[] {60323, 83.0, 234289, 2356, 1590, 107608, 1947});
+ data.put(1, new double[] {61122, 88.5, 259426, 2325, 1456, 108632, 1948});
+ data.put(2, new double[] {60171, 88.2, 258054, 3682, 1616, 109773, 1949});
+ data.put(3, new double[] {61187, 89.5, 284599, 3351, 1650, 110929, 1950});
+ data.put(4, new double[] {63221, 96.2, 328975, 2099, 3099, 112075, 1951});
+ data.put(5, new double[] {63639, 98.1, 346999, 1932, 3594, 113270, 1952});
+ data.put(6, new double[] {64989, 99.0, 365385, 1870, 3547, 115094, 1953});
+ data.put(7, new double[] {63761, 100.0, 363112, 3578, 3350, 116219, 1954});
+ data.put(8, new double[] {66019, 101.2, 397469, 2904, 3048, 117388, 1955});
+ data.put(9, new double[] {68169, 108.4, 442769, 2936, 2798, 120445, 1957});
+ data.put(10, new double[] {66513, 110.8, 444546, 4681, 2637, 121950, 1958});
+ data.put(11, new double[] {68655, 112.6, 482704, 3813, 2552, 123366, 1959});
+ data.put(12, new double[] {69564, 114.2, 502601, 3931, 2514, 125368, 1960});
+ data.put(13, new double[] {69331, 115.7, 518173, 4806, 2572, 127852, 1961});
+ data.put(14, new double[] {70551, 116.9, 554894, 4007, 2827, 130081, 1962});
+
+ KNNRegressionTrainer trainer = new KNNRegressionTrainer();
+
+ KNNRegressionModel knnMdl = (KNNRegressionModel) trainer.fit(
+ new LocalDatasetBuilder<>(data, 2),
+ (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+ (k, v) -> v[0]
+ ).withK(3)
+ .withDistanceMeasure(new EuclideanDistance())
+ .withStrategy(KNNStrategy.SIMPLE);
+
+ Vector vector = new DenseLocalOnHeapVector(new double[] {104.6, 419180, 2822, 2857, 118734, 1956});
+ System.out.println(knnMdl.apply(vector));
+ Assert.assertEquals(67857, knnMdl.apply(vector), 2000);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4653b7c/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNTestSuite.java
index 95ebec5..55ef24e 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNTestSuite.java
@@ -26,6 +26,7 @@ import org.junit.runners.Suite;
@RunWith(Suite.class)
@Suite.SuiteClasses({
KNNClassificationTest.class,
+ KNNRegressionTest.class,
LabeledDatasetTest.class
})
public class KNNTestSuite {
[47/54] [abbrv] ignite git commit: IGNITE-8135: SQL: authentication
for CREATE TABLE and DROP TABLE commands. This closes #3801.
Posted by ag...@apache.org.
IGNITE-8135: SQL: authentication for CREATE TABLE and DROP TABLE commands. This closes #3801.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a57c9e1f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a57c9e1f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a57c9e1f
Branch: refs/heads/ignite-6083
Commit: a57c9e1f2a00fea310de5eabba92a642942b9796
Parents: 7a1d0ea
Author: devozerov <vo...@gridgain.com>
Authored: Thu Apr 12 15:02:57 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Apr 12 15:02:57 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/client/ClientException.java | 3 +-
.../internal/client/thin/ClientQueryCursor.java | 6 ++-
.../platform/client/ClientRequestHandler.java | 7 ++-
.../cache/ClientCacheSqlFieldsQueryRequest.java | 19 +++++--
.../security/SecurityContextHolder.java | 53 ++++++++++++++++++++
.../query/h2/ddl/DdlStatementsProcessor.java | 9 ++++
6 files changed, 91 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a57c9e1f/modules/core/src/main/java/org/apache/ignite/client/ClientException.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/client/ClientException.java b/modules/core/src/main/java/org/apache/ignite/client/ClientException.java
index 0555635..b0d9f6c 100644
--- a/modules/core/src/main/java/org/apache/ignite/client/ClientException.java
+++ b/modules/core/src/main/java/org/apache/ignite/client/ClientException.java
@@ -20,7 +20,7 @@ package org.apache.ignite.client;
/**
* Common thin client checked exception.
*/
-public class ClientException extends Exception {
+public class ClientException extends RuntimeException {
/** Serial version uid. */
private static final long serialVersionUID = 0L;
@@ -28,6 +28,7 @@ public class ClientException extends Exception {
* Constructs a new exception with {@code null} as its detail message.
*/
public ClientException() {
+ // No-op.
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/a57c9e1f/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientQueryCursor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientQueryCursor.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientQueryCursor.java
index 9367cfd..086fab8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientQueryCursor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientQueryCursor.java
@@ -54,6 +54,7 @@ class ClientQueryCursor<T> implements QueryCursor<T> {
pager.close();
}
catch (Exception ignored) {
+ // No-op.
}
}
@@ -76,7 +77,10 @@ class ClientQueryCursor<T> implements QueryCursor<T> {
currPageIt = currPage.iterator();
}
catch (ClientException e) {
- throw new RuntimeException("Failed to retrieve query results", e);
+ throw e;
+ }
+ catch (Exception e) {
+ throw new ClientException("Failed to retrieve query results", e);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a57c9e1f/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java
index faa50bc..5ed0d38 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java
@@ -22,6 +22,7 @@ import org.apache.ignite.internal.processors.authentication.AuthorizationContext
import org.apache.ignite.internal.processors.odbc.ClientListenerRequest;
import org.apache.ignite.internal.processors.odbc.ClientListenerRequestHandler;
import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
+import org.apache.ignite.internal.processors.security.SecurityContextHolder;
/**
* Thin client request handler.
@@ -47,8 +48,10 @@ public class ClientRequestHandler implements ClientListenerRequestHandler {
/** {@inheritDoc} */
@Override public ClientListenerResponse handle(ClientListenerRequest req) {
- if (authCtx != null)
+ if (authCtx != null) {
AuthorizationContext.context(authCtx);
+ SecurityContextHolder.set(ctx.securityContext());
+ }
try {
return ((ClientRequest)req).process(ctx);
@@ -56,6 +59,8 @@ public class ClientRequestHandler implements ClientListenerRequestHandler {
finally {
if (authCtx != null)
AuthorizationContext.clear();
+
+ SecurityContextHolder.clear();
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a57c9e1f/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
index 3aa95bf..53f6353 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
@@ -28,8 +28,11 @@ import org.apache.ignite.internal.processors.odbc.jdbc.JdbcStatementType;
import org.apache.ignite.internal.processors.platform.cache.PlatformCache;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.internal.processors.platform.client.ClientStatus;
+import org.apache.ignite.internal.processors.platform.client.IgniteClientException;
import org.apache.ignite.internal.processors.query.QueryUtils;
-import org.apache.ignite.plugin.security.SecurityPermission;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.plugin.security.SecurityException;
/**
* Sql query request.
@@ -95,7 +98,7 @@ public class ClientCacheSqlFieldsQueryRequest extends ClientCacheRequest {
if (qry.getSchema() == null) {
String schema = QueryUtils.normalizeSchemaName(desc.cacheName(),
- desc.cacheConfiguration().getSqlSchema());
+ desc.cacheConfiguration().getSqlSchema());
qry.setSchema(schema);
}
@@ -108,7 +111,7 @@ public class ClientCacheSqlFieldsQueryRequest extends ClientCacheRequest {
FieldsQueryCursor cur = curs.get(0);
ClientCacheFieldsQueryCursor cliCur = new ClientCacheFieldsQueryCursor(
- cur, qry.getPageSize(), ctx);
+ cur, qry.getPageSize(), ctx);
long cursorId = ctx.resources().put(cliCur);
@@ -119,6 +122,16 @@ public class ClientCacheSqlFieldsQueryRequest extends ClientCacheRequest {
catch (Exception e) {
ctx.decrementCursors();
+ SecurityException securityEx = X.cause(e, SecurityException.class);
+
+ if (securityEx != null) {
+ throw new IgniteClientException(
+ ClientStatus.SECURITY_VIOLATION,
+ "Client is not authorized to perform this operation",
+ securityEx
+ );
+ }
+
throw e;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a57c9e1f/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContextHolder.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContextHolder.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContextHolder.java
new file mode 100644
index 0000000..14d70c9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContextHolder.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.security;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Thread-local security context.
+ */
+public class SecurityContextHolder {
+ /** Context. */
+ private static final ThreadLocal<SecurityContext> CTX = new ThreadLocal<>();
+
+ /**
+ * Get security context.
+ *
+ * @return Security context.
+ */
+ @Nullable public static SecurityContext get() {
+ return CTX.get();
+ }
+
+ /**
+ * Set security context.
+ *
+ * @param ctx Context.
+ */
+ public static void set(@Nullable SecurityContext ctx) {
+ CTX.set(ctx);
+ }
+
+ /**
+ * Clear security context.
+ */
+ public static void clear() {
+ set(null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a57c9e1f/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
index b148969..bc5c1e0 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
@@ -34,6 +34,8 @@ import org.apache.ignite.cache.query.FieldsQueryCursor;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.authentication.AuthorizationContext;
+import org.apache.ignite.internal.processors.authentication.UserManagementOperation;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
@@ -56,6 +58,8 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlDropTable;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlStatement;
import org.apache.ignite.internal.processors.query.schema.SchemaOperationException;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.internal.processors.security.SecurityContextHolder;
import org.apache.ignite.internal.sql.command.SqlAlterTableCommand;
import org.apache.ignite.internal.sql.command.SqlAlterUserCommand;
import org.apache.ignite.internal.sql.command.SqlCommand;
@@ -67,6 +71,7 @@ import org.apache.ignite.internal.sql.command.SqlIndexColumn;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.security.SecurityPermission;
import org.h2.command.Prepared;
import org.h2.command.ddl.AlterTableAlterColumn;
import org.h2.command.ddl.CreateIndex;
@@ -316,6 +321,8 @@ public class DdlStatementsProcessor {
}
}
else if (stmt0 instanceof GridSqlCreateTable) {
+ ctx.security().authorize(null, SecurityPermission.CACHE_CREATE, SecurityContextHolder.get());
+
GridSqlCreateTable cmd = (GridSqlCreateTable)stmt0;
if (!F.eq(QueryUtils.DFLT_SCHEMA, cmd.schemaName()))
@@ -349,6 +356,8 @@ public class DdlStatementsProcessor {
}
}
else if (stmt0 instanceof GridSqlDropTable) {
+ ctx.security().authorize(null, SecurityPermission.CACHE_DESTROY, SecurityContextHolder.get());
+
GridSqlDropTable cmd = (GridSqlDropTable)stmt0;
if (!F.eq(QueryUtils.DFLT_SCHEMA, cmd.schemaName()))
[17/54] [abbrv] ignite git commit: IGNITE-8059: Integrate decision
tree with partition based dataset.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasure.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasure.java
new file mode 100644
index 0000000..3fc8515
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasure.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.mse;
+
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Mean squared error (variance) impurity measure which is calculated the following way:
+ * {@code \frac{1}{L}\sum_{i=0}^{n}(y_i - \mu)^2}.
+ */
+public class MSEImpurityMeasure implements ImpurityMeasure<MSEImpurityMeasure> {
+ /** */
+ private static final long serialVersionUID = 4536394578628409689L;
+
+ /** Sum of all elements in the left part. */
+ private final double leftY;
+
+ /** Sum of all squared elements in the left part. */
+ private final double leftY2;
+
+ /** Number of elements in the left part. */
+ private final long leftCnt;
+
+ /** Sum of all elements in the right part. */
+ private final double rightY;
+
+ /** Sum of all squared elements in the right part. */
+ private final double rightY2;
+
+ /** Number of elements in the right part. */
+ private final long rightCnt;
+
+ /**
+ * Constructs a new instance of mean squared error (variance) impurity measure.
+ *
+ * @param leftY Sum of all elements in the left part.
+ * @param leftY2 Sum of all squared elements in the left part.
+ * @param leftCnt Number of elements in the left part.
+ * @param rightY Sum of all elements in the right part.
+ * @param rightY2 Sum of all squared elements in the right part.
+ * @param rightCnt Number of elements in the right part.
+ */
+ public MSEImpurityMeasure(double leftY, double leftY2, long leftCnt, double rightY, double rightY2, long rightCnt) {
+ this.leftY = leftY;
+ this.leftY2 = leftY2;
+ this.leftCnt = leftCnt;
+ this.rightY = rightY;
+ this.rightY2 = rightY2;
+ this.rightCnt = rightCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double impurity() {
+ double impurity = 0;
+
+ if (leftCnt > 0)
+ impurity += leftY2 - 2.0 * leftY / leftCnt * leftY + Math.pow(leftY / leftCnt, 2) * leftCnt;
+
+ if (rightCnt > 0)
+ impurity += rightY2 - 2.0 * rightY / rightCnt * rightY + Math.pow(rightY / rightCnt, 2) * rightCnt;
+
+ return impurity;
+ }
+
+ /** {@inheritDoc} */
+ @Override public MSEImpurityMeasure add(MSEImpurityMeasure b) {
+ return new MSEImpurityMeasure(
+ leftY + b.leftY,
+ leftY2 + b.leftY2,
+ leftCnt + b.leftCnt,
+ rightY + b.rightY,
+ rightY2 + b.rightY2,
+ rightCnt + b.rightCnt
+ );
+ }
+
+ /** {@inheritDoc} */
+ @Override public MSEImpurityMeasure subtract(MSEImpurityMeasure b) {
+ return new MSEImpurityMeasure(
+ leftY - b.leftY,
+ leftY2 - b.leftY2,
+ leftCnt - b.leftCnt,
+ rightY - b.rightY,
+ rightY2 - b.rightY2,
+ rightCnt - b.rightCnt
+ );
+ }
+
+ /** */
+ public double getLeftY() {
+ return leftY;
+ }
+
+ /** */
+ public double getLeftY2() {
+ return leftY2;
+ }
+
+ /** */
+ public long getLeftCnt() {
+ return leftCnt;
+ }
+
+ /** */
+ public double getRightY() {
+ return rightY;
+ }
+
+ /** */
+ public double getRightY2() {
+ return rightY2;
+ }
+
+ /** */
+ public long getRightCnt() {
+ return rightCnt;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculator.java
new file mode 100644
index 0000000..cb5019c
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculator.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.mse;
+
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+
+/**
+ * Meas squared error (variance) impurity measure calculator.
+ */
+public class MSEImpurityMeasureCalculator implements ImpurityMeasureCalculator<MSEImpurityMeasure> {
+ /** */
+ private static final long serialVersionUID = 288747414953756824L;
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public StepFunction<MSEImpurityMeasure>[] calculate(DecisionTreeData data) {
+ double[][] features = data.getFeatures();
+ double[] labels = data.getLabels();
+
+ if (features.length > 0) {
+ StepFunction<MSEImpurityMeasure>[] res = new StepFunction[features[0].length];
+
+ for (int col = 0; col < res.length; col++) {
+ data.sort(col);
+
+ double[] x = new double[features.length + 1];
+ MSEImpurityMeasure[] y = new MSEImpurityMeasure[features.length + 1];
+
+ x[0] = Double.NEGATIVE_INFINITY;
+
+ for (int leftSize = 0; leftSize <= features.length; leftSize++) {
+ double leftY = 0;
+ double leftY2 = 0;
+ double rightY = 0;
+ double rightY2 = 0;
+
+ for (int i = 0; i < leftSize; i++) {
+ leftY += labels[i];
+ leftY2 += Math.pow(labels[i], 2);
+ }
+
+ for (int i = leftSize; i < features.length; i++) {
+ rightY += labels[i];
+ rightY2 += Math.pow(labels[i], 2);
+ }
+
+ if (leftSize < features.length)
+ x[leftSize + 1] = features[leftSize][col];
+
+ y[leftSize] = new MSEImpurityMeasure(
+ leftY, leftY2, leftSize, rightY, rightY2, features.length - leftSize
+ );
+ }
+
+ res[col] = new StepFunction<>(x, y);
+ }
+
+ return res;
+ }
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/package-info.java
new file mode 100644
index 0000000..23ec4e0
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains mean squared error impurity measure and calculator.
+ */
+package org.apache.ignite.ml.tree.impurity.mse;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/package-info.java
new file mode 100644
index 0000000..4155593
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Root package for decision tree impurity measures and calculators.
+ */
+package org.apache.ignite.ml.tree.impurity;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressor.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressor.java
new file mode 100644
index 0000000..2418571
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressor.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Simple step function compressor.
+ *
+ * @param <T> Type of step function values.
+ */
+public class SimpleStepFunctionCompressor<T extends ImpurityMeasure<T>> implements StepFunctionCompressor<T> {
+ /** */
+ private static final long serialVersionUID = -3231787633598409157L;
+
+ /** Min size of step function to be compressed. */
+ private final int minSizeToBeCompressed;
+
+ /** In case of compression min impurity increase that will be recorded. */
+ private final double minImpurityIncreaseForRecord;
+
+ /** In case of compression min impurity decrease that will be recorded. */
+ private final double minImpurityDecreaseForRecord;
+
+ /**
+ * Constructs a new instance of simple step function compressor with default parameters.
+ */
+ public SimpleStepFunctionCompressor() {
+ this(10, 0.1, 0.05);
+ }
+
+ /**
+ * Constructs a new instance of simple step function compressor.
+ *
+ * @param minSizeToBeCompressed Min size of step function to be compressed.
+ * @param minImpurityIncreaseForRecord In case of compression min impurity increase that will be recorded.
+ * @param minImpurityDecreaseForRecord In case of compression min impurity decrease that will be recorded.
+ */
+ public SimpleStepFunctionCompressor(int minSizeToBeCompressed, double minImpurityIncreaseForRecord,
+ double minImpurityDecreaseForRecord) {
+ this.minSizeToBeCompressed = minSizeToBeCompressed;
+ this.minImpurityIncreaseForRecord = minImpurityIncreaseForRecord;
+ this.minImpurityDecreaseForRecord = minImpurityDecreaseForRecord;
+ }
+
+ /** {@inheritDoc} */
+ @Override public StepFunction<T> compress(StepFunction<T> function) {
+ double[] arguments = function.getX();
+ T[] values = function.getY();
+
+ if (arguments.length >= minSizeToBeCompressed) {
+ List<StepFunctionPoint> points = new ArrayList<>();
+
+ for (int i = 0; i < arguments.length; i++)
+ points.add(new StepFunctionPoint(arguments[i], values[i]));
+
+ points = compress(points);
+
+ double[] resX = new double[points.size()];
+ T[] resY = Arrays.copyOf(values, points.size());
+
+ for (int i = 0; i < points.size(); i++) {
+ StepFunctionPoint pnt = points.get(i);
+ resX[i] = pnt.x;
+ resY[i] = pnt.y;
+ }
+
+ return new StepFunction<>(resX, resY);
+ }
+
+ return function;
+ }
+
+ /**
+ * Compresses list of step function points.
+ *
+ * @param points Step function points.
+ * @return Compressed step function points.
+ */
+ private List<StepFunctionPoint> compress(List<StepFunctionPoint> points) {
+ List<StepFunctionPoint> res = new ArrayList<>();
+
+ double minImpurity = Double.MAX_VALUE, maxImpurity = Double.MIN_VALUE;
+ for (int i = 0; i < points.size(); i++) {
+ StepFunctionPoint pnt = points.get(i);
+
+ double impurity = pnt.y.impurity();
+
+ if (impurity > maxImpurity)
+ maxImpurity = impurity;
+
+ if (impurity < minImpurity)
+ minImpurity = impurity;
+ }
+
+ Double prev = null;
+ for (StepFunctionPoint pnt : points) {
+ double impurity = (pnt.y.impurity() - minImpurity) / (maxImpurity - minImpurity);
+ if (prev == null ||
+ prev - impurity >= minImpurityDecreaseForRecord ||
+ impurity - prev >= minImpurityIncreaseForRecord) {
+ prev = impurity;
+ res.add(pnt);
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Util class that represents step function point.
+ */
+ private class StepFunctionPoint {
+ /** Argument of the step start. */
+ private final double x;
+
+ /** Value of the step. */
+ private final T y;
+
+ /**
+ * Constructs a new instance of util class that represents step function point.
+ *
+ * @param x Argument of the step start.
+ * @param y Value of the step.
+ */
+ StepFunctionPoint(double x, T y) {
+ this.x = x;
+ this.y = y;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunction.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunction.java
new file mode 100644
index 0000000..431503d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunction.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import java.util.Arrays;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Step function described by {@code x} and {@code y} points.
+ *
+ * @param <T> Type of function values.
+ */
+public class StepFunction<T extends ImpurityMeasure<T>> {
+ /** Argument of every steps start. Should be ascendingly sorted all the time. */
+ private final double[] x;
+
+ /** Value of every step. */
+ private final T[] y;
+
+ /**
+ * Constructs a new instance of step function.
+ *
+ * @param x Argument of every steps start.
+ * @param y Value of every step.
+ */
+ public StepFunction(double[] x, T[] y) {
+ assert x.length == y.length : "Argument and value arrays have to be the same length";
+
+ this.x = x;
+ this.y = y;
+
+ sort(x, y, 0, x.length - 1);
+ }
+
+ /**
+ * Adds the given step function to this.
+ *
+ * @param b Another step function.
+ * @return Sum of this and the given function.
+ */
+ public StepFunction<T> add(StepFunction<T> b) {
+ int resSize = 0, leftPtr = 0, rightPtr = 0;
+ double previousPnt = 0;
+
+ while (leftPtr < x.length || rightPtr < b.x.length) {
+ if (rightPtr >= b.x.length || (leftPtr < x.length && x[leftPtr] < b.x[rightPtr])) {
+ if (resSize == 0 || x[leftPtr] != previousPnt) {
+ previousPnt = x[leftPtr];
+ resSize++;
+ }
+
+ leftPtr++;
+ }
+ else {
+ if (resSize == 0 || b.x[rightPtr] != previousPnt) {
+ previousPnt = b.x[rightPtr];
+ resSize++;
+ }
+
+ rightPtr++;
+ }
+ }
+
+ double[] resX = new double[resSize];
+ T[] resY = Arrays.copyOf(y, resSize);
+
+ leftPtr = 0;
+ rightPtr = 0;
+
+ for (int i = 0; leftPtr < x.length || rightPtr < b.x.length; i++) {
+ if (rightPtr >= b.x.length || (leftPtr < x.length && x[leftPtr] < b.x[rightPtr])) {
+ boolean override = i > 0 && x[leftPtr] == resX[i - 1];
+ int target = override ? i - 1 : i;
+
+ resY[target] = override ? resY[target] : null;
+ resY[target] = i > 0 ? resY[i - 1] : null;
+ resY[target] = resY[target] == null ? y[leftPtr] : resY[target].add(y[leftPtr]);
+
+ if (leftPtr > 0)
+ resY[target] = resY[target].subtract(y[leftPtr - 1]);
+
+ resX[target] = x[leftPtr];
+ i = target;
+
+ leftPtr++;
+ }
+ else {
+ boolean override = i > 0 && b.x[rightPtr] == resX[i - 1];
+ int target = override ? i - 1 : i;
+
+ resY[target] = override ? resY[target] : null;
+ resY[target] = i > 0 ? resY[i - 1] : null;
+
+ resY[target] = resY[target] == null ? b.y[rightPtr] : resY[target].add(b.y[rightPtr]);
+
+ if (rightPtr > 0)
+ resY[target] = resY[target].subtract(b.y[rightPtr - 1]);
+
+ resX[target] = b.x[rightPtr];
+ i = target;
+
+ rightPtr++;
+ }
+ }
+
+ return new StepFunction<>(resX, resY);
+ }
+
+ /** */
+ private void sort(double[] x, T[] y, int from, int to) {
+ if (from < to) {
+ double pivot = x[(from + to) / 2];
+
+ int i = from, j = to;
+ while (i <= j) {
+ while (x[i] < pivot) i++;
+ while (x[j] > pivot) j--;
+
+ if (i <= j) {
+ double tmpX = x[i];
+ x[i] = x[j];
+ x[j] = tmpX;
+
+ T tmpY = y[i];
+ y[i] = y[j];
+ y[j] = tmpY;
+
+ i++;
+ j--;
+ }
+ }
+
+ sort(x, y, from, j);
+ sort(x, y, i, to);
+ }
+ }
+
+ /** */
+ public double[] getX() {
+ return x;
+ }
+
+ /** */
+ public T[] getY() {
+ return y;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionCompressor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionCompressor.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionCompressor.java
new file mode 100644
index 0000000..41baa29
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionCompressor.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Base interface for step function compressors which reduces step function size.
+ *
+ * @param <T> Type of step function value.
+ */
+public interface StepFunctionCompressor<T extends ImpurityMeasure<T>> extends Serializable {
+ /**
+ * Compresses the given step function.
+ *
+ * @param function Step function.
+ * @return Compressed step function.
+ */
+ public StepFunction<T> compress(StepFunction<T> function);
+
+ /**
+ * Compresses every step function in the given array.
+ *
+ * @param functions Array of step functions.
+ * @return Arrays of compressed step function.
+ */
+ default public StepFunction<T>[] compress(StepFunction<T>[] functions) {
+ if (functions == null)
+ return null;
+
+ StepFunction<T>[] res = Arrays.copyOf(functions, functions.length);
+
+ for (int i = 0; i < res.length; i++)
+ res[i] = compress(res[i]);
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/package-info.java
new file mode 100644
index 0000000..99df618
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains util classes used in decision tree impurity calculators.
+ */
+package org.apache.ignite.ml.tree.impurity.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/DecisionTreeLeafBuilder.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/DecisionTreeLeafBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/DecisionTreeLeafBuilder.java
new file mode 100644
index 0000000..976e30d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/DecisionTreeLeafBuilder.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.leaf;
+
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.DecisionTreeLeafNode;
+import org.apache.ignite.ml.tree.TreeFilter;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+
+/**
+ * Base interface for decision tree leaf builders.
+ */
+public interface DecisionTreeLeafBuilder {
+ /**
+ * Creates new leaf node for given dataset and node predicate.
+ *
+ * @param dataset Dataset.
+ * @param pred Node predicate.
+ * @return Leaf node.
+ */
+ public DecisionTreeLeafNode createLeafNode(Dataset<EmptyContext, DecisionTreeData> dataset, TreeFilter pred);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MeanDecisionTreeLeafBuilder.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MeanDecisionTreeLeafBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MeanDecisionTreeLeafBuilder.java
new file mode 100644
index 0000000..2e05215
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MeanDecisionTreeLeafBuilder.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.leaf;
+
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.DecisionTreeLeafNode;
+import org.apache.ignite.ml.tree.TreeFilter;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+
+/**
+ * Decision tree leaf node builder that chooses mean value as a leaf value.
+ */
+public class MeanDecisionTreeLeafBuilder implements DecisionTreeLeafBuilder {
+ /** {@inheritDoc} */
+ @Override public DecisionTreeLeafNode createLeafNode(Dataset<EmptyContext, DecisionTreeData> dataset,
+ TreeFilter pred) {
+ double[] aa = dataset.compute(part -> {
+ double mean = 0;
+ int cnt = 0;
+
+ for (int i = 0; i < part.getFeatures().length; i++) {
+ if (pred.test(part.getFeatures()[i])) {
+ mean += part.getLabels()[i];
+ cnt++;
+ }
+ }
+
+ if (cnt != 0) {
+ mean = mean / cnt;
+
+ return new double[] {mean, cnt};
+ }
+
+ return null;
+ }, this::reduce);
+
+ return aa != null ? new DecisionTreeLeafNode(aa[0]) : null;
+ }
+
+ /** */
+ private double[] reduce(double[] a, double[] b) {
+ if (a == null)
+ return b;
+ else if (b == null)
+ return a;
+ else {
+ double aMean = a[0];
+ double aCnt = a[1];
+ double bMean = b[0];
+ double bCnt = b[1];
+
+ double mean = (aMean * aCnt + bMean * bCnt) / (aCnt + bCnt);
+
+ return new double[] {mean, aCnt + bCnt};
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MostCommonDecisionTreeLeafBuilder.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MostCommonDecisionTreeLeafBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MostCommonDecisionTreeLeafBuilder.java
new file mode 100644
index 0000000..1e8b941
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MostCommonDecisionTreeLeafBuilder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.leaf;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.DecisionTreeLeafNode;
+import org.apache.ignite.ml.tree.TreeFilter;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+
+/**
+ * Decision tree leaf node builder that chooses most common value as a leaf node value.
+ */
+public class MostCommonDecisionTreeLeafBuilder implements DecisionTreeLeafBuilder {
+ /** {@inheritDoc} */
+ @Override public DecisionTreeLeafNode createLeafNode(Dataset<EmptyContext, DecisionTreeData> dataset,
+ TreeFilter pred) {
+ Map<Double, Integer> cnt = dataset.compute(part -> {
+
+ if (part.getFeatures() != null) {
+ Map<Double, Integer> map = new HashMap<>();
+
+ for (int i = 0; i < part.getFeatures().length; i++) {
+ if (pred.test(part.getFeatures()[i])) {
+ double lb = part.getLabels()[i];
+
+ if (map.containsKey(lb))
+ map.put(lb, map.get(lb) + 1);
+ else
+ map.put(lb, 1);
+ }
+ }
+
+ return map;
+ }
+
+ return null;
+ }, this::reduce);
+
+ double bestVal = 0;
+ int bestCnt = -1;
+
+ for (Map.Entry<Double, Integer> e : cnt.entrySet()) {
+ if (e.getValue() > bestCnt) {
+ bestCnt = e.getValue();
+ bestVal = e.getKey();
+ }
+ }
+
+ return new DecisionTreeLeafNode(bestVal);
+ }
+
+ /** */
+ private Map<Double, Integer> reduce(Map<Double, Integer> a, Map<Double, Integer> b) {
+ if (a == null)
+ return b;
+ else if (b == null)
+ return a;
+ else {
+ for (Map.Entry<Double, Integer> e : b.entrySet()) {
+ if (a.containsKey(e.getKey()))
+ a.put(e.getKey(), a.get(e.getKey()) + e.getValue());
+ else
+ a.put(e.getKey(), e.getValue());
+ }
+ return a;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/package-info.java
new file mode 100644
index 0000000..26ec67d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Root package for decision trees leaf builders.
+ */
+package org.apache.ignite.ml.tree.leaf;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/tree/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/package-info.java
new file mode 100644
index 0000000..660f3f3
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Root package for decision trees.
+ */
+package org.apache.ignite.ml.tree;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalRegionInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalRegionInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalRegionInfo.java
deleted file mode 100644
index 3ae474e..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalRegionInfo.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.BitSet;
-
-/**
- * Information about categorical region.
- */
-public class CategoricalRegionInfo extends RegionInfo implements Externalizable {
- /**
- * Bitset representing categories of this region.
- */
- private BitSet cats;
-
- /**
- * @param impurity Impurity of region.
- * @param cats Bitset representing categories of this region.
- */
- public CategoricalRegionInfo(double impurity, BitSet cats) {
- super(impurity);
-
- this.cats = cats;
- }
-
- /**
- * No-op constructor for serialization/deserialization.
- */
- public CategoricalRegionInfo() {
- // No-op
- }
-
- /**
- * Get bitset representing categories of this region.
- *
- * @return Bitset representing categories of this region.
- */
- public BitSet cats() {
- return cats;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- super.writeExternal(out);
- out.writeObject(cats);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- super.readExternal(in);
- cats = (BitSet)in.readObject();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalSplitInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalSplitInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalSplitInfo.java
deleted file mode 100644
index 94cb1e8..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalSplitInfo.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.BitSet;
-import org.apache.ignite.ml.trees.nodes.CategoricalSplitNode;
-import org.apache.ignite.ml.trees.nodes.SplitNode;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-
-/**
- * Information about split of categorical feature.
- *
- * @param <D> Class representing information of left and right subregions.
- */
-public class CategoricalSplitInfo<D extends RegionInfo> extends SplitInfo<D> {
- /** Bitset indicating which vectors are assigned to left subregion. */
- private final BitSet bs;
-
- /**
- * @param regionIdx Index of region which is split.
- * @param leftData Data of left subregion.
- * @param rightData Data of right subregion.
- * @param bs Bitset indicating which vectors are assigned to left subregion.
- */
- public CategoricalSplitInfo(int regionIdx, D leftData, D rightData,
- BitSet bs) {
- super(regionIdx, leftData, rightData);
- this.bs = bs;
- }
-
- /** {@inheritDoc} */
- @Override public SplitNode createSplitNode(int featureIdx) {
- return new CategoricalSplitNode(featureIdx, bs);
- }
-
- /**
- * Get bitset indicating which vectors are assigned to left subregion.
- */
- public BitSet bitSet() {
- return bs;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "CategoricalSplitInfo [" +
- "infoGain=" + infoGain +
- ", regionIdx=" + regionIdx +
- ", leftData=" + leftData +
- ", bs=" + bs +
- ", rightData=" + rightData +
- ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousRegionInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousRegionInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousRegionInfo.java
deleted file mode 100644
index e98bb72..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousRegionInfo.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-
-/**
- * Information about region used by continuous features.
- */
-public class ContinuousRegionInfo extends RegionInfo {
- /**
- * Count of samples in this region.
- */
- private int size;
-
- /**
- * @param impurity Impurity of the region.
- * @param size Size of this region
- */
- public ContinuousRegionInfo(double impurity, int size) {
- super(impurity);
- this.size = size;
- }
-
- /**
- * No-op constructor for serialization/deserialization.
- */
- public ContinuousRegionInfo() {
- // No-op
- }
-
- /**
- * Get the size of region.
- */
- public int getSize() {
- return size;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "ContinuousRegionInfo [" +
- "size=" + size +
- ']';
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- super.writeExternal(out);
- out.writeInt(size);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- super.readExternal(in);
- size = in.readInt();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousSplitCalculator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousSplitCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousSplitCalculator.java
deleted file mode 100644
index 3a0e9da..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousSplitCalculator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.ContinuousFeatureProcessor;
-
-/**
- * This class is used for calculation of best split by continuous feature.
- *
- * @param <C> Class in which information about region will be stored.
- */
-public interface ContinuousSplitCalculator<C extends ContinuousRegionInfo> {
- /**
- * Calculate region info 'from scratch'.
- *
- * @param s Stream of labels in this region.
- * @param l Index of sample projection on this feature in array sorted by this projection value and intervals
- * bitsets. ({@link ContinuousFeatureProcessor}).
- * @return Region info.
- */
- C calculateRegionInfo(DoubleStream s, int l);
-
- /**
- * Calculate split info of best split of region given information about this region.
- *
- * @param sampleIndexes Indexes of samples of this region.
- * @param values All values of this feature.
- * @param labels All labels of this feature.
- * @param regionIdx Index of region being split.
- * @param data Information about region being split which can be used for computations.
- * @return Information about best split of region with index given by regionIdx.
- */
- SplitInfo<C> splitRegion(Integer[] sampleIndexes, double[] values, double[] labels, int regionIdx, C data);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/RegionInfo.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/RegionInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/RegionInfo.java
deleted file mode 100644
index 8ec7db3..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/RegionInfo.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-
-/** Class containing information about region. */
-public class RegionInfo implements Externalizable {
- /** Impurity in this region. */
- private double impurity;
-
- /**
- * @param impurity Impurity of this region.
- */
- public RegionInfo(double impurity) {
- this.impurity = impurity;
- }
-
- /**
- * No-op constructor for serialization/deserialization.
- */
- public RegionInfo() {
- // No-op
- }
-
- /**
- * Get impurity in this region.
- *
- * @return Impurity of this region.
- */
- public double impurity() {
- return impurity;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeDouble(impurity);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- impurity = in.readDouble();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/DecisionTreeModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/DecisionTreeModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/DecisionTreeModel.java
deleted file mode 100644
index 572e64a..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/DecisionTreeModel.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.models;
-
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.trees.nodes.DecisionTreeNode;
-
-/**
- * Model for decision tree.
- */
-public class DecisionTreeModel implements Model<Vector, Double> {
- /** Root node of the decision tree. */
- private final DecisionTreeNode root;
-
- /**
- * Construct decision tree model.
- *
- * @param root Root of decision tree.
- */
- public DecisionTreeModel(DecisionTreeNode root) {
- this.root = root;
- }
-
- /** {@inheritDoc} */
- @Override public Double apply(Vector val) {
- return root.process(val);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/package-info.java
deleted file mode 100644
index ce8418e..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains decision tree models.
- */
-package org.apache.ignite.ml.trees.models;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/CategoricalSplitNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/CategoricalSplitNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/CategoricalSplitNode.java
deleted file mode 100644
index cae6d4a..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/CategoricalSplitNode.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import java.util.BitSet;
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Split node by categorical feature.
- */
-public class CategoricalSplitNode extends SplitNode {
- /** Bitset specifying which categories belong to left subregion. */
- private final BitSet bs;
-
- /**
- * Construct categorical split node.
- *
- * @param featureIdx Index of feature by which split is done.
- * @param bs Bitset specifying which categories go to the left subtree.
- */
- public CategoricalSplitNode(int featureIdx, BitSet bs) {
- super(featureIdx);
- this.bs = bs;
- }
-
- /** {@inheritDoc} */
- @Override public boolean goLeft(Vector v) {
- return bs.get((int)v.getX(featureIdx));
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "CategoricalSplitNode [bs=" + bs + ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/ContinuousSplitNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/ContinuousSplitNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/ContinuousSplitNode.java
deleted file mode 100644
index 285cfcd..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/ContinuousSplitNode.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Split node representing split of continuous feature.
- */
-public class ContinuousSplitNode extends SplitNode {
- /** Threshold. Values which are less or equal then threshold are assigned to the left subregion. */
- private final double threshold;
-
- /**
- * Construct ContinuousSplitNode by threshold and feature index.
- *
- * @param threshold Threshold.
- * @param featureIdx Feature index.
- */
- public ContinuousSplitNode(double threshold, int featureIdx) {
- super(featureIdx);
- this.threshold = threshold;
- }
-
- /** {@inheritDoc} */
- @Override public boolean goLeft(Vector v) {
- return v.getX(featureIdx) <= threshold;
- }
-
- /** Threshold. Values which are less or equal then threshold are assigned to the left subregion. */
- public double threshold() {
- return threshold;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "ContinuousSplitNode [" +
- "threshold=" + threshold +
- ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/DecisionTreeNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/DecisionTreeNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/DecisionTreeNode.java
deleted file mode 100644
index d31623d..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/DecisionTreeNode.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Node of decision tree.
- */
-public interface DecisionTreeNode {
- /**
- * Assign the double value to the given vector.
- *
- * @param v Vector.
- * @return Value assigned to the given vector.
- */
- double process(Vector v);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/Leaf.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/Leaf.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/Leaf.java
deleted file mode 100644
index 79b441f..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/Leaf.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Terminal node of the decision tree.
- */
-public class Leaf implements DecisionTreeNode {
- /**
- * Value in subregion represented by this node.
- */
- private final double val;
-
- /**
- * Construct the leaf of decision tree.
- *
- * @param val Value in subregion represented by this node.
- */
- public Leaf(double val) {
- this.val = val;
- }
-
- /**
- * Return value in subregion represented by this node.
- *
- * @param v Vector.
- * @return Value in subregion represented by this node.
- */
- @Override public double process(Vector v) {
- return val;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/SplitNode.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/SplitNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/SplitNode.java
deleted file mode 100644
index 4c258d1..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/SplitNode.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Node in decision tree representing a split.
- */
-public abstract class SplitNode implements DecisionTreeNode {
- /** Left subtree. */
- protected DecisionTreeNode l;
-
- /** Right subtree. */
- protected DecisionTreeNode r;
-
- /** Feature index. */
- protected final int featureIdx;
-
- /**
- * Constructs SplitNode with a given feature index.
- *
- * @param featureIdx Feature index.
- */
- public SplitNode(int featureIdx) {
- this.featureIdx = featureIdx;
- }
-
- /**
- * Indicates if the given vector is in left subtree.
- *
- * @param v Vector
- * @return Status of given vector being left subtree.
- */
- abstract boolean goLeft(Vector v);
-
- /**
- * Left subtree.
- *
- * @return Left subtree.
- */
- public DecisionTreeNode left() {
- return l;
- }
-
- /**
- * Right subtree.
- *
- * @return Right subtree.
- */
- public DecisionTreeNode right() {
- return r;
- }
-
- /**
- * Set the left subtree.
- *
- * @param n left subtree.
- */
- public void setLeft(DecisionTreeNode n) {
- l = n;
- }
-
- /**
- * Set the right subtree.
- *
- * @param n right subtree.
- */
- public void setRight(DecisionTreeNode n) {
- r = n;
- }
-
- /**
- * Delegates processing to subtrees.
- *
- * @param v Vector.
- * @return Value assigned to the given vector.
- */
- @Override public double process(Vector v) {
- if (left() != null && goLeft(v))
- return left().process(v);
- else
- return right().process(v);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/package-info.java
deleted file mode 100644
index d6deb9d..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains classes representing decision tree nodes.
- */
-package org.apache.ignite.ml.trees.nodes;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/package-info.java
deleted file mode 100644
index b07ba4a..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains decision tree algorithms.
- */
-package org.apache.ignite.ml.trees;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndex.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndex.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndex.java
deleted file mode 100644
index 0d27c8a..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndex.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-
-/**
- * Class representing a simple index in 2d matrix in the form (row, col).
- */
-public class BiIndex implements Externalizable {
- /** Row. */
- private int row;
-
- /** Column. */
- @AffinityKeyMapped
- private int col;
-
- /**
- * No-op constructor for serialization/deserialization.
- */
- public BiIndex() {
- // No-op.
- }
-
- /**
- * Construct BiIndex from row and column.
- *
- * @param row Row.
- * @param col Column.
- */
- public BiIndex(int row, int col) {
- this.row = row;
- this.col = col;
- }
-
- /**
- * Returns row.
- *
- * @return Row.
- */
- public int row() {
- return row;
- }
-
- /**
- * Returns column.
- *
- * @return Column.
- */
- public int col() {
- return col;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- BiIndex idx = (BiIndex)o;
-
- if (row != idx.row)
- return false;
- return col == idx.col;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = row;
- res = 31 * res + col;
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "BiIndex [" +
- "row=" + row +
- ", col=" + col +
- ']';
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- out.writeInt(row);
- out.writeInt(col);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- row = in.readInt();
- col = in.readInt();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndexedCacheColumnDecisionTreeTrainerInput.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndexedCacheColumnDecisionTreeTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndexedCacheColumnDecisionTreeTrainerInput.java
deleted file mode 100644
index 04281fb..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndexedCacheColumnDecisionTreeTrainerInput.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.util.Map;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.lang.IgniteBiTuple;
-
-/**
- * Adapter for column decision tree trainer for bi-indexed cache.
- */
-public class BiIndexedCacheColumnDecisionTreeTrainerInput extends CacheColumnDecisionTreeTrainerInput<BiIndex, Double> {
- /**
- * Construct an input for {@link ColumnDecisionTreeTrainer}.
- *
- * @param cache Bi-indexed cache.
- * @param catFeaturesInfo Information about categorical feature in the form (feature index -> number of
- * categories).
- * @param samplesCnt Count of samples.
- * @param featuresCnt Count of features.
- */
- public BiIndexedCacheColumnDecisionTreeTrainerInput(IgniteCache<BiIndex, Double> cache,
- Map<Integer, Integer> catFeaturesInfo, int samplesCnt, int featuresCnt) {
- super(cache,
- () -> IntStream.range(0, samplesCnt).mapToObj(s -> new BiIndex(s, featuresCnt)),
- e -> Stream.of(new IgniteBiTuple<>(e.getKey().row(), e.getValue())),
- DoubleStream::of,
- fIdx -> IntStream.range(0, samplesCnt).mapToObj(s -> new BiIndex(s, fIdx)),
- catFeaturesInfo,
- featuresCnt,
- samplesCnt);
- }
-
- /** {@inheritDoc} */
- @Override public Object affinityKey(int idx, Ignite ignite) {
- return idx;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/139c2af6/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/CacheColumnDecisionTreeTrainerInput.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/CacheColumnDecisionTreeTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/CacheColumnDecisionTreeTrainerInput.java
deleted file mode 100644
index 40927b7..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/CacheColumnDecisionTreeTrainerInput.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.Stream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-
-/**
- * Adapter of a given cache to {@link CacheColumnDecisionTreeTrainerInput}
- *
- * @param <K> Class of keys of the cache.
- * @param <V> Class of values of the cache.
- */
-public abstract class CacheColumnDecisionTreeTrainerInput<K, V> implements ColumnDecisionTreeTrainerInput {
- /** Supplier of labels key. */
- private final IgniteSupplier<Stream<K>> labelsKeys;
-
- /** Count of features. */
- private final int featuresCnt;
-
- /** Function which maps feature index to Stream of keys corresponding to this feature index. */
- private final IgniteFunction<Integer, Stream<K>> keyMapper;
-
- /** Information about which features are categorical in form of feature index -> number of categories. */
- private final Map<Integer, Integer> catFeaturesInfo;
-
- /** Cache name. */
- private final String cacheName;
-
- /** Count of samples. */
- private final int samplesCnt;
-
- /** Function used for mapping cache values to stream of tuples. */
- private final IgniteFunction<Cache.Entry<K, V>, Stream<IgniteBiTuple<Integer, Double>>> valuesMapper;
-
- /**
- * Function which map value of entry with label key to DoubleStream.
- * Look at {@code CacheColumnDecisionTreeTrainerInput::labels} for understanding how {@code labelsKeys} and
- * {@code labelsMapper} interact.
- */
- private final IgniteFunction<V, DoubleStream> labelsMapper;
-
- /**
- * Constructs input for {@link ColumnDecisionTreeTrainer}.
- *
- * @param c Cache.
- * @param valuesMapper Function for mapping cache entry to stream used by {@link ColumnDecisionTreeTrainer}.
- * @param labelsMapper Function used for mapping cache value to labels array.
- * @param keyMapper Function used for mapping feature index to the cache key.
- * @param catFeaturesInfo Information about which features are categorical in form of feature index -> number of
- * categories.
- * @param featuresCnt Count of features.
- * @param samplesCnt Count of samples.
- */
- // TODO: IGNITE-5724 think about boxing/unboxing
- public CacheColumnDecisionTreeTrainerInput(IgniteCache<K, V> c,
- IgniteSupplier<Stream<K>> labelsKeys,
- IgniteFunction<Cache.Entry<K, V>, Stream<IgniteBiTuple<Integer, Double>>> valuesMapper,
- IgniteFunction<V, DoubleStream> labelsMapper,
- IgniteFunction<Integer, Stream<K>> keyMapper,
- Map<Integer, Integer> catFeaturesInfo,
- int featuresCnt, int samplesCnt) {
-
- cacheName = c.getName();
- this.labelsKeys = labelsKeys;
- this.valuesMapper = valuesMapper;
- this.labelsMapper = labelsMapper;
- this.keyMapper = keyMapper;
- this.catFeaturesInfo = catFeaturesInfo;
- this.samplesCnt = samplesCnt;
- this.featuresCnt = featuresCnt;
- }
-
- /** {@inheritDoc} */
- @Override public Stream<IgniteBiTuple<Integer, Double>> values(int idx) {
- return cache(Ignition.localIgnite()).getAll(keyMapper.apply(idx).collect(Collectors.toSet())).
- entrySet().
- stream().
- flatMap(ent -> valuesMapper.apply(new CacheEntryImpl<>(ent.getKey(), ent.getValue())));
- }
-
- /** {@inheritDoc} */
- @Override public double[] labels(Ignite ignite) {
- return labelsKeys.get().map(k -> get(k, ignite)).flatMapToDouble(labelsMapper).toArray();
- }
-
- /** {@inheritDoc} */
- @Override public Map<Integer, Integer> catFeaturesInfo() {
- return catFeaturesInfo;
- }
-
- /** {@inheritDoc} */
- @Override public int featuresCount() {
- return featuresCnt;
- }
-
- /** {@inheritDoc} */
- @Override public Object affinityKey(int idx, Ignite ignite) {
- return ignite.affinity(cacheName).affinityKey(keyMapper.apply(idx));
- }
-
- /** */
- private V get(K k, Ignite ignite) {
- V res = cache(ignite).localPeek(k);
-
- if (res == null)
- res = cache(ignite).get(k);
-
- return res;
- }
-
- /** */
- private IgniteCache<K, V> cache(Ignite ignite) {
- return ignite.getOrCreateCache(cacheName);
- }
-}
[49/54] [abbrv] ignite git commit: IGNITE-8141 Acknowledge SWAPPINESS
reduction using a range (<= 10). - Fixes #3727.
Posted by ag...@apache.org.
IGNITE-8141 Acknowledge SWAPPINESS reduction using a range (<= 10). - Fixes #3727.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/47ca3e0d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/47ca3e0d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/47ca3e0d
Branch: refs/heads/ignite-6083
Commit: 47ca3e0da7fd7468d7ea96ff512c3d1f3ed5bf81
Parents: b93595c
Author: Reed Sandberg <re...@drawbridge.com>
Authored: Thu Apr 12 20:18:10 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Thu Apr 12 20:18:10 2018 +0300
----------------------------------------------------------------------
.../suggestions/OsConfigurationSuggestions.java | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/47ca3e0d/modules/core/src/main/java/org/apache/ignite/internal/suggestions/OsConfigurationSuggestions.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/suggestions/OsConfigurationSuggestions.java b/modules/core/src/main/java/org/apache/ignite/internal/suggestions/OsConfigurationSuggestions.java
index e5b4c12..695b423 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/suggestions/OsConfigurationSuggestions.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/suggestions/OsConfigurationSuggestions.java
@@ -76,8 +76,18 @@ public class OsConfigurationSuggestions {
(dwcParamFlag && decParamFlag ? "s" : ""),
expected));
- if ((value = readVmParam(SWAPPINESS)) != null && !value.equals(expected = "10"))
- suggestions.add(String.format("Reduce pages swapping ratio (set vm.%s=%s)", SWAPPINESS, expected));
+ if ((value = readVmParam(SWAPPINESS)) != null) {
+ try {
+ double maxSwappiness = 10.0;
+
+ if (Float.parseFloat(value) > maxSwappiness)
+ suggestions.add(String.format("Reduce pages swapping ratio (set vm.%s=%f or less)", SWAPPINESS,
+ maxSwappiness));
+ }
+ catch (NumberFormatException ignored) {
+ // OS param not parsable as a number
+ }
+ }
if ((value = readVmParam(ZONE_RECLAIM_MODE)) != null && !value.equals(expected = "0"))
suggestions.add(String.format("Disable NUMA memory reclaim (set vm.%s=%s)", ZONE_RECLAIM_MODE,
[40/54] [abbrv] ignite git commit: IGNITE-6892 OOM should be covered
by failure handling
Posted by ag...@apache.org.
IGNITE-6892 OOM should be covered by failure handling
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d1be9b85
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d1be9b85
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d1be9b85
Branch: refs/heads/ignite-6083
Commit: d1be9b85507eb3358327e93b81031f92e660531b
Parents: 32fc6c3
Author: Aleksey Plekhanov <pl...@gmail.com>
Authored: Wed Apr 11 18:24:51 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Wed Apr 11 18:24:51 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 8 +
.../org/apache/ignite/internal/IgnitionEx.java | 50 +++-
.../discovery/GridDiscoveryManager.java | 3 +
.../processors/cache/WalStateManager.java | 8 +-
.../continuous/GridContinuousProcessor.java | 3 +
.../datastreamer/DataStreamProcessor.java | 3 +
.../processors/failure/FailureProcessor.java | 11 +
.../internal/processors/job/GridJobWorker.java | 8 +-
.../service/GridServiceProcessor.java | 15 +-
.../thread/IgniteStripedThreadPoolExecutor.java | 8 +-
.../ignite/thread/IgniteThreadFactory.java | 30 ++-
.../ignite/thread/IgniteThreadPoolExecutor.java | 12 +-
.../ignite/thread/OomExceptionHandler.java | 44 ++++
.../ignite/failure/OomFailureHandlerTest.java | 255 +++++++++++++++++++
.../ignite/testsuites/IgniteBasicTestSuite.java | 2 +
15 files changed, 430 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 662338c..437f49f 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -863,6 +863,14 @@ public final class IgniteSystemProperties {
public static final String IGNITE_BPLUS_TREE_LOCK_RETRIES = "IGNITE_BPLUS_TREE_LOCK_RETRIES";
/**
+ * Amount of memory reserved in the heap at node start, which can be dropped to increase the chances of success when
+ * handling OutOfMemoryError.
+ *
+ * Default is {@code 64kb}.
+ */
+ public static final String IGNITE_FAILURE_HANDLER_RESERVE_BUFFER_SIZE = "IGNITE_FAILURE_HANDLER_RESERVE_BUFFER_SIZE";
+
+ /**
* The threshold of uneven distribution above which partition distribution will be logged.
*
* The default is '50', that means: warn about nodes with 50+% difference.
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 10a0752..b3c3ee8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.lang.Thread.UncaughtExceptionHandler;
import java.lang.management.ManagementFactory;
import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
@@ -88,6 +89,7 @@ import org.apache.ignite.internal.util.typedef.CA;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -1764,6 +1766,13 @@ public class IgnitionEx {
validateThreadPoolSize(cfg.getPublicThreadPoolSize(), "public");
+ UncaughtExceptionHandler oomeHnd = new UncaughtExceptionHandler() {
+ @Override public void uncaughtException(Thread t, Throwable e) {
+ if (grid != null && X.hasCause(e, OutOfMemoryError.class))
+ grid.context().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+ }
+ };
+
execSvc = new IgniteThreadPoolExecutor(
"pub",
cfg.getIgniteInstanceName(),
@@ -1771,7 +1780,8 @@ public class IgnitionEx {
cfg.getPublicThreadPoolSize(),
DFLT_THREAD_KEEP_ALIVE_TIME,
new LinkedBlockingQueue<Runnable>(),
- GridIoPolicy.PUBLIC_POOL);
+ GridIoPolicy.PUBLIC_POOL,
+ oomeHnd);
execSvc.allowCoreThreadTimeOut(true);
@@ -1784,7 +1794,8 @@ public class IgnitionEx {
cfg.getServiceThreadPoolSize(),
DFLT_THREAD_KEEP_ALIVE_TIME,
new LinkedBlockingQueue<Runnable>(),
- GridIoPolicy.SERVICE_POOL);
+ GridIoPolicy.SERVICE_POOL,
+ oomeHnd);
svcExecSvc.allowCoreThreadTimeOut(true);
@@ -1797,7 +1808,8 @@ public class IgnitionEx {
cfg.getSystemThreadPoolSize(),
DFLT_THREAD_KEEP_ALIVE_TIME,
new LinkedBlockingQueue<Runnable>(),
- GridIoPolicy.SYSTEM_POOL);
+ GridIoPolicy.SYSTEM_POOL,
+ oomeHnd);
sysExecSvc.allowCoreThreadTimeOut(true);
@@ -1828,7 +1840,8 @@ public class IgnitionEx {
cfg.getManagementThreadPoolSize(),
DFLT_THREAD_KEEP_ALIVE_TIME,
new LinkedBlockingQueue<Runnable>(),
- GridIoPolicy.MANAGEMENT_POOL);
+ GridIoPolicy.MANAGEMENT_POOL,
+ oomeHnd);
mgmtExecSvc.allowCoreThreadTimeOut(true);
@@ -1844,7 +1857,8 @@ public class IgnitionEx {
cfg.getPeerClassLoadingThreadPoolSize(),
DFLT_THREAD_KEEP_ALIVE_TIME,
new LinkedBlockingQueue<Runnable>(),
- GridIoPolicy.P2P_POOL);
+ GridIoPolicy.P2P_POOL,
+ oomeHnd);
p2pExecSvc.allowCoreThreadTimeOut(true);
@@ -1879,7 +1893,8 @@ public class IgnitionEx {
callbackExecSvc = new IgniteStripedThreadPoolExecutor(
cfg.getAsyncCallbackPoolSize(),
cfg.getIgniteInstanceName(),
- "callback");
+ "callback",
+ oomeHnd);
if (myCfg.getConnectorConfiguration() != null) {
validateThreadPoolSize(myCfg.getConnectorConfiguration().getThreadPoolSize(), "connector");
@@ -1890,7 +1905,9 @@ public class IgnitionEx {
myCfg.getConnectorConfiguration().getThreadPoolSize(),
myCfg.getConnectorConfiguration().getThreadPoolSize(),
DFLT_THREAD_KEEP_ALIVE_TIME,
- new LinkedBlockingQueue<Runnable>()
+ new LinkedBlockingQueue<Runnable>(),
+ GridIoPolicy.UNDEFINED,
+ oomeHnd
);
restExecSvc.allowCoreThreadTimeOut(true);
@@ -1905,7 +1922,8 @@ public class IgnitionEx {
myCfg.getUtilityCacheThreadPoolSize(),
myCfg.getUtilityCacheKeepAliveTime(),
new LinkedBlockingQueue<Runnable>(),
- GridIoPolicy.UTILITY_CACHE_POOL);
+ GridIoPolicy.UTILITY_CACHE_POOL,
+ oomeHnd);
utilityCacheExecSvc.allowCoreThreadTimeOut(true);
@@ -1916,7 +1934,8 @@ public class IgnitionEx {
1,
DFLT_THREAD_KEEP_ALIVE_TIME,
new LinkedBlockingQueue<Runnable>(),
- GridIoPolicy.AFFINITY_POOL);
+ GridIoPolicy.AFFINITY_POOL,
+ oomeHnd);
affExecSvc.allowCoreThreadTimeOut(true);
@@ -1930,7 +1949,8 @@ public class IgnitionEx {
cpus * 2,
3000L,
new LinkedBlockingQueue<Runnable>(1000),
- GridIoPolicy.IDX_POOL
+ GridIoPolicy.IDX_POOL,
+ oomeHnd
);
}
@@ -1943,7 +1963,8 @@ public class IgnitionEx {
cfg.getQueryThreadPoolSize(),
DFLT_THREAD_KEEP_ALIVE_TIME,
new LinkedBlockingQueue<Runnable>(),
- GridIoPolicy.QUERY_POOL);
+ GridIoPolicy.QUERY_POOL,
+ oomeHnd);
qryExecSvc.allowCoreThreadTimeOut(true);
@@ -1954,7 +1975,8 @@ public class IgnitionEx {
2,
DFLT_THREAD_KEEP_ALIVE_TIME,
new LinkedBlockingQueue<Runnable>(),
- GridIoPolicy.SCHEMA_POOL);
+ GridIoPolicy.SCHEMA_POOL,
+ oomeHnd);
schemaExecSvc.allowCoreThreadTimeOut(true);
@@ -1970,7 +1992,9 @@ public class IgnitionEx {
execCfg.getSize(),
execCfg.getSize(),
DFLT_THREAD_KEEP_ALIVE_TIME,
- new LinkedBlockingQueue<Runnable>());
+ new LinkedBlockingQueue<Runnable>(),
+ GridIoPolicy.UNDEFINED,
+ oomeHnd);
customExecSvcs.put(execCfg.getName(), exec);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 400bb5f..77c9657 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -130,6 +130,7 @@ import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.thread.IgniteThread;
+import org.apache.ignite.thread.OomExceptionHandler;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -924,6 +925,8 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
segChkThread = new IgniteThread(segChkWrk);
+ segChkThread.setUncaughtExceptionHandler(new OomExceptionHandler(ctx));
+
segChkThread.start();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
index 0ac699f..64a6819 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
@@ -38,6 +38,7 @@ import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.thread.OomExceptionHandler;
import org.apache.ignite.thread.IgniteThread;
import org.jetbrains.annotations.Nullable;
@@ -473,7 +474,12 @@ public class WalStateManager extends GridCacheSharedManagerAdapter {
// not-yet-flushed dirty pages have been logged.
WalStateChangeWorker worker = new WalStateChangeWorker(msg, cpFut);
- new IgniteThread(worker).start();
+ IgniteThread thread = new IgniteThread(worker);
+
+ thread.setUncaughtExceptionHandler(new OomExceptionHandler(
+ cctx.kernalContext()));
+
+ thread.start();
}
else {
// Disable: not-yet-flushed operations are not logged, so wait for them
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index cebe4b1..2d48b7d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -88,6 +88,7 @@ import org.apache.ignite.spi.discovery.DiscoveryDataBag;
import org.apache.ignite.spi.discovery.DiscoveryDataBag.GridDiscoveryData;
import org.apache.ignite.spi.discovery.DiscoveryDataBag.JoiningNodeDiscoveryData;
import org.apache.ignite.thread.IgniteThread;
+import org.apache.ignite.thread.OomExceptionHandler;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
@@ -1727,6 +1728,8 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
}
});
+ checker.setUncaughtExceptionHandler(new OomExceptionHandler(ctx));
+
bufCheckThreads.put(routineId, checker);
checker.start();
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
index 8b984c0..e63d7d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
@@ -44,6 +44,7 @@ import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.marshaller.Marshaller;
import org.apache.ignite.stream.StreamReceiver;
import org.apache.ignite.thread.IgniteThread;
+import org.apache.ignite.thread.OomExceptionHandler;
import org.jetbrains.annotations.Nullable;
import java.util.Collection;
@@ -125,6 +126,8 @@ public class DataStreamProcessor<K, V> extends GridProcessorAdapter {
}
});
+ flusher.setUncaughtExceptionHandler(new OomExceptionHandler(ctx));
+
flusher.start();
if (log.isDebugEnabled())
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
index 615fb9f..0234e84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
@@ -19,12 +19,14 @@ package org.apache.ignite.internal.processors.failure;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.failure.FailureHandler;
import org.apache.ignite.failure.StopNodeOrHaltFailureHandler;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.processors.GridProcessorAdapter;
+import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
/**
@@ -40,6 +42,9 @@ public class FailureProcessor extends GridProcessorAdapter {
/** Failure context. */
private volatile FailureContext failureCtx;
+ /** Reserve buffer, which can be dropped to handle OOME. */
+ private volatile byte[] reserveBuf;
+
/**
* @param ctx Context.
*/
@@ -56,6 +61,9 @@ public class FailureProcessor extends GridProcessorAdapter {
if (hnd == null)
hnd = getDefaultFailureHandler();
+ reserveBuf = new byte[IgniteSystemProperties.getInteger(
+ IgniteSystemProperties.IGNITE_FAILURE_HANDLER_RESERVE_BUFFER_SIZE, 64 * 1024)];
+
assert hnd != null;
this.hnd = hnd;
@@ -102,6 +110,9 @@ public class FailureProcessor extends GridProcessorAdapter {
U.error(ignite.log(), "Critical failure. Will be handled accordingly to configured handler [hnd=" +
hnd.getClass() + ", failureCtx=" + failureCtx + ']', failureCtx.error());
+ if (reserveBuf != null && X.hasCause(failureCtx.error(), OutOfMemoryError.class))
+ reserveBuf = null;
+
boolean invalidated = hnd.onFailure(ignite, failureCtx);
if (invalidated) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index 6d2e621..f7c07f5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -36,6 +36,8 @@ import org.apache.ignite.compute.ComputeJobContext;
import org.apache.ignite.compute.ComputeJobMasterLeaveAware;
import org.apache.ignite.compute.ComputeUserUndeclaredException;
import org.apache.ignite.events.JobEvent;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.igfs.IgfsOutOfSpaceException;
import org.apache.ignite.internal.GridInternalException;
import org.apache.ignite.internal.GridJobContextImpl;
@@ -603,9 +605,13 @@ public class GridJobWorker extends GridWorker implements GridTimeoutObject {
X.hasCause(e, ClusterTopologyCheckedException.class))
// Should be throttled, because GridServiceProxy continuously retry getting service.
LT.error(log, e, "Failed to execute job [jobId=" + ses.getJobId() + ", ses=" + ses + ']');
- else
+ else {
U.error(log, "Failed to execute job [jobId=" + ses.getJobId() + ", ses=" + ses + ']', e);
+ if (X.hasCause(e, OutOfMemoryError.class))
+ ctx.failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+ }
+
ex = e;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index ff68e72..63f5027 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.service;
+import java.lang.Thread.UncaughtExceptionHandler;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -103,6 +104,7 @@ import org.apache.ignite.services.ServiceConfiguration;
import org.apache.ignite.services.ServiceDeploymentException;
import org.apache.ignite.services.ServiceDescriptor;
import org.apache.ignite.thread.IgniteThreadFactory;
+import org.apache.ignite.thread.OomExceptionHandler;
import org.apache.ignite.transactions.Transaction;
import org.jetbrains.annotations.Nullable;
import java.util.concurrent.ConcurrentHashMap;
@@ -112,7 +114,6 @@ import static org.apache.ignite.IgniteSystemProperties.getString;
import static org.apache.ignite.configuration.DeploymentMode.ISOLATED;
import static org.apache.ignite.configuration.DeploymentMode.PRIVATE;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SERVICES_COMPATIBILITY_MODE;
-import static org.apache.ignite.internal.processors.cache.GridCacheUtils.UTILITY_CACHE_NAME;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
@@ -154,8 +155,12 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
/** Busy lock. */
private volatile GridSpinBusyLock busyLock = new GridSpinBusyLock();
+ /** Uncaught exception handler for thread pools. */
+ private final UncaughtExceptionHandler oomeHnd = new OomExceptionHandler(ctx);
+
/** Thread factory. */
- private ThreadFactory threadFactory = new IgniteThreadFactory(ctx.igniteInstanceName(), "service");
+ private ThreadFactory threadFactory = new IgniteThreadFactory(ctx.igniteInstanceName(), "service",
+ oomeHnd);
/** Thread local for service name. */
private ThreadLocal<String> svcName = new ThreadLocal<>();
@@ -175,7 +180,8 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
public GridServiceProcessor(GridKernalContext ctx) {
super(ctx);
- depExe = Executors.newSingleThreadExecutor(new IgniteThreadFactory(ctx.igniteInstanceName(), "srvc-deploy"));
+ depExe = Executors.newSingleThreadExecutor(new IgniteThreadFactory(ctx.igniteInstanceName(),
+ "srvc-deploy", oomeHnd));
String servicesCompatibilityMode = getString(IGNITE_SERVICES_COMPATIBILITY_MODE);
@@ -373,7 +379,8 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
busyLock = new GridSpinBusyLock();
- depExe = Executors.newSingleThreadExecutor(new IgniteThreadFactory(ctx.igniteInstanceName(), "srvc-deploy"));
+ depExe = Executors.newSingleThreadExecutor(new IgniteThreadFactory(ctx.igniteInstanceName(),
+ "srvc-deploy", oomeHnd));
start();
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/thread/IgniteStripedThreadPoolExecutor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteStripedThreadPoolExecutor.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteStripedThreadPoolExecutor.java
index 3cd7484..418812f 100644
--- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteStripedThreadPoolExecutor.java
+++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteStripedThreadPoolExecutor.java
@@ -17,6 +17,7 @@
package org.apache.ignite.thread;
+import java.lang.Thread.UncaughtExceptionHandler;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -45,10 +46,11 @@ public class IgniteStripedThreadPoolExecutor implements ExecutorService {
* @param igniteInstanceName Node name.
* @param threadNamePrefix Thread name prefix.
*/
- public IgniteStripedThreadPoolExecutor(int concurrentLvl, String igniteInstanceName, String threadNamePrefix) {
+ public IgniteStripedThreadPoolExecutor(int concurrentLvl, String igniteInstanceName, String threadNamePrefix,
+ UncaughtExceptionHandler eHnd) {
execs = new ExecutorService[concurrentLvl];
- ThreadFactory factory = new IgniteThreadFactory(igniteInstanceName, threadNamePrefix);
+ ThreadFactory factory = new IgniteThreadFactory(igniteInstanceName, threadNamePrefix, eHnd);
for (int i = 0; i < concurrentLvl; i++)
execs[i] = Executors.newSingleThreadExecutor(factory);
@@ -173,4 +175,4 @@ public class IgniteStripedThreadPoolExecutor implements ExecutorService {
@Override public String toString() {
return S.toString(IgniteStripedThreadPoolExecutor.class, this);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
index 062c973..23bf14d 100644
--- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
@@ -17,9 +17,9 @@
package org.apache.ignite.thread;
+import java.lang.Thread.UncaughtExceptionHandler;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
-
import org.apache.ignite.internal.managers.communication.GridIoPolicy;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.NotNull;
@@ -41,6 +41,9 @@ public class IgniteThreadFactory implements ThreadFactory {
/** */
private final byte plc;
+ /** Exception handler. */
+ private final UncaughtExceptionHandler eHnd;
+
/**
* Constructs new thread factory for given grid. All threads will belong
* to the same default thread group.
@@ -49,7 +52,19 @@ public class IgniteThreadFactory implements ThreadFactory {
* @param threadName Thread name.
*/
public IgniteThreadFactory(String igniteInstanceName, String threadName) {
- this(igniteInstanceName, threadName, GridIoPolicy.UNDEFINED);
+ this(igniteInstanceName, threadName, null);
+ }
+
+ /**
+ * Constructs new thread factory for given grid. All threads will belong
+ * to the same default thread group.
+ *
+ * @param igniteInstanceName Ignite instance name.
+ * @param threadName Thread name.
+ * @param eHnd Uncaught exception handler.
+ */
+ public IgniteThreadFactory(String igniteInstanceName, String threadName, UncaughtExceptionHandler eHnd) {
+ this(igniteInstanceName, threadName, GridIoPolicy.UNDEFINED, eHnd);
}
/**
@@ -59,16 +74,23 @@ public class IgniteThreadFactory implements ThreadFactory {
* @param igniteInstanceName Ignite instance name.
* @param threadName Thread name.
* @param plc {@link GridIoPolicy} for thread pool.
+ * @param eHnd Uncaught exception handler.
*/
- public IgniteThreadFactory(String igniteInstanceName, String threadName, byte plc) {
+ public IgniteThreadFactory(String igniteInstanceName, String threadName, byte plc, UncaughtExceptionHandler eHnd) {
this.igniteInstanceName = igniteInstanceName;
this.threadName = threadName;
this.plc = plc;
+ this.eHnd = eHnd;
}
/** {@inheritDoc} */
@Override public Thread newThread(@NotNull Runnable r) {
- return new IgniteThread(igniteInstanceName, threadName, r, idxGen.incrementAndGet(), -1, plc);
+ Thread thread = new IgniteThread(igniteInstanceName, threadName, r, idxGen.incrementAndGet(), -1, plc);
+
+ if (eHnd != null)
+ thread.setUncaughtExceptionHandler(eHnd);
+
+ return thread;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java
index 83c64c3..fed77ad 100644
--- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java
+++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java
@@ -17,6 +17,7 @@
package org.apache.ignite.thread;
+import java.lang.Thread.UncaughtExceptionHandler;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
@@ -53,7 +54,8 @@ public class IgniteThreadPoolExecutor extends ThreadPoolExecutor {
maxPoolSize,
keepAliveTime,
workQ,
- GridIoPolicy.UNDEFINED);
+ GridIoPolicy.UNDEFINED,
+ null);
}
/**
@@ -68,6 +70,7 @@ public class IgniteThreadPoolExecutor extends ThreadPoolExecutor {
* @param workQ The queue to use for holding tasks before they are executed. This queue will hold only
* runnable tasks submitted by the {@link #execute(Runnable)} method.
* @param plc {@link GridIoPolicy} for thread pool.
+ * @param eHnd Uncaught exception handler for thread pool.
*/
public IgniteThreadPoolExecutor(
String threadNamePrefix,
@@ -76,14 +79,15 @@ public class IgniteThreadPoolExecutor extends ThreadPoolExecutor {
int maxPoolSize,
long keepAliveTime,
BlockingQueue<Runnable> workQ,
- byte plc) {
+ byte plc,
+ UncaughtExceptionHandler eHnd) {
super(
corePoolSize,
maxPoolSize,
keepAliveTime,
TimeUnit.MILLISECONDS,
workQ,
- new IgniteThreadFactory(igniteInstanceName, threadNamePrefix, plc)
+ new IgniteThreadFactory(igniteInstanceName, threadNamePrefix, plc, eHnd)
);
}
@@ -114,4 +118,4 @@ public class IgniteThreadPoolExecutor extends ThreadPoolExecutor {
new AbortPolicy()
);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/main/java/org/apache/ignite/thread/OomExceptionHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/OomExceptionHandler.java b/modules/core/src/main/java/org/apache/ignite/thread/OomExceptionHandler.java
new file mode 100644
index 0000000..3a62ad8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/thread/OomExceptionHandler.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.thread;
+
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.util.typedef.X;
+
+/**
+ * OOM exception handler for system threads.
+ */
+public class OomExceptionHandler implements Thread.UncaughtExceptionHandler {
+ /** Context. */
+ private final GridKernalContext ctx;
+
+ /**
+ * @param ctx Context.
+ */
+ public OomExceptionHandler(GridKernalContext ctx) {
+ this.ctx = ctx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void uncaughtException(Thread t, Throwable e) {
+ if (X.hasCause(e, OutOfMemoryError.class))
+ ctx.failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/test/java/org/apache/ignite/failure/OomFailureHandlerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/OomFailureHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/failure/OomFailureHandlerTest.java
new file mode 100644
index 0000000..2af94b8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/failure/OomFailureHandlerTest.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Out of memory error failure handler test.
+ */
+public class OomFailureHandlerTest extends AbstractFailureHandlerTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setCacheConfiguration(new CacheConfiguration()
+ .setName(DEFAULT_CACHE_NAME)
+ .setCacheMode(CacheMode.PARTITIONED)
+ .setBackups(0)
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+ }
+
+ /**
+ * Test OOME in IgniteCompute.
+ */
+ public void testComputeOomError() throws Exception {
+ IgniteEx ignite0 = startGrid(0);
+ IgniteEx ignite1 = startGrid(1);
+
+ try {
+ IgniteFuture<Boolean> res = ignite0.compute(ignite0.cluster().forNodeId(ignite1.cluster().localNode().id()))
+ .callAsync(new IgniteCallable<Boolean>() {
+ @Override public Boolean call() throws Exception {
+ throw new OutOfMemoryError();
+ }
+ });
+
+ res.get();
+ }
+ catch (Throwable ignore) {
+ // Expected.
+ }
+
+ assertFailureState(ignite0, ignite1);
+ }
+
+ /**
+ * Test OOME in EntryProcessor.
+ */
+ public void testEntryProcessorOomError() throws Exception {
+ IgniteEx ignite0 = startGrid(0);
+ IgniteEx ignite1 = startGrid(1);
+
+ IgniteCache<Integer, Integer> cache0 = ignite0.getOrCreateCache(DEFAULT_CACHE_NAME);
+ IgniteCache<Integer, Integer> cache1 = ignite1.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+ awaitPartitionMapExchange();
+
+ Integer key = primaryKey(cache1);
+
+ cache1.put(key, key);
+
+ try {
+ IgniteFuture fut = cache0.invokeAsync(key, new EntryProcessor<Integer, Integer, Object>() {
+ @Override public Object process(MutableEntry<Integer, Integer> entry,
+ Object... arguments) throws EntryProcessorException {
+ throw new OutOfMemoryError();
+ }
+ });
+
+ fut.get();
+ }
+ catch (Throwable ignore) {
+ // Expected.
+ }
+
+ assertFailureState(ignite0, ignite1);
+ }
+
+ /**
+ * Test OOME in service method invocation.
+ */
+ public void testServiceInvokeOomError() throws Exception {
+ IgniteEx ignite0 = startGrid(0);
+ IgniteEx ignite1 = startGrid(1);
+
+ IgniteCache<Integer, Integer> cache1 = ignite1.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+ awaitPartitionMapExchange();
+
+ Integer key = primaryKey(cache1);
+
+ ignite0.services().deployKeyAffinitySingleton("fail-invoke-service", new FailServiceImpl(false),
+ DEFAULT_CACHE_NAME, key);
+
+ FailService svc = ignite0.services().serviceProxy("fail-invoke-service", FailService.class, false);
+
+ try {
+ svc.fail();
+ }
+ catch (Throwable ignore) {
+ // Expected.
+ }
+
+ assertFailureState(ignite0, ignite1);
+ }
+
+ /**
+ * Test OOME in service execute.
+ */
+ public void testServiceExecuteOomError() throws Exception {
+ IgniteEx ignite0 = startGrid(0);
+ IgniteEx ignite1 = startGrid(1);
+
+ IgniteCache<Integer, Integer> cache1 = ignite1.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+ awaitPartitionMapExchange();
+
+ Integer key = primaryKey(cache1);
+
+ ignite0.services().deployKeyAffinitySingleton("fail-execute-service", new FailServiceImpl(true),
+ DEFAULT_CACHE_NAME, key);
+
+ assertFailureState(ignite0, ignite1);
+ }
+
+ /**
+ * Test OOME in event listener.
+ */
+ public void testEventListenerOomError() throws Exception {
+ IgniteEx ignite0 = startGrid(0);
+ IgniteEx ignite1 = startGrid(1);
+
+ IgniteCache<Integer, Integer> cache0 = ignite0.getOrCreateCache(DEFAULT_CACHE_NAME);
+ IgniteCache<Integer, Integer> cache1 = ignite1.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+ awaitPartitionMapExchange();
+
+ ignite1.events().localListen(new IgnitePredicate<Event>() {
+ @Override public boolean apply(Event evt) {
+ throw new OutOfMemoryError();
+ }
+ }, EventType.EVT_CACHE_OBJECT_PUT);
+
+ Integer key = primaryKey(cache1);
+
+ try {
+ cache0.put(key, key);
+ }
+ catch (Throwable ignore) {
+ // Expected.
+ }
+
+ assertFailureState(ignite0, ignite1);
+ }
+
+ /**
+ * @param igniteWork Working ignite instance.
+ * @param igniteFail Failed ignite instance.
+ */
+ private static void assertFailureState(Ignite igniteWork, Ignite igniteFail) throws IgniteInterruptedCheckedException {
+ assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return dummyFailureHandler(igniteFail).failure();
+ }
+ }, 5000L));
+
+ assertFalse(dummyFailureHandler(igniteWork).failure());
+ }
+
+ /**
+ *
+ */
+ private interface FailService extends Service {
+ /**
+ * Fail.
+ */
+ void fail();
+ }
+
+ /**
+ *
+ */
+ private static class FailServiceImpl implements FailService {
+ /** Fail on execute. */
+ private final boolean failOnExec;
+
+ /**
+ * @param failOnExec Fail on execute.
+ */
+ private FailServiceImpl(boolean failOnExec) {
+ this.failOnExec = failOnExec;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void fail() {
+ throw new OutOfMemoryError();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancel(ServiceContext ctx) {
+ }
+
+ /** {@inheritDoc} */
+ @Override public void init(ServiceContext ctx) throws Exception {
+ }
+
+ /** {@inheritDoc} */
+ @Override public void execute(ServiceContext ctx) throws Exception {
+ if (failOnExec)
+ throw new OutOfMemoryError();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1be9b85/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index c4b7d92..c388f1d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -22,6 +22,7 @@ import junit.framework.TestSuite;
import org.apache.ignite.GridSuppressedExceptionSelfTest;
import org.apache.ignite.failure.FailureHandlerTriggeredTest;
import org.apache.ignite.failure.IoomFailureHandlerTest;
+import org.apache.ignite.failure.OomFailureHandlerTest;
import org.apache.ignite.failure.StopNodeFailureHandlerTest;
import org.apache.ignite.failure.StopNodeOrHaltFailureHandlerTest;
import org.apache.ignite.internal.ClassSetTest;
@@ -199,6 +200,7 @@ public class IgniteBasicTestSuite extends TestSuite {
suite.addTestSuite(StopNodeFailureHandlerTest.class);
suite.addTestSuite(StopNodeOrHaltFailureHandlerTest.class);
suite.addTestSuite(IoomFailureHandlerTest.class);
+ suite.addTestSuite(OomFailureHandlerTest.class);
return suite;
}
[54/54] [abbrv] ignite git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/ignite into ignite-6083
Posted by ag...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/ignite into ignite-6083
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ecefdd33
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ecefdd33
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ecefdd33
Branch: refs/heads/ignite-6083
Commit: ecefdd335fedfe4c8020a401a63ce4477bd15737
Parents: 6e92fff 44c4656
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Fri Apr 13 12:32:50 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Apr 13 12:32:50 2018 +0300
----------------------------------------------------------------------
.gitignore | 1 +
.../ignite/examples/ml/genetic/change/Coin.java | 7 +-
.../OptimizeMakeChangeFitnessFunction.java | 3 +-
.../change/OptimizeMakeChangeGAExample.java | 8 +-
.../OptimizeMakeChangeTerminateCriteria.java | 3 +-
.../helloworld/HelloWorldFitnessFunction.java | 3 +-
.../genetic/helloworld/HelloWorldGAExample.java | 9 +-
.../helloworld/HelloWorldTerminateCriteria.java | 3 +-
.../ignite/examples/ml/genetic/movie/Movie.java | 4 +-
.../ml/genetic/movie/MovieFitnessFunction.java | 3 +-
.../ml/genetic/movie/MovieGAExample.java | 4 +-
.../genetic/movie/MovieTerminateCriteria.java | 4 +-
.../ml/knn/KNNClassificationExample.java | 11 +-
.../examples/ml/nn/MLPTrainerExample.java | 4 +-
.../ml/preprocessing/NormalizationExample.java | 17 +-
...nWithLSQRTrainerAndNormalizationExample.java | 23 +-
...dLinearRegressionWithLSQRTrainerExample.java | 14 +-
...tedLinearRegressionWithQRTrainerExample.java | 9 +-
...edLinearRegressionWithSGDTrainerExample.java | 78 +-
.../binary/SVMBinaryClassificationExample.java | 11 +-
.../SVMMultiClassClassificationExample.java | 24 +-
...ecisionTreeClassificationTrainerExample.java | 148 +
.../DecisionTreeRegressionTrainerExample.java | 124 +
.../ignite/examples/ml/tree/package-info.java | 22 +
.../examples/ml/trees/DecisionTreesExample.java | 354 --
.../ignite/examples/ml/trees/package-info.java | 22 -
.../jdbc2/JdbcAbstractDmlStatementSelfTest.java | 6 +-
.../jdbc/thin/JdbcThinConnectionSelfTest.java | 233 +-
.../jdbc/thin/JdbcThinMetadataSelfTest.java | 22 +-
.../junits/IgniteCompatibilityAbstractTest.java | 4 +-
.../junits/IgniteCompatibilityNodeRunner.java | 4 +-
.../util/CompatibilityTestsUtils.java | 93 +
.../apache/ignite/IgniteSystemProperties.java | 33 +
.../org/apache/ignite/cache/CacheMetrics.java | 7 +
.../org/apache/ignite/cache/QueryEntity.java | 41 +-
.../cache/query/annotations/QuerySqlField.java | 16 +-
.../client/ClientAuthenticationException.java | 18 +-
.../client/ClientAuthorizationException.java | 46 +
.../apache/ignite/client/ClientException.java | 3 +-
.../ignite/compute/ComputeTaskAdapter.java | 2 +-
.../CommunicationFailureContext.java | 62 +
.../CommunicationFailureResolver.java | 28 +
.../configuration/DataStorageConfiguration.java | 6 +-
.../DefaultCommunicationFailureResolver.java | 305 ++
.../configuration/IgniteConfiguration.java | 22 +
.../failure/StopNodeOrHaltFailureHandler.java | 2 +-
.../ignite/internal/GridKernalContext.java | 8 +
.../ignite/internal/GridKernalContextImpl.java | 10 +
.../org/apache/ignite/internal/GridTopic.java | 8 +-
.../apache/ignite/internal/IgniteKernal.java | 23 +-
.../org/apache/ignite/internal/IgnitionEx.java | 68 +-
.../GridClientConnectionManagerAdapter.java | 6 +
.../impl/GridTcpRouterNioListenerAdapter.java | 6 +
.../internal/client/thin/ClientChannel.java | 3 +-
.../internal/client/thin/ClientQueryCursor.java | 6 +-
.../internal/client/thin/ClientUtils.java | 38 +-
.../internal/client/thin/TcpClientChannel.java | 48 +-
.../jdbc/thin/ConnectionPropertiesImpl.java | 161 +-
.../jdbc/thin/JdbcThinDatabaseMetadata.java | 6 +-
.../internal/managers/GridManagerAdapter.java | 8 +
.../managers/communication/GridIoManager.java | 4 +-
.../communication/GridIoMessageFactory.java | 18 +
.../discovery/CustomMessageWrapper.java | 5 +
.../internal/managers/discovery/DiscoCache.java | 8 +
.../discovery/DiscoveryCustomMessage.java | 10 +-
.../DiscoveryMessageResultsCollector.java | 222 +
.../discovery/GridDiscoveryManager.java | 159 +-
.../managers/discovery/IgniteClusterNode.java | 69 +
.../managers/discovery/IgniteDiscoverySpi.java | 67 +
.../IgniteDiscoverySpiInternalListener.java | 42 +
.../pagemem/impl/PageMemoryNoStoreImpl.java | 17 +-
.../MetaPageUpdatePartitionDataRecord.java | 2 +-
.../affinity/GridAffinityAssignmentCache.java | 50 +-
.../IgniteAuthenticationProcessor.java | 5 +-
.../authentication/UserAcceptedMessage.java | 5 +
.../authentication/UserProposedMessage.java | 5 +
.../cache/CacheAffinityChangeMessage.java | 5 +
.../cache/CacheAffinitySharedManager.java | 26 +-
.../processors/cache/CacheMetricsImpl.java | 2 +-
.../cache/CacheStatisticsModeChangeMessage.java | 5 +
.../ClientCacheChangeDiscoveryMessage.java | 5 +
.../ClientCacheChangeDummyDiscoveryMessage.java | 5 +
.../cache/DynamicCacheChangeBatch.java | 5 +
.../processors/cache/GridCacheAdapter.java | 3 +-
.../processors/cache/GridCacheMvccManager.java | 38 +
.../GridCachePartitionExchangeManager.java | 48 +-
.../processors/cache/GridCacheProcessor.java | 36 +-
.../cache/GridCacheSharedContext.java | 9 +-
.../cache/GridCacheSharedTtlCleanupManager.java | 44 +-
.../processors/cache/GridCacheUtils.java | 16 +-
.../cache/IgniteCacheOffheapManager.java | 8 +-
.../cache/IgniteCacheOffheapManagerImpl.java | 10 +-
.../processors/cache/IgniteCacheProxyImpl.java | 3 +
.../processors/cache/WalStateFinishMessage.java | 5 +
.../processors/cache/WalStateManager.java | 8 +-
.../cache/WalStateProposeMessage.java | 5 +
.../cache/binary/BinaryMetadataTransport.java | 24 +-
.../binary/MetadataUpdateAcceptedMessage.java | 5 +
.../binary/MetadataUpdateProposedMessage.java | 5 +
.../dht/GridClientPartitionTopology.java | 44 +
.../distributed/dht/GridDhtCacheAdapter.java | 4 +
.../distributed/dht/GridDhtLocalPartition.java | 9 +-
.../dht/GridDhtPartitionTopology.java | 12 +
.../dht/GridDhtPartitionTopologyImpl.java | 65 +-
.../dht/GridDhtPartitionsStateValidator.java | 255 +
.../cache/distributed/dht/GridDhtTxLocal.java | 5 +
.../GridDhtPartitionsExchangeFuture.java | 122 +-
.../GridDhtPartitionsSingleMessage.java | 68 +-
.../dht/preloader/InitNewCoordinatorFuture.java | 2 +-
.../preloader/latch/ExchangeLatchManager.java | 696 +++
.../distributed/dht/preloader/latch/Latch.java | 52 +
.../dht/preloader/latch/LatchAckMessage.java | 165 +
.../cache/distributed/near/GridNearTxLocal.java | 19 +
.../GridCacheDatabaseSharedManager.java | 60 +-
.../persistence/GridCacheOffheapManager.java | 10 +-
.../persistence/pagemem/PageMemoryImpl.java | 17 +-
.../TrackingPageIsCorruptedException.java | 60 +
.../cache/persistence/tree/BPlusTree.java | 41 +-
.../cache/persistence/tree/io/PageMetaIO.java | 6 +-
.../persistence/tree/io/TrackingPageIO.java | 158 +-
.../wal/FileWriteAheadLogManager.java | 157 +-
.../wal/FsyncModeFileWriteAheadLogManager.java | 34 +-
.../wal/reader/StandaloneGridKernalContext.java | 6 +
.../cache/query/QueryEntityTypeDescriptor.java | 21 +
.../cache/store/GridCacheWriteBehindStore.java | 2 +-
.../cache/transactions/IgniteTxAdapter.java | 2 +-
.../cache/transactions/IgniteTxManager.java | 36 +-
.../cluster/ChangeGlobalStateFinishMessage.java | 5 +
.../cluster/ChangeGlobalStateMessage.java | 5 +
.../cluster/ClusterMetricsUpdateMessage.java | 158 +
.../processors/cluster/ClusterNodeMetrics.java | 62 +
.../processors/cluster/ClusterProcessor.java | 249 +-
.../continuous/AbstractContinuousMessage.java | 5 +
.../continuous/ContinuousRoutineInfo.java | 100 +
.../ContinuousRoutineStartResultMessage.java | 206 +
.../ContinuousRoutinesCommonDiscoveryData.java | 45 +
.../continuous/ContinuousRoutinesInfo.java | 132 +
...tinuousRoutinesJoiningNodeDiscoveryData.java | 45 +
.../continuous/GridContinuousProcessor.java | 865 +++-
.../continuous/StartRequestDataV2.java | 164 +
.../StartRoutineDiscoveryMessageV2.java | 77 +
.../StopRoutineAckDiscoveryMessage.java | 5 +
.../datastreamer/DataStreamProcessor.java | 3 +
.../datastreamer/DataStreamerImpl.java | 27 +-
.../processors/failure/FailureProcessor.java | 11 +
.../processors/igfs/IgfsMetaManager.java | 30 +-
.../internal/processors/job/GridJobWorker.java | 10 +-
.../marshaller/MappingAcceptedMessage.java | 5 +
.../marshaller/MappingProposedMessage.java | 5 +
.../odbc/ClientListenerNioListener.java | 19 +
.../processors/odbc/jdbc/JdbcColumnMeta.java | 14 +
.../processors/odbc/jdbc/JdbcColumnMetaV4.java | 90 +
.../odbc/jdbc/JdbcConnectionContext.java | 4 +-
.../odbc/jdbc/JdbcMetaColumnsResultV4.java | 50 +
.../odbc/jdbc/JdbcRequestHandler.java | 14 +-
.../processors/odbc/jdbc/JdbcResult.java | 8 +
.../client/ClientConnectionContext.java | 53 +-
.../platform/client/ClientRequest.java | 29 +
.../platform/client/ClientRequestHandler.java | 7 +-
.../platform/client/ClientStatus.java | 6 +
.../cache/ClientCacheClearKeyRequest.java | 3 +
.../cache/ClientCacheClearKeysRequest.java | 3 +
.../client/cache/ClientCacheClearRequest.java | 3 +
.../cache/ClientCacheContainsKeyRequest.java | 3 +
.../cache/ClientCacheContainsKeysRequest.java | 3 +
...ientCacheCreateWithConfigurationRequest.java | 6 +-
.../cache/ClientCacheCreateWithNameRequest.java | 3 +
.../client/cache/ClientCacheDestroyRequest.java | 3 +
.../client/cache/ClientCacheGetAllRequest.java | 3 +
.../ClientCacheGetAndPutIfAbsentRequest.java | 3 +
.../cache/ClientCacheGetAndPutRequest.java | 3 +
.../cache/ClientCacheGetAndRemoveRequest.java | 3 +
.../cache/ClientCacheGetAndReplaceRequest.java | 3 +
...acheGetOrCreateWithConfigurationRequest.java | 6 +-
.../ClientCacheGetOrCreateWithNameRequest.java | 3 +
.../client/cache/ClientCacheGetRequest.java | 3 +
.../client/cache/ClientCacheGetSizeRequest.java | 3 +
.../client/cache/ClientCachePutAllRequest.java | 3 +
.../cache/ClientCachePutIfAbsentRequest.java | 3 +
.../client/cache/ClientCachePutRequest.java | 3 +
.../cache/ClientCacheRemoveAllRequest.java | 3 +
.../cache/ClientCacheRemoveIfEqualsRequest.java | 3 +
.../cache/ClientCacheRemoveKeyRequest.java | 3 +
.../cache/ClientCacheRemoveKeysRequest.java | 3 +
.../ClientCacheReplaceIfEqualsRequest.java | 3 +
.../client/cache/ClientCacheReplaceRequest.java | 3 +
.../client/cache/ClientCacheRequest.java | 32 +
.../cache/ClientCacheScanQueryRequest.java | 3 +
.../cache/ClientCacheSqlFieldsQueryRequest.java | 18 +-
.../cache/ClientCacheSqlQueryRequest.java | 1 +
.../platform/services/PlatformServices.java | 8 +-
.../utils/PlatformConfigurationUtils.java | 20 +
.../processors/query/GridQueryProcessor.java | 9 +-
.../processors/query/GridQueryProperty.java | 14 +
.../internal/processors/query/QueryField.java | 26 +-
.../internal/processors/query/QueryUtils.java | 28 +-
.../query/property/QueryBinaryProperty.java | 23 +-
.../query/property/QueryClassProperty.java | 10 +
.../message/SchemaFinishDiscoveryMessage.java | 5 +
.../message/SchemaProposeDiscoveryMessage.java | 5 +
.../cluster/GridChangeStateCommandHandler.java | 3 +-
.../security/SecurityContextHolder.java | 53 +
.../service/GridServiceProcessor.java | 15 +-
.../processors/service/GridServiceProxy.java | 27 +-
.../processors/task/GridTaskProcessor.java | 2 +-
.../timeout/GridTimeoutProcessor.java | 105 +-
.../suggestions/OsConfigurationSuggestions.java | 14 +-
.../ignite/internal/util/IgniteUtils.java | 39 +-
.../ignite/internal/util/StripedExecutor.java | 69 +-
.../ignite/internal/util/nio/GridNioServer.java | 61 +-
.../util/nio/GridNioServerListener.java | 6 +
.../util/nio/GridNioServerListenerAdapter.java | 6 +
.../util/nio/ssl/BlockingSslHandler.java | 10 +-
.../apache/ignite/internal/util/typedef/X.java | 37 +-
.../visor/baseline/VisorBaselineTask.java | 24 +-
.../cache/VisorCacheLostPartitionsTask.java | 85 +
.../cache/VisorCacheLostPartitionsTaskArg.java | 73 +
.../VisorCacheLostPartitionsTaskResult.java | 74 +
.../VisorCacheResetLostPartitionsTask.java | 65 +
.../VisorCacheResetLostPartitionsTaskArg.java | 73 +
.../visor/util/VisorExceptionWrapper.java | 11 +-
.../worker/WorkersControlMXBeanImpl.java | 62 +
.../ignite/internal/worker/WorkersRegistry.java | 80 +
.../ignite/internal/worker/package-info.java | 22 +
.../ignite/mxbean/WorkersControlMXBean.java | 49 +
.../plugin/security/AuthenticationContext.java | 40 +
.../plugin/security/SecurityPermission.java | 11 +-
.../org/apache/ignite/spi/IgniteSpiAdapter.java | 10 +
.../org/apache/ignite/spi/IgniteSpiContext.java | 11 +
.../communication/tcp/TcpCommunicationSpi.java | 259 +-
.../tcp/internal/ConnectionKey.java | 117 +
.../TcpCommunicationConnectionCheckFuture.java | 519 ++
...pCommunicationNodeConnectionCheckFuture.java | 30 +
.../discovery/DiscoverySpiCustomMessage.java | 15 +-
...DiscoverySpiMutableCustomMessageSupport.java | 40 +
.../ignite/spi/discovery/tcp/ServerImpl.java | 63 +-
.../spi/discovery/tcp/TcpDiscoverySpi.java | 55 +-
.../tcp/internal/TcpDiscoveryNode.java | 32 +-
.../thread/IgniteStripedThreadPoolExecutor.java | 8 +-
.../ignite/thread/IgniteThreadFactory.java | 30 +-
.../ignite/thread/IgniteThreadPoolExecutor.java | 12 +-
.../ignite/thread/OomExceptionHandler.java | 44 +
.../resources/META-INF/classnames.properties | 2 +
.../ignite/GridSuppressedExceptionSelfTest.java | 23 +-
.../AffinityDistributionLoggingTest.java | 268 +
...unctionExcludeNeighborsAbstractSelfTest.java | 8 +-
.../failure/AbstractFailureHandlerTest.java | 74 +
.../failure/FailureHandlerTriggeredTest.java | 4 +
.../ignite/failure/IoomFailureHandlerTest.java | 144 +
.../ignite/failure/OomFailureHandlerTest.java | 255 +
.../internal/ClusterGroupHostsSelfTest.java | 3 +
.../ignite/internal/ClusterGroupSelfTest.java | 2 +
.../internal/ClusterNodeMetricsUpdateTest.java | 173 +
.../internal/DiscoverySpiTestListener.java | 162 +
.../ignite/internal/GridDiscoverySelfTest.java | 14 +-
.../GridJobMasterLeaveAwareSelfTest.java | 2 +
.../internal/GridJobStealingSelfTest.java | 2 +
.../internal/GridSameVmStartupSelfTest.java | 19 +-
.../apache/ignite/internal/GridSelfTest.java | 2 +
.../IgniteClientReconnectAbstractTest.java | 53 +-
.../IgniteClientReconnectApiExceptionTest.java | 21 +-
.../IgniteClientReconnectAtomicsTest.java | 30 +-
.../IgniteClientReconnectCacheTest.java | 49 +-
.../IgniteClientReconnectCollectionsTest.java | 14 +-
.../IgniteClientReconnectComputeTest.java | 6 +-
...eClientReconnectContinuousProcessorTest.java | 13 +-
...IgniteClientReconnectDiscoveryStateTest.java | 22 +-
...niteClientReconnectFailoverAbstractTest.java | 12 +-
.../IgniteClientReconnectServicesTest.java | 8 +-
.../internal/IgniteClientReconnectStopTest.java | 12 +-
.../IgniteClientReconnectStreamerTest.java | 4 +-
.../ignite/internal/IgniteClientRejoinTest.java | 3 +
.../IgniteComputeResultExceptionTest.java | 186 +
.../GridDiscoveryManagerAliveCacheSelfTest.java | 16 +-
.../GridAffinityProcessorAbstractSelfTest.java | 4 +-
...cheDhtLocalPartitionAfterRemoveSelfTest.java | 2 +-
.../cache/CacheGroupsMetricsRebalanceTest.java | 31 +-
.../CacheMetricsForClusterGroupSelfTest.java | 12 +-
.../DataStorageConfigurationValidationTest.java | 33 +-
.../cache/GridCacheAbstractFullApiSelfTest.java | 9 +-
.../cache/GridCacheAbstractSelfTest.java | 2 +
.../processors/cache/IgniteCacheGroupsTest.java | 1 +
.../cache/IgniteCacheNearLockValueSelfTest.java | 4 +-
.../IgniteCacheP2pUnmarshallingErrorTest.java | 11 +
.../IgniteClusterActivateDeactivateTest.java | 65 +
.../IgniteDaemonNodeMarshallerCacheTest.java | 3 +-
.../binary/BinaryMetadataUpdatesFlowTest.java | 12 +-
...ntNodeBinaryObjectMetadataMultinodeTest.java | 2 +-
.../GridCacheQueueClientDisconnectTest.java | 10 +
.../IgniteClientDataStructuresAbstractTest.java | 3 +-
...ExchangeLatchManagerCoordinatorFailTest.java | 244 +
.../CacheLateAffinityAssignmentTest.java | 127 +-
.../GridCacheNodeFailureAbstractTest.java | 5 +-
.../distributed/IgniteCache150ClientsTest.java | 2 +
.../distributed/IgniteCacheManyClientsTest.java | 44 +-
.../IgniteOptimisticTxSuspendResumeTest.java | 2 +
...ridCacheDhtPreloadMultiThreadedSelfTest.java | 4 +
.../dht/GridCacheDhtPreloadSelfTest.java | 2 +
.../GridCachePartitionsStateValidationTest.java | 316 ++
...idCachePartitionsStateValidatorSelfTest.java | 158 +
.../dht/TxRecoveryStoreEnabledTest.java | 15 +-
...titionedExplicitLockNodeFailureSelfTest.java | 3 +-
.../db/wal/IgniteWalFlushFailoverTest.java | 4 +-
...lFlushMultiNodeFailoverAbstractSelfTest.java | 4 +-
.../persistence/pagemem/PageMemoryImplTest.java | 9 +
.../persistence/tree/io/TrackingPageIOTest.java | 116 +-
.../ClientReconnectContinuousQueryTest.java | 19 +-
...yRemoteFilterMissingInClassPathSelfTest.java | 23 +-
...CacheContinuousQueryClientReconnectTest.java | 3 +
.../GridCacheWriteBehindStoreSelfTest.java | 41 +-
.../TxOptimisticOnPartitionExchangeTest.java | 322 ++
.../CacheVersionedEntryAbstractTest.java | 33 +-
.../closure/GridClosureSerializationTest.java | 2 +-
.../continuous/GridEventConsumeSelfTest.java | 34 +-
.../processors/database/BPlusTreeSelfTest.java | 29 +
.../service/ClosureServiceClientsNodesTest.java | 19 +-
.../GridServiceProcessorProxySelfTest.java | 12 +-
.../internal/util/GridTestClockTimer.java | 9 +
.../internal/util/StripedExecutorTest.java | 2 +-
.../GridMarshallerMappingConsistencyTest.java | 4 +
.../ignite/messaging/GridMessagingSelfTest.java | 126 +-
.../GridTcpCommunicationSpiAbstractTest.java | 71 +
.../TcpCommunicationSpiSkipMessageSendTest.java | 414 ++
.../FilterDataForClientNodeDiscoveryTest.java | 5 +
.../testframework/GridSpiTestContext.java | 10 +
.../ignite/testframework/GridTestUtils.java | 2 +-
.../config/GridTestProperties.java | 9 +
.../testframework/junits/GridAbstractTest.java | 129 +-
.../junits/multijvm/IgniteNodeRunner.java | 2 +
.../ignite/testsuites/IgniteBasicTestSuite.java | 4 +
.../ignite/testsuites/IgniteCacheTestSuite.java | 4 +
.../testsuites/IgniteCacheTestSuite5.java | 9 +-
.../testsuites/IgniteCacheTestSuite6.java | 6 +
.../testsuites/IgniteComputeGridTestSuite.java | 4 +
.../IgniteSpiCommunicationSelfTestSuite.java | 3 +
.../ignite/util/GridCommandHandlerTest.java | 43 +
.../query/h2/ddl/DdlStatementsProcessor.java | 22 +-
.../processors/query/h2/sql/GridSqlColumn.java | 17 +-
.../query/h2/sql/GridSqlCreateTable.java | 7 +-
.../query/h2/twostep/GridMapQueryExecutor.java | 7 +
.../query/h2/twostep/MapQueryLazyWorker.java | 13 +-
...niteCacheLockPartitionOnAffinityRunTest.java | 46 +-
...niteCacheDistributedQueryCancelSelfTest.java | 2 +-
.../DynamicIndexAbstractBasicSelfTest.java | 5 +-
.../cache/index/H2DynamicTableSelfTest.java | 37 +
.../cache/index/IgniteDecimalSelfTest.java | 265 +
.../h2/GridIndexingSpiAbstractSelfTest.java | 12 +-
.../IgniteCacheQuerySelfTestSuite.java | 2 +
.../GridJtaTransactionManagerSelfTest.java | 21 +-
.../GridPartitionedCacheJtaFactorySelfTest.java | 19 +-
.../main/java/org/apache/ignite/ml/Trainer.java | 3 -
.../apache/ignite/ml/genetic/Chromosome.java | 8 +-
.../apache/ignite/ml/genetic/CrossOverJob.java | 5 +-
.../apache/ignite/ml/genetic/CrossOverTask.java | 3 +-
.../apache/ignite/ml/genetic/FitnessJob.java | 6 +-
.../apache/ignite/ml/genetic/FitnessTask.java | 6 +-
.../org/apache/ignite/ml/genetic/GAGrid.java | 38 +-
.../java/org/apache/ignite/ml/genetic/Gene.java | 6 +-
.../ignite/ml/genetic/IFitnessFunction.java | 2 -
.../org/apache/ignite/ml/genetic/MutateJob.java | 10 +-
.../apache/ignite/ml/genetic/MutateTask.java | 19 +-
.../ignite/ml/genetic/TruncateSelectionJob.java | 11 +-
.../ml/genetic/TruncateSelectionTask.java | 64 +-
.../ml/genetic/cache/GeneCacheConfig.java | 2 -
.../ml/genetic/cache/PopulationCacheConfig.java | 2 -
.../ml/genetic/functions/GAGridFunction.java | 33 +-
.../genetic/parameter/ChromosomeCriteria.java | 7 +-
.../ml/genetic/parameter/GAConfiguration.java | 6 +-
.../ml/genetic/parameter/GAGridConstants.java | 4 +-
.../ignite/ml/genetic/utils/GAGridUtils.java | 10 +-
.../java/org/apache/ignite/ml/knn/KNNUtils.java | 59 +
.../KNNClassificationTrainer.java | 23 +-
.../ml/knn/regression/KNNRegressionModel.java | 87 +
.../ml/knn/regression/KNNRegressionTrainer.java | 40 +
.../ignite/ml/knn/regression/package-info.java | 22 +
.../org/apache/ignite/ml/nn/Activators.java | 20 +
.../org/apache/ignite/ml/nn/MLPTrainer.java | 46 +-
.../ml/preprocessing/PreprocessingTrainer.java | 41 +-
.../normalization/NormalizationTrainer.java | 35 +-
.../linear/FeatureExtractorWrapper.java | 55 +
.../linear/LinearRegressionLSQRTrainer.java | 38 +-
.../linear/LinearRegressionSGDTrainer.java | 118 +-
.../ignite/ml/trainers/DatasetTrainer.java | 46 +
.../org/apache/ignite/ml/tree/DecisionTree.java | 252 +
.../tree/DecisionTreeClassificationTrainer.java | 93 +
.../ml/tree/DecisionTreeConditionalNode.java | 78 +
.../ignite/ml/tree/DecisionTreeLeafNode.java | 48 +
.../apache/ignite/ml/tree/DecisionTreeNode.java | 26 +
.../ml/tree/DecisionTreeRegressionTrainer.java | 60 +
.../org/apache/ignite/ml/tree/TreeFilter.java | 38 +
.../ignite/ml/tree/data/DecisionTreeData.java | 128 +
.../ml/tree/data/DecisionTreeDataBuilder.java | 73 +
.../ignite/ml/tree/data/package-info.java | 22 +
.../ml/tree/impurity/ImpurityMeasure.java | 55 +
.../impurity/ImpurityMeasureCalculator.java | 38 +
.../tree/impurity/gini/GiniImpurityMeasure.java | 115 +
.../gini/GiniImpurityMeasureCalculator.java | 110 +
.../ml/tree/impurity/gini/package-info.java | 22 +
.../tree/impurity/mse/MSEImpurityMeasure.java | 133 +
.../mse/MSEImpurityMeasureCalculator.java | 80 +
.../ml/tree/impurity/mse/package-info.java | 22 +
.../ignite/ml/tree/impurity/package-info.java | 22 +
.../util/SimpleStepFunctionCompressor.java | 149 +
.../ml/tree/impurity/util/StepFunction.java | 162 +
.../impurity/util/StepFunctionCompressor.java | 55 +
.../ml/tree/impurity/util/package-info.java | 22 +
.../ml/tree/leaf/DecisionTreeLeafBuilder.java | 38 +
.../tree/leaf/MeanDecisionTreeLeafBuilder.java | 73 +
.../leaf/MostCommonDecisionTreeLeafBuilder.java | 86 +
.../ignite/ml/tree/leaf/package-info.java | 22 +
.../org/apache/ignite/ml/tree/package-info.java | 22 +
.../ignite/ml/trees/CategoricalRegionInfo.java | 72 -
.../ignite/ml/trees/CategoricalSplitInfo.java | 68 -
.../ignite/ml/trees/ContinuousRegionInfo.java | 74 -
.../ml/trees/ContinuousSplitCalculator.java | 51 -
.../org/apache/ignite/ml/trees/RegionInfo.java | 62 -
.../ml/trees/models/DecisionTreeModel.java | 44 -
.../ignite/ml/trees/models/package-info.java | 22 -
.../ml/trees/nodes/CategoricalSplitNode.java | 50 -
.../ml/trees/nodes/ContinuousSplitNode.java | 56 -
.../ignite/ml/trees/nodes/DecisionTreeNode.java | 33 -
.../org/apache/ignite/ml/trees/nodes/Leaf.java | 49 -
.../apache/ignite/ml/trees/nodes/SplitNode.java | 100 -
.../ignite/ml/trees/nodes/package-info.java | 22 -
.../apache/ignite/ml/trees/package-info.java | 22 -
.../ml/trees/trainers/columnbased/BiIndex.java | 113 -
...exedCacheColumnDecisionTreeTrainerInput.java | 57 -
.../CacheColumnDecisionTreeTrainerInput.java | 141 -
.../columnbased/ColumnDecisionTreeTrainer.java | 568 --
.../ColumnDecisionTreeTrainerInput.java | 55 -
.../MatrixColumnDecisionTreeTrainerInput.java | 83 -
.../trainers/columnbased/RegionProjection.java | 109 -
.../trainers/columnbased/TrainingContext.java | 166 -
.../columnbased/caches/ContextCache.java | 68 -
.../columnbased/caches/FeaturesCache.java | 151 -
.../columnbased/caches/ProjectionsCache.java | 286 --
.../trainers/columnbased/caches/SplitCache.java | 206 -
.../columnbased/caches/package-info.java | 22 -
.../ContinuousSplitCalculators.java | 34 -
.../contsplitcalcs/GiniSplitCalculator.java | 234 -
.../contsplitcalcs/VarianceSplitCalculator.java | 179 -
.../contsplitcalcs/package-info.java | 22 -
.../trainers/columnbased/package-info.java | 22 -
.../columnbased/regcalcs/RegionCalculators.java | 85 -
.../columnbased/regcalcs/package-info.java | 22 -
.../vectors/CategoricalFeatureProcessor.java | 212 -
.../vectors/ContinuousFeatureProcessor.java | 111 -
.../vectors/ContinuousSplitInfo.java | 71 -
.../columnbased/vectors/FeatureProcessor.java | 82 -
.../vectors/FeatureVectorProcessorUtils.java | 57 -
.../columnbased/vectors/SampleInfo.java | 80 -
.../trainers/columnbased/vectors/SplitInfo.java | 106 -
.../columnbased/vectors/package-info.java | 22 -
.../org/apache/ignite/ml/IgniteMLTestSuite.java | 4 +-
.../ml/genetic/GAGridCalculateFitnessTest.java | 6 +-
.../genetic/GAGridInitializePopulationTest.java | 7 +-
.../ignite/ml/knn/KNNClassificationTest.java | 20 +-
.../apache/ignite/ml/knn/KNNRegressionTest.java | 143 +
.../org/apache/ignite/ml/knn/KNNTestSuite.java | 1 +
.../ignite/ml/nn/MLPTrainerIntegrationTest.java | 14 +-
.../org/apache/ignite/ml/nn/MLPTrainerTest.java | 22 +-
.../MLPTrainerMnistIntegrationTest.java | 7 +-
.../ml/nn/performance/MLPTrainerMnistTest.java | 11 +-
.../ml/nn/performance/MnistMLPTestUtil.java | 9 +-
.../normalization/NormalizationTrainerTest.java | 10 +-
.../ml/regressions/RegressionsTestSuite.java | 15 +-
...stributedLinearRegressionSGDTrainerTest.java | 35 -
...stributedLinearRegressionSGDTrainerTest.java | 35 -
...wareAbstractLinearRegressionTrainerTest.java | 3 +
.../linear/LinearRegressionLSQRTrainerTest.java | 14 +-
.../linear/LinearRegressionSGDTrainerTest.java | 94 +
.../LocalLinearRegressionSGDTrainerTest.java | 35 -
.../ignite/ml/svm/SVMBinaryTrainerTest.java | 11 +-
.../ignite/ml/svm/SVMMultiClassTrainerTest.java | 11 +-
...reeClassificationTrainerIntegrationTest.java | 101 +
.../DecisionTreeClassificationTrainerTest.java | 87 +
...ionTreeRegressionTrainerIntegrationTest.java | 101 +
.../tree/DecisionTreeRegressionTrainerTest.java | 87 +
.../ignite/ml/tree/DecisionTreeTestSuite.java | 48 +
.../ml/tree/data/DecisionTreeDataTest.java | 59 +
.../gini/GiniImpurityMeasureCalculatorTest.java | 103 +
.../impurity/gini/GiniImpurityMeasureTest.java | 131 +
.../mse/MSEImpurityMeasureCalculatorTest.java | 59 +
.../impurity/mse/MSEImpurityMeasureTest.java | 109 +
.../util/SimpleStepFunctionCompressorTest.java | 75 +
.../ml/tree/impurity/util/StepFunctionTest.java | 71 +
.../tree/impurity/util/TestImpurityMeasure.java | 88 +
.../DecisionTreeMNISTIntegrationTest.java | 106 +
.../tree/performance/DecisionTreeMNISTTest.java | 75 +
.../ignite/ml/trees/BaseDecisionTreeTest.java | 70 -
.../ml/trees/ColumnDecisionTreeTrainerTest.java | 191 -
.../ignite/ml/trees/DecisionTreesTestSuite.java | 33 -
.../ml/trees/GiniSplitCalculatorTest.java | 141 -
.../ignite/ml/trees/SplitDataGenerator.java | 390 --
.../ml/trees/VarianceSplitCalculatorTest.java | 84 -
.../ColumnDecisionTreeTrainerBenchmark.java | 456 --
.../IgniteSessionStateStoreProviderTest.cs | 7 +-
.../Apache.Ignite.Benchmarks/BenchmarkRunner.cs | 1 -
.../ApiParity/IgniteConfigurationParityTest.cs | 3 +-
.../QueryEntityConfigurationParityTest.cs | 5 +-
.../Binary/BinaryDynamicRegistrationTest.cs | 2 +-
.../Cache/CacheAbstractTransactionalTest.cs | 5 +-
.../Cache/CacheConfigurationTest.cs | 2 +
.../Client/ClientConnectionTest.cs | 164 +-
.../Apache.Ignite.Core.Tests/EventsTest.cs | 9 +-
.../Apache.Ignite.Core.Tests/ExceptionsTest.cs | 3 +-
.../IgniteStartStopTest.cs | 5 +-
.../Apache.Ignite.Core.Tests/MessagingTest.cs | 5 +-
.../Apache.Ignite.Core.csproj | 1 +
.../Cache/Configuration/QueryEntity.cs | 4 +-
.../Cache/Configuration/QueryField.cs | 16 +-
.../Configuration/QuerySqlFieldAttribute.cs | 10 +
.../Client/ClientStatusCode.cs | 12 +-
.../Client/IgniteClientConfiguration.cs | 13 +
.../IgniteClientConfigurationSection.xsd | 10 +
.../IgniteConfigurationSection.xsd | 10 +
.../Impl/Client/ClientProtocolVersion.cs | 22 +-
.../Impl/Client/ClientSocket.cs | 89 +-
.../Impl/Common/TaskRunner.cs | 70 +
.../Impl/Datastream/DataStreamerBatch.cs | 2 +-
.../Impl/Datastream/DataStreamerImpl.cs | 2 +-
.../Apache.Ignite.Core/Impl/Events/Events.cs | 2 +-
.../Impl/Transactions/TransactionImpl.cs | 3 +-
.../org/apache/ignite/spark/IgniteRDD.scala | 9 +-
.../jdbc/CacheJdbcPojoStoreFactorySelfTest.java | 11 +-
.../ignite/internal/GridFactorySelfTest.java | 3 +-
.../resource/GridServiceInjectionSelfTest.java | 64 +-
.../GridSpringResourceInjectionSelfTest.java | 58 +-
.../p2p/GridP2PUserVersionChangeSelfTest.java | 5 +-
.../commands/cache/VisorCacheCommand.scala | 34 +-
.../cache/VisorCacheLostPartitionsCommand.scala | 170 +
.../VisorCacheResetLostPartitionsCommand.scala | 132 +
modules/web-console/backend/routes/demo.js | 2 +
modules/web-console/frontend/app/app.config.js | 14 +-
modules/web-console/frontend/app/app.js | 2 -
.../components/cache-edit-form/template.tpl.pug | 22 +-
.../cache-edit-form/templates/affinity.pug | 86 +
.../cache-edit-form/templates/concurrency.pug | 64 +
.../cache-edit-form/templates/general.pug | 113 +
.../cache-edit-form/templates/memory.pug | 158 +
.../templates/near-cache-client.pug | 50 +
.../templates/near-cache-server.pug | 51 +
.../cache-edit-form/templates/node-filter.pug | 53 +
.../cache-edit-form/templates/query.pug | 114 +
.../cache-edit-form/templates/rebalance.pug | 66 +
.../cache-edit-form/templates/statistics.pug | 34 +
.../cache-edit-form/templates/store.pug | 310 ++
.../cluster-edit-form/template.tpl.pug | 62 +-
.../cluster-edit-form/templates/atomic.pug | 75 +
.../cluster-edit-form/templates/attributes.pug | 40 +
.../cluster-edit-form/templates/binary.pug | 80 +
.../templates/cache-key-cfg.pug | 63 +
.../cluster-edit-form/templates/checkpoint.pug | 82 +
.../templates/checkpoint/fs.pug | 36 +
.../templates/checkpoint/jdbc.pug | 47 +
.../templates/checkpoint/s3.pug | 204 +
.../templates/client-connector.pug | 76 +
.../cluster-edit-form/templates/collision.pug | 58 +
.../templates/collision/custom.pug | 23 +
.../templates/collision/fifo-queue.pug | 26 +
.../templates/collision/job-stealing.pug | 51 +
.../templates/collision/priority-queue.pug | 41 +
.../templates/communication.pug | 134 +
.../cluster-edit-form/templates/connector.pug | 100 +
.../templates/data-storage.pug | 301 ++
.../cluster-edit-form/templates/deployment.pug | 192 +
.../cluster-edit-form/templates/discovery.pug | 97 +
.../cluster-edit-form/templates/events.pug | 66 +
.../cluster-edit-form/templates/failover.pug | 89 +
.../cluster-edit-form/templates/general.pug | 89 +
.../templates/general/discovery/cloud.pug | 78 +
.../templates/general/discovery/google.pug | 38 +
.../templates/general/discovery/jdbc.pug | 35 +
.../templates/general/discovery/kubernetes.pug | 38 +
.../templates/general/discovery/multicast.pug | 63 +
.../templates/general/discovery/s3.pug | 38 +
.../templates/general/discovery/shared.pug | 24 +
.../templates/general/discovery/vm.pug | 55 +
.../templates/general/discovery/zookeeper.pug | 84 +
.../retrypolicy/bounded-exponential-backoff.pug | 26 +
.../discovery/zookeeper/retrypolicy/custom.pug | 25 +
.../retrypolicy/exponential-backoff.pug | 26 +
.../discovery/zookeeper/retrypolicy/forever.pug | 23 +
.../discovery/zookeeper/retrypolicy/n-times.pug | 24 +
.../zookeeper/retrypolicy/one-time.pug | 23 +
.../zookeeper/retrypolicy/until-elapsed.pug | 24 +
.../cluster-edit-form/templates/hadoop.pug | 87 +
.../cluster-edit-form/templates/igfs.pug | 34 +
.../templates/load-balancing.pug | 115 +
.../cluster-edit-form/templates/logger.pug | 60 +
.../templates/logger/custom.pug | 24 +
.../templates/logger/log4j.pug | 49 +
.../templates/logger/log4j2.pug | 38 +
.../cluster-edit-form/templates/marshaller.pug | 75 +
.../cluster-edit-form/templates/memory.pug | 195 +
.../cluster-edit-form/templates/metrics.pug | 46 +
.../cluster-edit-form/templates/misc.pug | 58 +
.../cluster-edit-form/templates/odbc.pug | 70 +
.../cluster-edit-form/templates/persistence.pug | 82 +
.../cluster-edit-form/templates/service.pug | 89 +
.../templates/sql-connector.pug | 58 +
.../cluster-edit-form/templates/ssl.pug | 89 +
.../cluster-edit-form/templates/swap.pug | 74 +
.../cluster-edit-form/templates/thread.pug | 144 +
.../cluster-edit-form/templates/time.pug | 44 +
.../templates/transactions.pug | 65 +
.../components/igfs-edit-form/template.tpl.pug | 12 +-
.../igfs-edit-form/templates/dual.pug | 42 +
.../igfs-edit-form/templates/fragmentizer.pug | 37 +
.../igfs-edit-form/templates/general.pug | 72 +
.../components/igfs-edit-form/templates/ipc.pug | 55 +
.../igfs-edit-form/templates/misc.pug | 110 +
.../igfs-edit-form/templates/secondary.pug | 55 +
.../components/model-edit-form/template.tpl.pug | 6 +-
.../model-edit-form/templates/general.pug | 57 +
.../model-edit-form/templates/query.pug | 255 +
.../model-edit-form/templates/store.pug | 123 +
.../page-configure-basic/template.pug | 18 +-
.../components/modal-import-models/component.js | 4 +-
.../components/preview-panel/directive.js | 246 +
.../components/preview-panel/index.js | 23 +
.../app/components/page-configure/index.js | 25 +-
.../services/ConfigurationResource.js | 49 +
.../page-configure/services/SummaryZipper.js | 44 +
.../page-configure/services/summary.worker.js | 147 +
.../app/components/page-configure/states.js | 270 +
.../app/components/page-profile/controller.js | 4 +-
.../frontend/app/modules/ace.module.js | 47 +-
.../app/modules/states/configuration.state.js | 297 --
.../configuration/Configuration.resource.js | 42 -
.../states/configuration/caches/affinity.pug | 86 -
.../states/configuration/caches/concurrency.pug | 64 -
.../states/configuration/caches/general.pug | 113 -
.../states/configuration/caches/memory.pug | 158 -
.../configuration/caches/near-cache-client.pug | 50 -
.../configuration/caches/near-cache-server.pug | 51 -
.../states/configuration/caches/node-filter.pug | 53 -
.../states/configuration/caches/query.pug | 114 -
.../states/configuration/caches/rebalance.pug | 66 -
.../states/configuration/caches/statistics.pug | 34 -
.../states/configuration/caches/store.pug | 310 --
.../states/configuration/clusters/atomic.pug | 75 -
.../configuration/clusters/attributes.pug | 40 -
.../states/configuration/clusters/binary.pug | 80 -
.../configuration/clusters/cache-key-cfg.pug | 63 -
.../configuration/clusters/checkpoint.pug | 82 -
.../configuration/clusters/checkpoint/fs.pug | 36 -
.../configuration/clusters/checkpoint/jdbc.pug | 47 -
.../configuration/clusters/checkpoint/s3.pug | 204 -
.../configuration/clusters/client-connector.pug | 76 -
.../states/configuration/clusters/collision.pug | 58 -
.../configuration/clusters/collision/custom.pug | 23 -
.../clusters/collision/fifo-queue.pug | 26 -
.../clusters/collision/job-stealing.pug | 51 -
.../clusters/collision/priority-queue.pug | 41 -
.../configuration/clusters/communication.pug | 134 -
.../states/configuration/clusters/connector.pug | 100 -
.../configuration/clusters/data-storage.pug | 301 --
.../configuration/clusters/deployment.pug | 192 -
.../states/configuration/clusters/discovery.pug | 97 -
.../states/configuration/clusters/events.pug | 66 -
.../states/configuration/clusters/failover.pug | 89 -
.../states/configuration/clusters/general.pug | 89 -
.../clusters/general/discovery/cloud.pug | 78 -
.../clusters/general/discovery/google.pug | 38 -
.../clusters/general/discovery/jdbc.pug | 35 -
.../clusters/general/discovery/kubernetes.pug | 38 -
.../clusters/general/discovery/multicast.pug | 63 -
.../clusters/general/discovery/s3.pug | 38 -
.../clusters/general/discovery/shared.pug | 24 -
.../clusters/general/discovery/vm.pug | 55 -
.../clusters/general/discovery/zookeeper.pug | 84 -
.../retrypolicy/bounded-exponential-backoff.pug | 26 -
.../discovery/zookeeper/retrypolicy/custom.pug | 25 -
.../retrypolicy/exponential-backoff.pug | 26 -
.../discovery/zookeeper/retrypolicy/forever.pug | 23 -
.../discovery/zookeeper/retrypolicy/n-times.pug | 24 -
.../zookeeper/retrypolicy/one-time.pug | 23 -
.../zookeeper/retrypolicy/until-elapsed.pug | 24 -
.../states/configuration/clusters/hadoop.pug | 87 -
.../states/configuration/clusters/igfs.pug | 34 -
.../configuration/clusters/load-balancing.pug | 115 -
.../states/configuration/clusters/logger.pug | 60 -
.../configuration/clusters/logger/custom.pug | 24 -
.../configuration/clusters/logger/log4j.pug | 49 -
.../configuration/clusters/logger/log4j2.pug | 38 -
.../configuration/clusters/marshaller.pug | 75 -
.../states/configuration/clusters/memory.pug | 195 -
.../states/configuration/clusters/metrics.pug | 46 -
.../states/configuration/clusters/misc.pug | 58 -
.../states/configuration/clusters/odbc.pug | 70 -
.../configuration/clusters/persistence.pug | 82 -
.../states/configuration/clusters/service.pug | 89 -
.../configuration/clusters/sql-connector.pug | 58 -
.../states/configuration/clusters/ssl.pug | 89 -
.../states/configuration/clusters/swap.pug | 74 -
.../states/configuration/clusters/thread.pug | 144 -
.../states/configuration/clusters/time.pug | 44 -
.../configuration/clusters/transactions.pug | 65 -
.../states/configuration/domains/general.pug | 57 -
.../states/configuration/domains/query.pug | 255 -
.../states/configuration/domains/store.pug | 123 -
.../modules/states/configuration/igfs/dual.pug | 42 -
.../states/configuration/igfs/fragmentizer.pug | 37 -
.../states/configuration/igfs/general.pug | 72 -
.../modules/states/configuration/igfs/ipc.pug | 55 -
.../modules/states/configuration/igfs/misc.pug | 110 -
.../states/configuration/igfs/secondary.pug | 55 -
.../configuration/preview-panel.directive.js | 239 -
.../summary/summary-zipper.service.js | 39 -
.../configuration/summary/summary.worker.js | 147 -
.../services/AngularStrapSelect.decorator.js | 5 +-
.../services/AngularStrapTooltip.decorator.js | 8 +-
.../frontend/app/services/FormUtils.service.js | 3 +-
modules/web-console/frontend/package-lock.json | 929 ++--
.../demo/service/DemoCachesLoadService.java | 22 +-
modules/yardstick/pom-standalone.xml | 6 +
modules/yardstick/pom.xml | 6 +
.../IgniteColumnDecisionTreeGiniBenchmark.java | 70 -
...niteColumnDecisionTreeVarianceBenchmark.java | 71 -
.../yardstick/ml/trees/SplitDataGenerator.java | 426 --
.../ignite/yardstick/ml/trees/package-info.java | 22 -
modules/zookeeper/pom.xml | 40 +
.../spi/discovery/zk/ZookeeperDiscoverySpi.java | 557 ++
.../zk/internal/ZkAbstractCallabck.java | 83 +
.../zk/internal/ZkAbstractChildrenCallback.java | 61 +
.../zk/internal/ZkAbstractWatcher.java | 55 +
.../discovery/zk/internal/ZkAliveNodeData.java | 40 +
.../zk/internal/ZkBulkJoinContext.java | 50 +
.../discovery/zk/internal/ZkClusterNodes.java | 103 +
.../internal/ZkCommunicationErrorNodeState.java | 46 +
.../ZkCommunicationErrorProcessFuture.java | 411 ++
...kCommunicationErrorResolveFinishMessage.java | 69 +
.../ZkCommunicationErrorResolveResult.java | 45 +
...ZkCommunicationErrorResolveStartMessage.java | 61 +
.../internal/ZkCommunicationFailureContext.java | 188 +
.../zk/internal/ZkDiscoveryCustomEventData.java | 89 +
.../zk/internal/ZkDiscoveryEventData.java | 165 +
.../zk/internal/ZkDiscoveryEventsData.java | 121 +
.../internal/ZkDiscoveryNodeFailEventData.java | 55 +
.../internal/ZkDiscoveryNodeJoinEventData.java | 60 +
.../ZkDistributedCollectDataFuture.java | 250 +
.../zk/internal/ZkForceNodeFailMessage.java | 65 +
.../discovery/zk/internal/ZkIgnitePaths.java | 307 ++
.../zk/internal/ZkInternalJoinErrorMessage.java | 44 +
.../zk/internal/ZkInternalMessage.java | 27 +
.../zk/internal/ZkJoinEventDataForJoined.java | 83 +
.../zk/internal/ZkJoinedNodeEvtData.java | 79 +
.../zk/internal/ZkJoiningNodeData.java | 87 +
.../zk/internal/ZkNoServersMessage.java | 50 +
.../zk/internal/ZkNodeValidateResult.java | 43 +
.../spi/discovery/zk/internal/ZkRunnable.java | 51 +
.../discovery/zk/internal/ZkRuntimeState.java | 135 +
.../discovery/zk/internal/ZkTimeoutObject.java | 54 +
.../discovery/zk/internal/ZookeeperClient.java | 1219 +++++
.../ZookeeperClientFailedException.java | 40 +
.../zk/internal/ZookeeperClusterNode.java | 362 ++
.../zk/internal/ZookeeperDiscoveryImpl.java | 4464 ++++++++++++++++
.../java/org/apache/ZookeeperNodeStart.java | 46 +
...CacheEntryListenerWithZkDiscoAtomicTest.java | 32 +
.../ZookeeperDiscoverySpiAbstractTestSuite.java | 118 +
.../zk/ZookeeperDiscoverySpiTestSuite1.java | 44 +
.../zk/ZookeeperDiscoverySpiTestSuite2.java | 94 +
...ZookeeperDiscoverySuitePreprocessorTest.java | 101 +
.../zk/internal/ZookeeperClientTest.java | 495 ++
...okeeperDiscoverySpiSaslAuthAbstractTest.java | 247 +
...ZookeeperDiscoverySpiSaslFailedAuthTest.java | 44 +
...eeperDiscoverySpiSaslSuccessfulAuthTest.java | 48 +
.../zk/internal/ZookeeperDiscoverySpiTest.java | 4847 ++++++++++++++++++
.../zookeeper/ZkTestClientCnxnSocketNIO.java | 137 +
parent/pom.xml | 4 +
771 files changed, 41784 insertions(+), 15883 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ecefdd33/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ecefdd33/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
----------------------------------------------------------------------
[27/54] [abbrv] ignite git commit: IGNITE-7871 Implemented additional
synchronization phase for correct partition counters update
Posted by ag...@apache.org.
IGNITE-7871 Implemented additional synchronization phase for correct partition counters update
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/da77b981
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/da77b981
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/da77b981
Branch: refs/heads/ignite-6083
Commit: da77b9818a70495b7afdf6899ebd9180dadd7f68
Parents: f4de6df
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Wed Apr 11 11:23:46 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Apr 11 11:23:46 2018 +0300
----------------------------------------------------------------------
.../org/apache/ignite/internal/GridTopic.java | 5 +-
.../communication/GridIoMessageFactory.java | 6 +
.../discovery/GridDiscoveryManager.java | 10 +
.../MetaPageUpdatePartitionDataRecord.java | 2 +-
.../processors/cache/CacheMetricsImpl.java | 2 +-
.../processors/cache/GridCacheMvccManager.java | 38 +
.../GridCachePartitionExchangeManager.java | 17 +
.../cache/GridCacheSharedContext.java | 9 +-
.../processors/cache/GridCacheUtils.java | 2 +-
.../cache/IgniteCacheOffheapManager.java | 8 +-
.../cache/IgniteCacheOffheapManagerImpl.java | 10 +-
.../dht/GridClientPartitionTopology.java | 5 +
.../distributed/dht/GridDhtLocalPartition.java | 9 +-
.../dht/GridDhtPartitionTopology.java | 6 +
.../dht/GridDhtPartitionTopologyImpl.java | 26 +-
.../dht/GridDhtPartitionsStateValidator.java | 255 +++++++
.../cache/distributed/dht/GridDhtTxLocal.java | 5 +
.../GridDhtPartitionsExchangeFuture.java | 96 ++-
.../GridDhtPartitionsSingleMessage.java | 68 +-
.../dht/preloader/InitNewCoordinatorFuture.java | 2 +-
.../preloader/latch/ExchangeLatchManager.java | 695 +++++++++++++++++++
.../distributed/dht/preloader/latch/Latch.java | 52 ++
.../dht/preloader/latch/LatchAckMessage.java | 165 +++++
.../cache/distributed/near/GridNearTxLocal.java | 10 +
.../persistence/GridCacheOffheapManager.java | 10 +-
.../cache/transactions/IgniteTxAdapter.java | 2 +-
.../cache/transactions/IgniteTxManager.java | 36 +-
...cheDhtLocalPartitionAfterRemoveSelfTest.java | 2 +-
.../processors/cache/IgniteCacheGroupsTest.java | 1 +
...ExchangeLatchManagerCoordinatorFailTest.java | 244 +++++++
.../GridCachePartitionsStateValidationTest.java | 316 +++++++++
...idCachePartitionsStateValidatorSelfTest.java | 158 +++++
.../TxOptimisticOnPartitionExchangeTest.java | 322 +++++++++
.../ignite/testsuites/IgniteCacheTestSuite.java | 4 +
.../testsuites/IgniteCacheTestSuite6.java | 6 +
35 files changed, 2568 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index 1227e8c..0b2d41a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -124,7 +124,10 @@ public enum GridTopic {
TOPIC_METRICS,
/** */
- TOPIC_AUTH;
+ TOPIC_AUTH,
+
+ /** */
+ TOPIC_EXCHANGE;
/** Enum values. */
private static final GridTopic[] VALS = values();
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 5616fd0..581c32e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -53,6 +53,7 @@ import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
import org.apache.ignite.internal.processors.cache.WalStateAckMessage;
import org.apache.ignite.internal.processors.cache.binary.MetadataRequestMessage;
import org.apache.ignite.internal.processors.cache.binary.MetadataResponseMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.LatchAckMessage;
import org.apache.ignite.internal.processors.cache.distributed.GridCacheTtlUpdateRequest;
import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryRequest;
import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryResponse;
@@ -921,6 +922,11 @@ public class GridIoMessageFactory implements MessageFactory {
break;
+ case 135:
+ msg = new LatchAckMessage();
+
+ break;
+
// [-3..119] [124..129] [-23..-27] [-36..-55]- this
// [120..123] - DR
// [-4..-22, -30..-35] - SQL
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index a1d84e5..400bb5f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -793,6 +793,9 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
((IgniteKernal)ctx.grid()).onDisconnected();
+ if (!locJoin.isDone())
+ locJoin.onDone(new IgniteCheckedException("Node disconnected"));
+
locJoin = new GridFutureAdapter<>();
registeredCaches.clear();
@@ -2142,6 +2145,13 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
}
/**
+ * @return Local join future.
+ */
+ public GridFutureAdapter<DiscoveryLocalJoinData> localJoinFuture() {
+ return locJoin;
+ }
+
+ /**
* @param msg Custom message.
* @throws IgniteCheckedException If failed.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
index bafbf47..e5bd343 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
@@ -32,7 +32,7 @@ public class MetaPageUpdatePartitionDataRecord extends PageDeltaRecord {
/** */
private long globalRmvId;
- /** */
+ /** TODO: Partition size may be long */
private int partSize;
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
index 6fae8fe..b402ff2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
@@ -792,7 +792,7 @@ public class CacheMetricsImpl implements CacheMetrics {
if (cctx.cache() == null)
continue;
- int cacheSize = part.dataStore().cacheSize(cctx.cacheId());
+ long cacheSize = part.dataStore().cacheSize(cctx.cacheId());
offHeapEntriesCnt += cacheSize;
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
index a9fa3c7..fade833 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
@@ -44,6 +44,8 @@ import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.distributed.GridCacheMappedVersion;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishFuture;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -314,6 +316,42 @@ public class GridCacheMvccManager extends GridCacheSharedManagerAdapter {
}
/**
+ * Creates a future that will wait for finishing all remote transactions (primary -> backup)
+ * with topology version less or equal to {@code topVer}.
+ *
+ * @param topVer Topology version.
+ * @return Compound future of all {@link GridDhtTxFinishFuture} futures.
+ */
+ public IgniteInternalFuture<?> finishRemoteTxs(AffinityTopologyVersion topVer) {
+ GridCompoundFuture<?, ?> res = new CacheObjectsReleaseFuture<>("RemoteTx", topVer);
+
+ for (GridCacheFuture<?> fut : futs.values()) {
+ if (fut instanceof GridDhtTxFinishFuture) {
+ GridDhtTxFinishFuture finishTxFuture = (GridDhtTxFinishFuture) fut;
+
+ if (cctx.tm().needWaitTransaction(finishTxFuture.tx(), topVer))
+ res.add(ignoreErrors(finishTxFuture));
+ }
+ }
+
+ res.markInitialized();
+
+ return res;
+ }
+
+ /**
+ * Future wrapper which ignores any underlying future errors.
+ *
+ * @param f Underlying future.
+ * @return Future wrapper which ignore any underlying future errors.
+ */
+ private IgniteInternalFuture ignoreErrors(IgniteInternalFuture<?> f) {
+ GridFutureAdapter<?> wrapper = new GridFutureAdapter();
+ f.listen(future -> wrapper.onDone());
+ return wrapper;
+ }
+
+ /**
* @param leftNodeId Left node ID.
* @param topVer Topology version.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 1a0e65f..20a3ccb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -64,6 +64,7 @@ import org.apache.ignite.internal.managers.discovery.DiscoveryLocalJoinData;
import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
@@ -216,6 +217,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
/** For tests only. */
private volatile AffinityTopologyVersion exchMergeTestWaitVer;
+ /** Distributed latch manager. */
+ private ExchangeLatchManager latchMgr;
+
/** Discovery listener. */
private final DiscoveryEventListener discoLsnr = new DiscoveryEventListener() {
@Override public void onEvent(DiscoveryEvent evt, DiscoCache cache) {
@@ -309,6 +313,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
exchWorker = new ExchangeWorker();
+ latchMgr = new ExchangeLatchManager(cctx.kernalContext());
+
cctx.gridEvents().addDiscoveryEventListener(discoLsnr, EVT_NODE_JOINED, EVT_NODE_LEFT, EVT_NODE_FAILED,
EVT_DISCOVERY_CUSTOM_EVT);
@@ -1255,6 +1261,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
m.addPartitionUpdateCounters(grp.groupId(),
newCntrMap ? cntrsMap : CachePartitionPartialCountersMap.toCountersMap(cntrsMap));
+
+ m.addPartitionSizes(grp.groupId(), grp.topology().partitionSizes());
}
}
}
@@ -1277,6 +1285,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
m.addPartitionUpdateCounters(top.groupId(),
newCntrMap ? cntrsMap : CachePartitionPartialCountersMap.toCountersMap(cntrsMap));
+
+ m.addPartitionSizes(top.groupId(), top.partitionSizes());
}
}
@@ -1570,6 +1580,13 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
}
/**
+ * @return Latch manager instance.
+ */
+ public ExchangeLatchManager latch() {
+ return latchMgr;
+ }
+
+ /**
* @param exchFut Optional current exchange future.
* @throws Exception If failed.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index c2f9229..b3b4f0d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -26,7 +26,6 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerArray;
-import java.util.function.BiFunction;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
@@ -711,7 +710,7 @@ public class GridCacheSharedContext<K, V> {
/**
* @return Ttl cleanup manager.
- * */
+ */
public GridCacheSharedTtlCleanupManager ttl() {
return ttlMgr;
}
@@ -854,10 +853,14 @@ public class GridCacheSharedContext<K, V> {
GridCompoundFuture f = new CacheObjectsReleaseFuture("Partition", topVer);
f.add(mvcc().finishExplicitLocks(topVer));
- f.add(tm().finishTxs(topVer));
f.add(mvcc().finishAtomicUpdates(topVer));
f.add(mvcc().finishDataStreamerUpdates(topVer));
+ IgniteInternalFuture<?> finishLocalTxsFuture = tm().finishLocalTxs(topVer);
+ // To properly track progress of finishing local tx updates we explicitly add this future to compound set.
+ f.add(finishLocalTxsFuture);
+ f.add(tm().finishAllTxs(finishLocalTxsFuture, topVer));
+
f.markInitialized();
return f;
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index a5169d2..d672420 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -1732,7 +1732,7 @@ public class GridCacheUtils {
ver,
expiryPlc == null ? 0 : expiryPlc.forCreate(),
expiryPlc == null ? 0 : toExpireTime(expiryPlc.forCreate()),
- false,
+ true,
topVer,
GridDrType.DR_BACKUP,
true);
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
index 3d83f87..a12c033 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
@@ -22,11 +22,11 @@ import javax.cache.Cache;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.persistence.RootPage;
import org.apache.ignite.internal.processors.cache.persistence.RowStore;
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.query.GridQueryRowCacheCleaner;
import org.apache.ignite.internal.util.GridAtomicLong;
@@ -344,7 +344,7 @@ public interface IgniteCacheOffheapManager {
* @param part Partition.
* @return Number of entries.
*/
- public int totalPartitionEntriesCount(int part);
+ public long totalPartitionEntriesCount(int part);
/**
*
@@ -381,7 +381,7 @@ public interface IgniteCacheOffheapManager {
* @param cacheId Cache ID.
* @return Size.
*/
- int cacheSize(int cacheId);
+ long cacheSize(int cacheId);
/**
* @return Cache sizes if store belongs to group containing multiple caches.
@@ -391,7 +391,7 @@ public interface IgniteCacheOffheapManager {
/**
* @return Total size.
*/
- int fullSize();
+ long fullSize();
/**
* @return Update counter.
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index b201935..f8cc86f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -252,7 +252,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
}
/** {@inheritDoc} */
- @Override public int totalPartitionEntriesCount(int p) {
+ @Override public long totalPartitionEntriesCount(int p) {
if (grp.isLocal())
return locCacheDataStore.fullSize();
else {
@@ -1152,14 +1152,14 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
}
/** {@inheritDoc} */
- @Override public int cacheSize(int cacheId) {
+ @Override public long cacheSize(int cacheId) {
if (grp.sharedGroup()) {
AtomicLong size = cacheSizes.get(cacheId);
return size != null ? (int)size.get() : 0;
}
- return (int)storageSize.get();
+ return storageSize.get();
}
/** {@inheritDoc} */
@@ -1176,8 +1176,8 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
}
/** {@inheritDoc} */
- @Override public int fullSize() {
- return (int)storageSize.get();
+ @Override public long fullSize() {
+ return storageSize.get();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index 5bbbb31..3e3bb0d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -1196,6 +1196,11 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
+ @Override public Map<Integer, Long> partitionSizes() {
+ return Collections.emptyMap();
+ }
+
+ /** {@inheritDoc} */
@Override public boolean rebalanceFinished(AffinityTopologyVersion topVer) {
assert false : "Should not be called on non-affinity node";
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 7a47f31..ea20dbf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -929,7 +929,7 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
/**
* @return Initial update counter.
*/
- public Long initialUpdateCounter() {
+ public long initialUpdateCounter() {
return store.initialUpdateCounter();
}
@@ -948,6 +948,13 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
}
/**
+ * @return Total size of all caches.
+ */
+ public long fullSize() {
+ return store.fullSize();
+ }
+
+ /**
* Removes all entries and rows from this partition.
*
* @return Number of rows cleared from page memory.
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index 7f900cb..6f68dbb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.cache.distributed.dht;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
@@ -345,6 +346,11 @@ public interface GridDhtPartitionTopology {
public CachePartitionPartialCountersMap localUpdateCounters(boolean skipZeros);
/**
+ * @return Partition cache sizes.
+ */
+ public Map<Integer, Long> partitionSizes();
+
+ /**
* @param part Partition to own.
* @return {@code True} if owned.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 538c57e..740903e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -31,6 +31,8 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReferenceArray;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
@@ -2526,6 +2528,28 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
+ @Override public Map<Integer, Long> partitionSizes() {
+ lock.readLock().lock();
+
+ try {
+ Map<Integer, Long> partitionSizes = new HashMap<>();
+
+ for (int p = 0; p < locParts.length(); p++) {
+ GridDhtLocalPartition part = locParts.get(p);
+ if (part == null || part.fullSize() == 0)
+ continue;
+
+ partitionSizes.put(part.id(), part.fullSize());
+ }
+
+ return partitionSizes;
+ }
+ finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public boolean rebalanceFinished(AffinityTopologyVersion topVer) {
AffinityTopologyVersion curTopVer = this.readyTopVer;
@@ -2587,7 +2611,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (part == null)
continue;
- int size = part.dataStore().fullSize();
+ long size = part.dataStore().fullSize();
if (size >= threshold)
X.println(">>> Local partition [part=" + part.id() + ", size=" + size + ']');
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
new file mode 100644
index 0000000..92a0584
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.apache.ignite.lang.IgniteProductVersion;
+
+import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
+
+/**
+ * Class to validate partitions update counters and cache sizes during exchange process.
+ */
+public class GridDhtPartitionsStateValidator {
+ /** Version since node is able to send cache sizes in {@link GridDhtPartitionsSingleMessage}. */
+ private static final IgniteProductVersion SIZES_VALIDATION_AVAILABLE_SINCE = IgniteProductVersion.fromString("2.5.0");
+
+ /** Cache shared context. */
+ private final GridCacheSharedContext<?, ?> cctx;
+
+ /**
+ * Constructor.
+ *
+ * @param cctx Cache shared context.
+ */
+ public GridDhtPartitionsStateValidator(GridCacheSharedContext<?, ?> cctx) {
+ this.cctx = cctx;
+ }
+
+ /**
+ * Validates partition states - update counters and cache sizes for all nodes.
+ * If update counter value or cache size for the same partitions are different on some nodes
+ * method throws exception with full information about inconsistent partitions.
+ *
+ * @param fut Current exchange future.
+ * @param top Topology to validate.
+ * @param messages Single messages received from all nodes.
+ * @throws IgniteCheckedException If validation failed. Exception message contains
+ * full information about all partitions which update counters or cache sizes are not consistent.
+ */
+ public void validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture fut,
+ GridDhtPartitionTopology top,
+ Map<UUID, GridDhtPartitionsSingleMessage> messages) throws IgniteCheckedException {
+ // Ignore just joined nodes.
+ final Set<UUID> ignoringNodes = new HashSet<>();
+
+ for (DiscoveryEvent evt : fut.events().events())
+ if (evt.type() == EVT_NODE_JOINED)
+ ignoringNodes.add(evt.eventNode().id());
+
+ AffinityTopologyVersion topVer = fut.context().events().topologyVersion();
+
+ // Validate update counters.
+ Map<Integer, Map<UUID, Long>> result = validatePartitionsUpdateCounters(top, messages, ignoringNodes);
+ if (!result.isEmpty())
+ throw new IgniteCheckedException("Partitions update counters are inconsistent for " + fold(topVer, result));
+
+ // For sizes validation ignore also nodes which are not able to send cache sizes.
+ for (UUID id : messages.keySet()) {
+ ClusterNode node = cctx.discovery().node(id);
+ if (node != null && node.version().compareTo(SIZES_VALIDATION_AVAILABLE_SINCE) < 0)
+ ignoringNodes.add(id);
+ }
+
+ // Validate cache sizes.
+ result = validatePartitionsSizes(top, messages, ignoringNodes);
+ if (!result.isEmpty())
+ throw new IgniteCheckedException("Partitions cache sizes are inconsistent for " + fold(topVer, result));
+ }
+
+ /**
+ * Validate partitions update counters for given {@code top}.
+ *
+ * @param top Topology to validate.
+ * @param messages Single messages received from all nodes.
+ * @param ignoringNodes Nodes for what we ignore validation.
+ * @return Invalid partitions map with following structure: (partId, (nodeId, updateCounter)).
+ * If map is empty validation is successful.
+ */
+ Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
+ GridDhtPartitionTopology top,
+ Map<UUID, GridDhtPartitionsSingleMessage> messages,
+ Set<UUID> ignoringNodes) {
+ Map<Integer, Map<UUID, Long>> invalidPartitions = new HashMap<>();
+
+ Map<Integer, T2<UUID, Long>> updateCountersAndNodesByPartitions = new HashMap<>();
+
+ // Populate counters statistics from local node partitions.
+ for (GridDhtLocalPartition part : top.currentLocalPartitions()) {
+ if (top.partitionState(cctx.localNodeId(), part.id()) != GridDhtPartitionState.OWNING)
+ continue;
+
+ updateCountersAndNodesByPartitions.put(part.id(), new T2<>(cctx.localNodeId(), part.updateCounter()));
+ }
+
+ int partitions = top.partitions();
+
+ // Then process and validate counters from other nodes.
+ for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> e : messages.entrySet()) {
+ UUID nodeId = e.getKey();
+ if (ignoringNodes.contains(nodeId))
+ continue;
+
+ CachePartitionPartialCountersMap countersMap = e.getValue().partitionUpdateCounters(top.groupId(), partitions);
+
+ for (int part = 0; part < partitions; part++) {
+ if (top.partitionState(nodeId, part) != GridDhtPartitionState.OWNING)
+ continue;
+
+ int partIdx = countersMap.partitionIndex(part);
+ long currentCounter = partIdx >= 0 ? countersMap.updateCounterAt(partIdx) : 0;
+
+ process(invalidPartitions, updateCountersAndNodesByPartitions, part, nodeId, currentCounter);
+ }
+ }
+
+ return invalidPartitions;
+ }
+
+ /**
+ * Validate partitions cache sizes for given {@code top}.
+ *
+ * @param top Topology to validate.
+ * @param messages Single messages received from all nodes.
+ * @param ignoringNodes Nodes for what we ignore validation.
+ * @return Invalid partitions map with following structure: (partId, (nodeId, cacheSize)).
+ * If map is empty validation is successful.
+ */
+ Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
+ GridDhtPartitionTopology top,
+ Map<UUID, GridDhtPartitionsSingleMessage> messages,
+ Set<UUID> ignoringNodes) {
+ Map<Integer, Map<UUID, Long>> invalidPartitions = new HashMap<>();
+
+ Map<Integer, T2<UUID, Long>> sizesAndNodesByPartitions = new HashMap<>();
+
+ // Populate sizes statistics from local node partitions.
+ for (GridDhtLocalPartition part : top.currentLocalPartitions()) {
+ if (top.partitionState(cctx.localNodeId(), part.id()) != GridDhtPartitionState.OWNING)
+ continue;
+
+ sizesAndNodesByPartitions.put(part.id(), new T2<>(cctx.localNodeId(), part.fullSize()));
+ }
+
+ int partitions = top.partitions();
+
+ // Then process and validate sizes from other nodes.
+ for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> e : messages.entrySet()) {
+ UUID nodeId = e.getKey();
+ if (ignoringNodes.contains(nodeId))
+ continue;
+
+ Map<Integer, Long> sizesMap = e.getValue().partitionSizes(top.groupId());
+
+ for (int part = 0; part < partitions; part++) {
+ if (top.partitionState(nodeId, part) != GridDhtPartitionState.OWNING)
+ continue;
+
+ long currentSize = sizesMap.containsKey(part) ? sizesMap.get(part) : 0L;
+
+ process(invalidPartitions, sizesAndNodesByPartitions, part, nodeId, currentSize);
+ }
+ }
+
+ return invalidPartitions;
+ }
+
+ /**
+ * Processes given {@code counter} for partition {@code part} reported by {@code node}.
+ * Populates {@code invalidPartitions} map if existing counter and current {@code counter} are different.
+ *
+ * @param invalidPartitions Invalid partitions map.
+ * @param countersAndNodes Current map of counters and nodes by partitions.
+ * @param part Processing partition.
+ * @param node Node id.
+ * @param counter Counter value reported by {@code node}.
+ */
+ private void process(Map<Integer, Map<UUID, Long>> invalidPartitions,
+ Map<Integer, T2<UUID, Long>> countersAndNodes,
+ int part,
+ UUID node,
+ long counter) {
+ T2<UUID, Long> existingData = countersAndNodes.get(part);
+
+ if (existingData == null)
+ countersAndNodes.put(part, new T2<>(node, counter));
+
+ if (existingData != null && counter != existingData.get2()) {
+ if (!invalidPartitions.containsKey(part)) {
+ Map<UUID, Long> map = new HashMap<>();
+ map.put(existingData.get1(), existingData.get2());
+ invalidPartitions.put(part, map);
+ }
+
+ invalidPartitions.get(part).put(node, counter);
+ }
+ }
+
+ /**
+ * Folds given map of invalid partition states to string representation in the following format:
+ * Part [id]: [consistentId=value*]
+ *
+ * Value can be both update counter or cache size.
+ *
+ * @param topVer Last topology version.
+ * @param invalidPartitions Invalid partitions map.
+ * @return String representation of invalid partitions.
+ */
+ private String fold(AffinityTopologyVersion topVer, Map<Integer, Map<UUID, Long>> invalidPartitions) {
+ SB sb = new SB();
+
+ NavigableMap<Integer, Map<UUID, Long>> sortedPartitions = new TreeMap<>(invalidPartitions);
+
+ for (Map.Entry<Integer, Map<UUID, Long>> p : sortedPartitions.entrySet()) {
+ sb.a("Part ").a(p.getKey()).a(": [");
+ for (Map.Entry<UUID, Long> e : p.getValue().entrySet()) {
+ Object consistentId = cctx.discovery().node(topVer, e.getKey()).consistentId();
+ sb.a(consistentId).a("=").a(e.getValue()).a(" ");
+ }
+ sb.a("] ");
+ }
+
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
index 28cc018..0609f04 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
@@ -447,6 +447,11 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
err = e;
}
+ catch (Throwable t) {
+ fut.onDone(t);
+
+ throw t;
+ }
if (primarySync)
sendFinishReply(err);
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index cbb4985..dd4a571 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -41,6 +41,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.NearCacheConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
@@ -75,10 +76,12 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext;
import org.apache.ignite.internal.processors.cache.StateChangeRequest;
import org.apache.ignite.internal.processors.cache.WalStateAbstractMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionsStateValidator;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
@@ -290,6 +293,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
@GridToStringExclude
private GridDhtPartitionsExchangeFuture mergedWith;
+ /** Validator for partition states. */
+ @GridToStringExclude
+ private final GridDhtPartitionsStateValidator validator;
+
/**
* @param cctx Cache context.
* @param busyLock Busy lock.
@@ -314,6 +321,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
this.exchId = exchId;
this.exchActions = exchActions;
this.affChangeMsg = affChangeMsg;
+ this.validator = new GridDhtPartitionsStateValidator(cctx);
log = cctx.logger(getClass());
exchLog = cctx.logger(EXCHANGE_LOG);
@@ -1099,7 +1107,11 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
// To correctly rebalance when persistence is enabled, it is necessary to reserve history within exchange.
partHistReserved = cctx.database().reserveHistoryForExchange();
- waitPartitionRelease();
+ // On first phase we wait for finishing all local tx updates, atomic updates and lock releases.
+ waitPartitionRelease(1);
+
+ // Second phase is needed to wait for finishing all tx updates from primary to backup nodes remaining after first phase.
+ waitPartitionRelease(2);
boolean topChanged = firstDiscoEvt.type() != EVT_DISCOVERY_CUSTOM_EVT || affChangeMsg != null;
@@ -1202,9 +1214,17 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
* For the exact list of the objects being awaited for see
* {@link GridCacheSharedContext#partitionReleaseFuture(AffinityTopologyVersion)} javadoc.
*
+ * @param phase Phase of partition release.
+ *
* @throws IgniteCheckedException If failed.
*/
- private void waitPartitionRelease() throws IgniteCheckedException {
+ private void waitPartitionRelease(int phase) throws IgniteCheckedException {
+ Latch releaseLatch = null;
+
+ // Wait for other nodes only on first phase.
+ if (phase == 1)
+ releaseLatch = cctx.exchange().latch().getOrCreate("exchange", initialVersion());
+
IgniteInternalFuture<?> partReleaseFut = cctx.partitionReleaseFuture(initialVersion());
// Assign to class variable so it will be included into toString() method.
@@ -1238,6 +1258,11 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
nextDumpTime = U.currentTimeMillis() + nextDumpTimeout(dumpCnt++, futTimeout);
}
}
+ catch (IgniteCheckedException e) {
+ U.warn(log,"Unable to await partitions release future", e);
+
+ throw e;
+ }
}
long waitEnd = U.currentTimeMillis();
@@ -1290,6 +1315,35 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
}
}
+
+ if (releaseLatch == null)
+ return;
+
+ releaseLatch.countDown();
+
+ if (!localJoinExchange()) {
+ try {
+ while (true) {
+ try {
+ releaseLatch.await(futTimeout, TimeUnit.MILLISECONDS);
+
+ if (log.isInfoEnabled())
+ log.info("Finished waiting for partitions release latch: " + releaseLatch);
+
+ break;
+ }
+ catch (IgniteFutureTimeoutCheckedException ignored) {
+ U.warn(log, "Unable to await partitions release latch within timeout: " + releaseLatch);
+
+ // Try to resend ack.
+ releaseLatch.countDown();
+ }
+ }
+ }
+ catch (IgniteCheckedException e) {
+ U.warn(log, "Stop waiting for partitions release latch: " + e.getMessage());
+ }
+ }
}
/**
@@ -2499,6 +2553,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
}
+ validatePartitionsState();
+
if (firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) {
assert firstDiscoEvt instanceof DiscoveryCustomEvent;
@@ -2683,6 +2739,42 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
/**
+ * Validates that partition update counters and cache sizes for all caches are consistent.
+ */
+ private void validatePartitionsState() {
+ for (Map.Entry<Integer, CacheGroupDescriptor> e : cctx.affinity().cacheGroups().entrySet()) {
+ CacheGroupDescriptor grpDesc = e.getValue();
+ if (grpDesc.config().getCacheMode() == CacheMode.LOCAL)
+ continue;
+
+ int grpId = e.getKey();
+
+ CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpId);
+
+ GridDhtPartitionTopology top = grpCtx != null ?
+ grpCtx.topology() :
+ cctx.exchange().clientTopology(grpId, events().discoveryCache());
+
+ // Do not validate read or write through caches or caches with disabled rebalance.
+ if (grpCtx == null
+ || grpCtx.config().isReadThrough()
+ || grpCtx.config().isWriteThrough()
+ || grpCtx.config().getCacheStoreFactory() != null
+ || grpCtx.config().getRebalanceDelay() != -1
+ || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE)
+ continue;
+
+ try {
+ validator.validatePartitionCountersAndSizes(this, top, msgs);
+ }
+ catch (IgniteCheckedException ex) {
+ log.warning("Partition states validation was failed for cache " + grpDesc.cacheOrGroupName(), ex);
+ // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833
+ }
+ }
+ }
+
+ /**
*
*/
private void assignPartitionsStates() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index 215152d..6ebafac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -17,9 +17,9 @@
package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
-import java.util.Collection;
import java.io.Externalizable;
import java.nio.ByteBuffer;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -67,6 +67,14 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
/** Serialized partitions counters. */
private byte[] partCntrsBytes;
+ /** Partitions sizes. */
+ @GridToStringInclude
+ @GridDirectTransient
+ private Map<Integer, Map<Integer, Long>> partSizes;
+
+ /** Serialized partitions counters. */
+ private byte[] partSizesBytes;
+
/** Partitions history reservation counters. */
@GridToStringInclude
@GridDirectTransient
@@ -220,6 +228,35 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
}
/**
+ * Adds partition sizes map for specified {@code grpId} to the current message.
+ *
+ * @param grpId Group id.
+ * @param partSizesMap Partition sizes map.
+ */
+ public void addPartitionSizes(int grpId, Map<Integer, Long> partSizesMap) {
+ if (partSizesMap.isEmpty())
+ return;
+
+ if (partSizes == null)
+ partSizes = new HashMap<>();
+
+ partSizes.put(grpId, partSizesMap);
+ }
+
+ /**
+ * Returns partition sizes map for specified {@code grpId}.
+ *
+ * @param grpId Group id.
+ * @return Partition sizes map (partId, partSize).
+ */
+ public Map<Integer, Long> partitionSizes(int grpId) {
+ if (partSizes == null)
+ return Collections.emptyMap();
+
+ return partSizes.getOrDefault(grpId, Collections.emptyMap());
+ }
+
+ /**
* @param grpId Cache group ID.
* @param cntrMap Partition history counters.
*/
@@ -287,12 +324,14 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
boolean marshal = (parts != null && partsBytes == null) ||
(partCntrs != null && partCntrsBytes == null) ||
(partHistCntrs != null && partHistCntrsBytes == null) ||
+ (partSizes != null && partSizesBytes == null) ||
(err != null && errBytes == null);
if (marshal) {
byte[] partsBytes0 = null;
byte[] partCntrsBytes0 = null;
byte[] partHistCntrsBytes0 = null;
+ byte[] partSizesBytes0 = null;
byte[] errBytes0 = null;
if (parts != null && partsBytes == null)
@@ -304,6 +343,9 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
if (partHistCntrs != null && partHistCntrsBytes == null)
partHistCntrsBytes0 = U.marshal(ctx, partHistCntrs);
+ if (partSizes != null && partSizesBytes == null)
+ partSizesBytes0 = U.marshal(ctx, partSizes);
+
if (err != null && errBytes == null)
errBytes0 = U.marshal(ctx, err);
@@ -314,11 +356,13 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
byte[] partsBytesZip = U.zip(partsBytes0);
byte[] partCntrsBytesZip = U.zip(partCntrsBytes0);
byte[] partHistCntrsBytesZip = U.zip(partHistCntrsBytes0);
+ byte[] partSizesBytesZip = U.zip(partSizesBytes0);
byte[] exBytesZip = U.zip(errBytes0);
partsBytes0 = partsBytesZip;
partCntrsBytes0 = partCntrsBytesZip;
partHistCntrsBytes0 = partHistCntrsBytesZip;
+ partSizesBytes0 = partSizesBytesZip;
errBytes0 = exBytesZip;
compressed(true);
@@ -331,6 +375,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
partsBytes = partsBytes0;
partCntrsBytes = partCntrsBytes0;
partHistCntrsBytes = partHistCntrsBytes0;
+ partSizesBytes = partSizesBytes0;
errBytes = errBytes0;
}
}
@@ -360,6 +405,13 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
partHistCntrs = U.unmarshal(ctx, partHistCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
}
+ if (partSizesBytes != null && partSizes == null) {
+ if (compressed())
+ partSizes = U.unmarshalZip(ctx.marshaller(), partSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+ else
+ partSizes = U.unmarshal(ctx, partSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+ }
+
if (errBytes != null && err == null) {
if (compressed())
err = U.unmarshalZip(ctx.marshaller(), errBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
@@ -451,6 +503,11 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
writer.incrementState();
+ case 13:
+ if (!writer.writeByteArray("partsSizesBytes", partSizesBytes))
+ return false;
+
+ writer.incrementState();
}
return true;
@@ -531,6 +588,13 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
reader.incrementState();
+ case 13:
+ partSizesBytes = reader.readByteArray("partsSizesBytes");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
}
return reader.afterMessageRead(GridDhtPartitionsSingleMessage.class);
@@ -543,7 +607,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 13;
+ return 14;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
index 596fa8c..42a9ba6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
@@ -235,7 +235,7 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
if (awaited.remove(node.id())) {
GridDhtPartitionsFullMessage fullMsg0 = msg.finishMessage();
- if (fullMsg0 != null) {
+ if (fullMsg0 != null && fullMsg0.resultTopologyVersion() != null) {
assert fullMsg == null || fullMsg.resultTopologyVersion().equals(fullMsg0.resultTopologyVersion());
fullMsg = fullMsg0;
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
new file mode 100644
index 0000000..c205cb1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -0,0 +1,695 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.GridTopic;
+import org.apache.ignite.internal.managers.communication.GridIoManager;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+
+/**
+ * Class is responsible to create and manage instances of distributed latches {@link Latch}.
+ */
+public class ExchangeLatchManager {
+ /** Version since latch management is available. */
+ private static final IgniteProductVersion VERSION_SINCE = IgniteProductVersion.fromString("2.5.0");
+
+ /** Logger. */
+ private final IgniteLogger log;
+
+ /** Context. */
+ private final GridKernalContext ctx;
+
+ /** Discovery manager. */
+ private final GridDiscoveryManager discovery;
+
+ /** IO manager. */
+ private final GridIoManager io;
+
+ /** Current coordinator. */
+ private volatile ClusterNode coordinator;
+
+ /** Pending acks collection. */
+ private final ConcurrentMap<T2<String, AffinityTopologyVersion>, Set<UUID>> pendingAcks = new ConcurrentHashMap<>();
+
+ /** Server latches collection. */
+ private final ConcurrentMap<T2<String, AffinityTopologyVersion>, ServerLatch> serverLatches = new ConcurrentHashMap<>();
+
+ /** Client latches collection. */
+ private final ConcurrentMap<T2<String, AffinityTopologyVersion>, ClientLatch> clientLatches = new ConcurrentHashMap<>();
+
+ /** Lock. */
+ private final ReentrantLock lock = new ReentrantLock();
+
+ /**
+ * Constructor.
+ *
+ * @param ctx Kernal context.
+ */
+ public ExchangeLatchManager(GridKernalContext ctx) {
+ this.ctx = ctx;
+ this.log = ctx.log(getClass());
+ this.discovery = ctx.discovery();
+ this.io = ctx.io();
+
+ if (!ctx.clientNode()) {
+ ctx.io().addMessageListener(GridTopic.TOPIC_EXCHANGE, (nodeId, msg, plc) -> {
+ if (msg instanceof LatchAckMessage) {
+ processAck(nodeId, (LatchAckMessage) msg);
+ }
+ });
+
+ // First coordinator initialization.
+ ctx.discovery().localJoinFuture().listen(f -> {
+ this.coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
+ });
+
+ ctx.event().addDiscoveryEventListener((e, cache) -> {
+ assert e != null;
+ assert e.type() == EVT_NODE_LEFT || e.type() == EVT_NODE_FAILED : this;
+
+ // Do not process from discovery thread.
+ ctx.closure().runLocalSafe(() -> processNodeLeft(e.eventNode()));
+ }, EVT_NODE_LEFT, EVT_NODE_FAILED);
+ }
+ }
+
+ /**
+ * Creates server latch with given {@code id} and {@code topVer}.
+ * Adds corresponding pending acks to it.
+ *
+ * @param id Latch id.
+ * @param topVer Latch topology version.
+ * @param participants Participant nodes.
+ * @return Server latch instance.
+ */
+ private Latch createServerLatch(String id, AffinityTopologyVersion topVer, Collection<ClusterNode> participants) {
+ final T2<String, AffinityTopologyVersion> latchId = new T2<>(id, topVer);
+
+ if (serverLatches.containsKey(latchId))
+ return serverLatches.get(latchId);
+
+ ServerLatch latch = new ServerLatch(id, topVer, participants);
+
+ serverLatches.put(latchId, latch);
+
+ if (log.isDebugEnabled())
+ log.debug("Server latch is created [latch=" + latchId + ", participantsSize=" + participants.size() + "]");
+
+ if (pendingAcks.containsKey(latchId)) {
+ Set<UUID> acks = pendingAcks.get(latchId);
+
+ for (UUID node : acks)
+ if (latch.hasParticipant(node) && !latch.hasAck(node))
+ latch.ack(node);
+
+ pendingAcks.remove(latchId);
+ }
+
+ if (latch.isCompleted())
+ serverLatches.remove(latchId);
+
+ return latch;
+ }
+
+ /**
+ * Creates client latch.
+ * If there is final ack corresponds to given {@code id} and {@code topVer}, latch will be completed immediately.
+ *
+ * @param id Latch id.
+ * @param topVer Latch topology version.
+ * @param coordinator Coordinator node.
+ * @param participants Participant nodes.
+ * @return Client latch instance.
+ */
+ private Latch createClientLatch(String id, AffinityTopologyVersion topVer, ClusterNode coordinator, Collection<ClusterNode> participants) {
+ final T2<String, AffinityTopologyVersion> latchId = new T2<>(id, topVer);
+
+ if (clientLatches.containsKey(latchId))
+ return clientLatches.get(latchId);
+
+ ClientLatch latch = new ClientLatch(id, topVer, coordinator, participants);
+
+ if (log.isDebugEnabled())
+ log.debug("Client latch is created [latch=" + latchId
+ + ", crd=" + coordinator
+ + ", participantsSize=" + participants.size() + "]");
+
+ // There is final ack for created latch.
+ if (pendingAcks.containsKey(latchId)) {
+ latch.complete();
+ pendingAcks.remove(latchId);
+ }
+ else
+ clientLatches.put(latchId, latch);
+
+ return latch;
+ }
+
+ /**
+ * Creates new latch with specified {@code id} and {@code topVer} or returns existing latch.
+ *
+ * Participants of latch are calculated from given {@code topVer} as alive server nodes.
+ * If local node is coordinator {@code ServerLatch} instance will be created, otherwise {@code ClientLatch} instance.
+ *
+ * @param id Latch id.
+ * @param topVer Latch topology version.
+ * @return Latch instance.
+ */
+ public Latch getOrCreate(String id, AffinityTopologyVersion topVer) {
+ lock.lock();
+
+ try {
+ ClusterNode coordinator = getLatchCoordinator(topVer);
+
+ if (coordinator == null) {
+ ClientLatch latch = new ClientLatch(id, AffinityTopologyVersion.NONE, null, Collections.emptyList());
+ latch.complete();
+
+ return latch;
+ }
+
+ Collection<ClusterNode> participants = getLatchParticipants(topVer);
+
+ return coordinator.isLocal()
+ ? createServerLatch(id, topVer, participants)
+ : createClientLatch(id, topVer, coordinator, participants);
+ }
+ finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * @param topVer Latch topology version.
+ * @return Collection of alive server nodes with latch functionality.
+ */
+ private Collection<ClusterNode> getLatchParticipants(AffinityTopologyVersion topVer) {
+ Collection<ClusterNode> aliveNodes = topVer == AffinityTopologyVersion.NONE
+ ? discovery.aliveServerNodes()
+ : discovery.discoCache(topVer).aliveServerNodes();
+
+ return aliveNodes
+ .stream()
+ .filter(node -> node.version().compareTo(VERSION_SINCE) >= 0)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * @param topVer Latch topology version.
+ * @return Oldest alive server node with latch functionality.
+ */
+ @Nullable private ClusterNode getLatchCoordinator(AffinityTopologyVersion topVer) {
+ Collection<ClusterNode> aliveNodes = topVer == AffinityTopologyVersion.NONE
+ ? discovery.aliveServerNodes()
+ : discovery.discoCache(topVer).aliveServerNodes();
+
+ return aliveNodes
+ .stream()
+ .filter(node -> node.version().compareTo(VERSION_SINCE) >= 0)
+ .findFirst()
+ .orElse(null);
+ }
+
+ /**
+ * Processes ack message from given {@code from} node.
+ *
+ * Completes client latch in case of final ack message.
+ *
+ * If no latch is associated with message, ack is placed to {@link #pendingAcks} set.
+ *
+ * @param from Node sent ack.
+ * @param message Ack message.
+ */
+ private void processAck(UUID from, LatchAckMessage message) {
+ lock.lock();
+
+ try {
+ ClusterNode coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
+
+ if (coordinator == null)
+ return;
+
+ T2<String, AffinityTopologyVersion> latchId = new T2<>(message.latchId(), message.topVer());
+
+ if (message.isFinal()) {
+ if (log.isDebugEnabled())
+ log.debug("Process final ack [latch=" + latchId + ", from=" + from + "]");
+
+ if (clientLatches.containsKey(latchId)) {
+ ClientLatch latch = clientLatches.remove(latchId);
+ latch.complete();
+ }
+ else if (!coordinator.isLocal()) {
+ pendingAcks.computeIfAbsent(latchId, (id) -> new GridConcurrentHashSet<>());
+ pendingAcks.get(latchId).add(from);
+ }
+ } else {
+ if (log.isDebugEnabled())
+ log.debug("Process ack [latch=" + latchId + ", from=" + from + "]");
+
+ if (serverLatches.containsKey(latchId)) {
+ ServerLatch latch = serverLatches.get(latchId);
+
+ if (latch.hasParticipant(from) && !latch.hasAck(from)) {
+ latch.ack(from);
+
+ if (latch.isCompleted())
+ serverLatches.remove(latchId);
+ }
+ }
+ else {
+ pendingAcks.computeIfAbsent(latchId, (id) -> new GridConcurrentHashSet<>());
+ pendingAcks.get(latchId).add(from);
+ }
+ }
+ }
+ finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Changes coordinator to current local node.
+ * Restores all server latches from pending acks and own client latches.
+ */
+ private void becomeNewCoordinator() {
+ if (log.isInfoEnabled())
+ log.info("Become new coordinator " + coordinator.id());
+
+ List<T2<String, AffinityTopologyVersion>> latchesToRestore = new ArrayList<>();
+ latchesToRestore.addAll(pendingAcks.keySet());
+ latchesToRestore.addAll(clientLatches.keySet());
+
+ for (T2<String, AffinityTopologyVersion> latchId : latchesToRestore) {
+ String id = latchId.get1();
+ AffinityTopologyVersion topVer = latchId.get2();
+ Collection<ClusterNode> participants = getLatchParticipants(topVer);
+
+ if (!participants.isEmpty())
+ createServerLatch(id, topVer, participants);
+ }
+ }
+
+ /**
+ * Handles node left discovery event.
+ *
+ * Summary:
+ * Removes pending acks corresponds to the left node.
+ * Adds fake acknowledgements to server latches where such node was participant.
+ * Changes client latches coordinator to oldest available server node where such node was coordinator.
+ * Detects coordinator change.
+ *
+ * @param left Left node.
+ */
+ private void processNodeLeft(ClusterNode left) {
+ assert this.coordinator != null : "Coordinator is not initialized";
+
+ lock.lock();
+
+ try {
+ if (log.isDebugEnabled())
+ log.debug("Process node left " + left.id());
+
+ ClusterNode coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
+
+ if (coordinator == null)
+ return;
+
+ // Clear pending acks.
+ for (Map.Entry<T2<String, AffinityTopologyVersion>, Set<UUID>> ackEntry : pendingAcks.entrySet())
+ if (ackEntry.getValue().contains(left.id()))
+ pendingAcks.get(ackEntry.getKey()).remove(left.id());
+
+ // Change coordinator for client latches.
+ for (Map.Entry<T2<String, AffinityTopologyVersion>, ClientLatch> latchEntry : clientLatches.entrySet()) {
+ ClientLatch latch = latchEntry.getValue();
+ if (latch.hasCoordinator(left.id())) {
+ // Change coordinator for latch and re-send ack if necessary.
+ if (latch.hasParticipant(coordinator.id()))
+ latch.newCoordinator(coordinator);
+ else {
+ /* If new coordinator is not able to take control on the latch,
+ it means that all other latch participants are left from topology
+ and there is no reason to track such latch. */
+ AffinityTopologyVersion topVer = latchEntry.getKey().get2();
+
+ assert getLatchParticipants(topVer).isEmpty();
+
+ latch.complete(new IgniteCheckedException("All latch participants are left from topology."));
+ clientLatches.remove(latchEntry.getKey());
+ }
+ }
+ }
+
+ // Add acknowledgements from left node.
+ for (Map.Entry<T2<String, AffinityTopologyVersion>, ServerLatch> latchEntry : serverLatches.entrySet()) {
+ ServerLatch latch = latchEntry.getValue();
+
+ if (latch.hasParticipant(left.id()) && !latch.hasAck(left.id())) {
+ if (log.isDebugEnabled())
+ log.debug("Process node left [latch=" + latchEntry.getKey() + ", left=" + left.id() + "]");
+
+ latch.ack(left.id());
+
+ if (latch.isCompleted())
+ serverLatches.remove(latchEntry.getKey());
+ }
+ }
+
+ // Coordinator is changed.
+ if (coordinator.isLocal() && this.coordinator.id() != coordinator.id()) {
+ this.coordinator = coordinator;
+
+ becomeNewCoordinator();
+ }
+ }
+ finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Latch creating on coordinator node.
+ * Latch collects acks from participants: non-coordinator nodes and current local node.
+ * Latch completes when all acks from all participants are received.
+ *
+ * After latch completion final ack is sent to all participants.
+ */
+ class ServerLatch extends CompletableLatch {
+ /** Number of latch permits. This is needed to track number of countDown invocations. */
+ private final AtomicInteger permits;
+
+ /** Set of received acks. */
+ private final Set<UUID> acks = new GridConcurrentHashSet<>();
+
+ /**
+ * Constructor.
+ *
+ * @param id Latch id.
+ * @param topVer Latch topology version.
+ * @param participants Participant nodes.
+ */
+ ServerLatch(String id, AffinityTopologyVersion topVer, Collection<ClusterNode> participants) {
+ super(id, topVer, participants);
+ this.permits = new AtomicInteger(participants.size());
+
+ // Send final acks when latch is completed.
+ this.complete.listen(f -> {
+ for (ClusterNode node : participants) {
+ try {
+ if (discovery.alive(node)) {
+ io.sendToGridTopic(node, GridTopic.TOPIC_EXCHANGE, new LatchAckMessage(id, topVer, true), GridIoPolicy.SYSTEM_POOL);
+
+ if (log.isDebugEnabled())
+ log.debug("Final ack is ackSent [latch=" + latchId() + ", to=" + node.id() + "]");
+ }
+ } catch (IgniteCheckedException e) {
+ if (log.isDebugEnabled())
+ log.debug("Unable to send final ack [latch=" + latchId() + ", to=" + node.id() + "]");
+ }
+ }
+ });
+ }
+
+ /**
+ * Checks if latch has ack from given node.
+ *
+ * @param from Node.
+ * @return {@code true} if latch has ack from given node.
+ */
+ private boolean hasAck(UUID from) {
+ return acks.contains(from);
+ }
+
+ /**
+ * Receives ack from given node.
+ * Count downs latch if ack was not already processed.
+ *
+ * @param from Node.
+ */
+ private void ack(UUID from) {
+ if (log.isDebugEnabled())
+ log.debug("Ack is accepted [latch=" + latchId() + ", from=" + from + "]");
+
+ countDown0(from);
+ }
+
+ /**
+ * Count down latch from ack of given node.
+ * Completes latch if all acks are received.
+ *
+ * @param node Node.
+ */
+ private void countDown0(UUID node) {
+ if (isCompleted() || acks.contains(node))
+ return;
+
+ acks.add(node);
+
+ int remaining = permits.decrementAndGet();
+
+ if (log.isDebugEnabled())
+ log.debug("Count down + [latch=" + latchId() + ", remaining=" + remaining + "]");
+
+ if (remaining == 0)
+ complete();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void countDown() {
+ countDown0(ctx.localNodeId());
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ Set<UUID> pendingAcks = participants.stream().filter(ack -> !acks.contains(ack)).collect(Collectors.toSet());
+
+ return S.toString(ServerLatch.class, this,
+ "pendingAcks", pendingAcks,
+ "super", super.toString());
+ }
+ }
+
+ /**
+ * Latch creating on non-coordinator node.
+ * Latch completes when final ack from coordinator is received.
+ */
+ class ClientLatch extends CompletableLatch {
+ /** Latch coordinator node. Can be changed if coordinator is left from topology. */
+ private volatile ClusterNode coordinator;
+
+ /** Flag indicates that ack is sent to coordinator. */
+ private boolean ackSent;
+
+ /**
+ * Constructor.
+ *
+ * @param id Latch id.
+ * @param topVer Latch topology version.
+ * @param coordinator Coordinator node.
+ * @param participants Participant nodes.
+ */
+ ClientLatch(String id, AffinityTopologyVersion topVer, ClusterNode coordinator, Collection<ClusterNode> participants) {
+ super(id, topVer, participants);
+
+ this.coordinator = coordinator;
+ }
+
+ /**
+ * Checks if latch coordinator is given {@code node}.
+ *
+ * @param node Node.
+ * @return {@code true} if latch coordinator is given node.
+ */
+ private boolean hasCoordinator(UUID node) {
+ return coordinator.id().equals(node);
+ }
+
+ /**
+ * Changes coordinator of latch and resends ack to new coordinator if needed.
+ *
+ * @param coordinator New coordinator.
+ */
+ private void newCoordinator(ClusterNode coordinator) {
+ if (log.isDebugEnabled())
+ log.debug("Coordinator is changed [latch=" + latchId() + ", crd=" + coordinator.id() + "]");
+
+ synchronized (this) {
+ this.coordinator = coordinator;
+
+ // Resend ack to new coordinator.
+ if (ackSent)
+ sendAck();
+ }
+ }
+
+ /**
+ * Sends ack to coordinator node.
+ * There is ack deduplication on coordinator. So it's fine to send same ack twice.
+ */
+ private void sendAck() {
+ try {
+ ackSent = true;
+
+ io.sendToGridTopic(coordinator, GridTopic.TOPIC_EXCHANGE, new LatchAckMessage(id, topVer, false), GridIoPolicy.SYSTEM_POOL);
+
+ if (log.isDebugEnabled())
+ log.debug("Ack is ackSent + [latch=" + latchId() + ", to=" + coordinator.id() + "]");
+ } catch (IgniteCheckedException e) {
+ // Coordinator is unreachable. On coodinator node left discovery event ack will be resent.
+ if (log.isDebugEnabled())
+ log.debug("Unable to send ack [latch=" + latchId() + ", to=" + coordinator.id() + "]: " + e.getMessage());
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void countDown() {
+ if (isCompleted())
+ return;
+
+ // Synchronize in case of changed coordinator.
+ synchronized (this) {
+ sendAck();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ClientLatch.class, this,
+ "super", super.toString());
+ }
+ }
+
+ /**
+ * Base latch functionality with implemented complete / await logic.
+ */
+ private abstract static class CompletableLatch implements Latch {
+ /** Latch id. */
+ @GridToStringInclude
+ protected final String id;
+
+ /** Latch topology version. */
+ @GridToStringInclude
+ protected final AffinityTopologyVersion topVer;
+
+ /** Latch node participants. Only participant nodes are able to change state of latch. */
+ @GridToStringExclude
+ protected final Set<UUID> participants;
+
+ /** Future indicates that latch is completed. */
+ @GridToStringExclude
+ protected final GridFutureAdapter<?> complete = new GridFutureAdapter<>();
+
+ /**
+ * Constructor.
+ *
+ * @param id Latch id.
+ * @param topVer Latch topology version.
+ * @param participants Participant nodes.
+ */
+ CompletableLatch(String id, AffinityTopologyVersion topVer, Collection<ClusterNode> participants) {
+ this.id = id;
+ this.topVer = topVer;
+ this.participants = participants.stream().map(ClusterNode::id).collect(Collectors.toSet());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void await() throws IgniteCheckedException {
+ complete.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void await(long timeout, TimeUnit timeUnit) throws IgniteCheckedException {
+ complete.get(timeout, timeUnit);
+ }
+
+ /**
+ * Checks if latch participants contain given {@code node}.
+ *
+ * @param node Node.
+ * @return {@code true} if latch participants contain given node.
+ */
+ boolean hasParticipant(UUID node) {
+ return participants.contains(node);
+ }
+
+ /**
+ * @return {@code true} if latch is completed.
+ */
+ boolean isCompleted() {
+ return complete.isDone();
+ }
+
+ /**
+ * Completes current latch.
+ */
+ void complete() {
+ complete.onDone();
+ }
+
+ /**
+ * Completes current latch with given {@code error}.
+ *
+ * @param error Error.
+ */
+ void complete(Throwable error) {
+ complete.onDone(error);
+ }
+
+ /**
+ * @return Full latch id.
+ */
+ String latchId() {
+ return id + "-" + topVer;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(CompletableLatch.class, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/Latch.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/Latch.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/Latch.java
new file mode 100644
index 0000000..9704c2e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/Latch.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Simple distributed count down latch interface.
+ * Latch supports count down and await logic.
+ * Latch functionality is not relied on caches and has own state management {@link ExchangeLatchManager}.
+ */
+public interface Latch {
+ /**
+ * Decrements count on current latch.
+ * Release all latch waiters on all nodes if count reaches zero.
+ *
+ * This is idempotent operation. Invoking this method twice or more on the same node doesn't have any effect.
+ */
+ void countDown();
+
+ /**
+ * Awaits current latch completion.
+ *
+ * @throws IgniteCheckedException If await is failed.
+ */
+ void await() throws IgniteCheckedException;
+
+ /**
+ * Awaits current latch completion with specified timeout.
+ *
+ * @param timeout Timeout value.
+ * @param timeUnit Timeout time unit.
+ * @throws IgniteCheckedException If await is failed.
+ */
+ void await(long timeout, TimeUnit timeUnit) throws IgniteCheckedException;
+}
[02/54] [abbrv] ignite git commit: IGNITE-7222 Added ZooKeeper
discovery SPI
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a64b941d/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
new file mode 100644
index 0000000..fb12c3a
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -0,0 +1,4847 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.curator.test.TestingCluster;
+import org.apache.curator.test.TestingZooKeeperServer;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.CommunicationFailureContext;
+import org.apache.ignite.configuration.CommunicationFailureResolver;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.DiscoverySpiTestListener;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.discovery.CustomEventListener;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.managers.discovery.DiscoveryLocalJoinData;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.TestCacheNodeExcludingFilter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
+import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.T3;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteOutClosure;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.security.SecurityCredentials;
+import org.apache.ignite.plugin.security.SecurityPermission;
+import org.apache.ignite.plugin.security.SecuritySubject;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiTestSuite2;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZKUtil;
+import org.apache.zookeeper.ZkTestClientCnxnSocketNIO;
+import org.apache.zookeeper.ZooKeeper;
+import org.jetbrains.annotations.Nullable;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
+import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
+import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2;
+import static org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD;
+import static org.apache.zookeeper.ZooKeeper.ZOOKEEPER_CLIENT_CNXN_SOCKET;
+
+/**
+ *
+ */
+@SuppressWarnings("deprecation")
+public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
+ /** */
+ private static final String IGNITE_ZK_ROOT = ZookeeperDiscoverySpi.DFLT_ROOT_PATH;
+
+ /** */
+ private static final int ZK_SRVS = 3;
+
+ /** */
+ private static TestingCluster zkCluster;
+
+ /** To run test with real local ZK. */
+ private static final boolean USE_TEST_CLUSTER = true;
+
+ /** */
+ private boolean client;
+
+ /** */
+ private static ThreadLocal<Boolean> clientThreadLoc = new ThreadLocal<>();
+
+ /** */
+ private static ConcurrentHashMap<UUID, Map<Long, DiscoveryEvent>> evts = new ConcurrentHashMap<>();
+
+ /** */
+ private static volatile boolean err;
+
+ /** */
+ private boolean testSockNio;
+
+ /** */
+ private boolean testCommSpi;
+
+ /** */
+ private long sesTimeout;
+
+ /** */
+ private long joinTimeout;
+
+ /** */
+ private boolean clientReconnectDisabled;
+
+ /** */
+ private ConcurrentHashMap<String, ZookeeperDiscoverySpi> spis = new ConcurrentHashMap<>();
+
+ /** */
+ private Map<String, Object> userAttrs;
+
+ /** */
+ private boolean dfltConsistenId;
+
+ /** */
+ private UUID nodeId;
+
+ /** */
+ private boolean persistence;
+
+ /** */
+ private IgniteOutClosure<CommunicationFailureResolver> commFailureRslvr;
+
+ /** */
+ private IgniteOutClosure<DiscoverySpiNodeAuthenticator> auth;
+
+ /** */
+ private String zkRootPath;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(final String igniteInstanceName) throws Exception {
+ if (testSockNio)
+ System.setProperty(ZOOKEEPER_CLIENT_CNXN_SOCKET, ZkTestClientCnxnSocketNIO.class.getName());
+
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ if (nodeId != null)
+ cfg.setNodeId(nodeId);
+
+ if (!dfltConsistenId)
+ cfg.setConsistentId(igniteInstanceName);
+
+ ZookeeperDiscoverySpi zkSpi = new ZookeeperDiscoverySpi();
+
+ if (joinTimeout != 0)
+ zkSpi.setJoinTimeout(joinTimeout);
+
+ zkSpi.setSessionTimeout(sesTimeout > 0 ? sesTimeout : 10_000);
+
+ zkSpi.setClientReconnectDisabled(clientReconnectDisabled);
+
+ // Set authenticator for basic sanity tests.
+ if (auth != null) {
+ zkSpi.setAuthenticator(auth.apply());
+
+ zkSpi.setInternalListener(new IgniteDiscoverySpiInternalListener() {
+ @Override public void beforeJoin(ClusterNode locNode, IgniteLogger log) {
+ ZookeeperClusterNode locNode0 = (ZookeeperClusterNode)locNode;
+
+ Map<String, Object> attrs = new HashMap<>(locNode0.getAttributes());
+
+ attrs.put(ATTR_SECURITY_CREDENTIALS, new SecurityCredentials(null, null, igniteInstanceName));
+
+ locNode0.setAttributes(attrs);
+ }
+
+ @Override public boolean beforeSendCustomEvent(DiscoverySpi spi, IgniteLogger log, DiscoverySpiCustomMessage msg) {
+ return false;
+ }
+ });
+ }
+
+ spis.put(igniteInstanceName, zkSpi);
+
+ if (USE_TEST_CLUSTER) {
+ assert zkCluster != null;
+
+ zkSpi.setZkConnectionString(zkCluster.getConnectString());
+
+ if (zkRootPath != null)
+ zkSpi.setZkRootPath(zkRootPath);
+ }
+ else
+ zkSpi.setZkConnectionString("localhost:2181");
+
+ cfg.setDiscoverySpi(zkSpi);
+
+ CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ Boolean clientMode = clientThreadLoc.get();
+
+ if (clientMode != null)
+ cfg.setClientMode(clientMode);
+ else
+ cfg.setClientMode(client);
+
+ if (userAttrs != null)
+ cfg.setUserAttributes(userAttrs);
+
+ Map<IgnitePredicate<? extends Event>, int[]> lsnrs = new HashMap<>();
+
+ lsnrs.put(new IgnitePredicate<Event>() {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+ @Override public boolean apply(Event evt) {
+ try {
+ DiscoveryEvent discoveryEvt = (DiscoveryEvent)evt;
+
+ UUID locId = ((IgniteKernal)ignite).context().localNodeId();
+
+ Map<Long, DiscoveryEvent> nodeEvts = evts.get(locId);
+
+ if (nodeEvts == null) {
+ Object old = evts.put(locId, nodeEvts = new TreeMap<>());
+
+ assertNull(old);
+
+ synchronized (nodeEvts) {
+ DiscoveryLocalJoinData locJoin = ((IgniteKernal)ignite).context().discovery().localJoin();
+
+ nodeEvts.put(locJoin.event().topologyVersion(), locJoin.event());
+ }
+ }
+
+ synchronized (nodeEvts) {
+ DiscoveryEvent old = nodeEvts.put(discoveryEvt.topologyVersion(), discoveryEvt);
+
+ assertNull(old);
+ }
+ }
+ catch (Throwable e) {
+ error("Unexpected error [evt=" + evt + ", err=" + e + ']', e);
+
+ err = true;
+ }
+
+ return true;
+ }
+ }, new int[]{EVT_NODE_JOINED, EVT_NODE_FAILED, EVT_NODE_LEFT});
+
+ cfg.setLocalEventListeners(lsnrs);
+
+ if (persistence) {
+ DataStorageConfiguration memCfg = new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setMaxSize(100 * 1024 * 1024).
+ setPersistenceEnabled(true))
+ .setPageSize(1024)
+ .setWalMode(WALMode.LOG_ONLY);
+
+ cfg.setDataStorageConfiguration(memCfg);
+ }
+
+ if (testCommSpi)
+ cfg.setCommunicationSpi(new ZkTestCommunicationSpi());
+
+ if (commFailureRslvr != null)
+ cfg.setCommunicationFailureResolver(commFailureRslvr.apply());
+
+ return cfg;
+ }
+
+ /**
+ * @param clientMode Client mode flag for started nodes.
+ */
+ private void clientMode(boolean clientMode) {
+ client = clientMode;
+ }
+
+ /**
+ * @param clientMode Client mode flag for nodes started from current thread.
+ */
+ private void clientModeThreadLocal(boolean clientMode) {
+ clientThreadLoc.set(clientMode);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ System.setProperty(ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_TIMEOUT, "1000");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopZkCluster();
+
+ System.clearProperty(ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_TIMEOUT);
+
+ super.afterTestsStopped();
+ }
+
+ /**
+ *
+ */
+ private void stopZkCluster() {
+ if (zkCluster != null) {
+ try {
+ zkCluster.close();
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to stop Zookeeper client: " + e, e);
+ }
+
+ zkCluster = null;
+ }
+ }
+
+ /**
+ *
+ */
+ private static void ackEveryEventSystemProperty() {
+ System.setProperty(IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD, "1");
+ }
+
+ /**
+ *
+ */
+ private void clearAckEveryEventSystemProperty() {
+ System.setProperty(IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD, "1");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ if (USE_TEST_CLUSTER && zkCluster == null) {
+ zkCluster = ZookeeperDiscoverySpiTestSuite2.createTestingCluster(ZK_SRVS);
+
+ zkCluster.start();
+ }
+
+ reset();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ clearAckEveryEventSystemProperty();
+
+ try {
+ assertFalse("Unexpected error, see log for details", err);
+
+ checkEventsConsistency();
+
+ checkInternalStructuresCleanup();
+
+ //TODO uncomment when https://issues.apache.org/jira/browse/IGNITE-8193 is fixed
+// checkZkNodesCleanup();
+ }
+ finally {
+ reset();
+
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void checkInternalStructuresCleanup() throws Exception {
+ for (Ignite node : G.allGrids()) {
+ final AtomicReference<?> res = GridTestUtils.getFieldValue(spi(node), "impl", "commErrProcFut");
+
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return res.get() == null;
+ }
+ }, 30_000);
+
+ assertNull(res.get());
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testZkRootNotExists() throws Exception {
+ zkRootPath = "/a/b/c";
+
+ for (int i = 0; i < 3; i++) {
+ reset();
+
+ startGridsMultiThreaded(5);
+
+ waitForTopology(5);
+
+ stopAllGrids();
+
+ checkEventsConsistency();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMetadataUpdate() throws Exception {
+ startGrid(0);
+
+ GridTestUtils.runMultiThreaded(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ ignite(0).configuration().getMarshaller().marshal(new C1());
+ ignite(0).configuration().getMarshaller().marshal(new C2());
+
+ return null;
+ }
+ }, 64, "marshal");
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testNodeAddresses() throws Exception {
+ startGridsMultiThreaded(3);
+
+ clientMode(true);
+
+ startGridsMultiThreaded(3, 3);
+
+ waitForTopology(6);
+
+ for (Ignite node : G.allGrids()) {
+ ClusterNode locNode0 = node.cluster().localNode();
+
+ assertTrue(locNode0.addresses().size() > 0);
+ assertTrue(locNode0.hostNames().size() > 0);
+
+ for (ClusterNode node0 : node.cluster().nodes()) {
+ assertTrue(node0.addresses().size() > 0);
+ assertTrue(node0.hostNames().size() > 0);
+ }
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSetConsistentId() throws Exception {
+ startGridsMultiThreaded(3);
+
+ clientMode(true);
+
+ startGridsMultiThreaded(3, 3);
+
+ waitForTopology(6);
+
+ for (Ignite node : G.allGrids()) {
+ ClusterNode locNode0 = node.cluster().localNode();
+
+ assertEquals(locNode0.attribute(ATTR_IGNITE_INSTANCE_NAME),
+ locNode0.consistentId());
+
+ for (ClusterNode node0 : node.cluster().nodes()) {
+ assertEquals(node0.attribute(ATTR_IGNITE_INSTANCE_NAME),
+ node0.consistentId());
+ }
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDefaultConsistentId() throws Exception {
+ dfltConsistenId = true;
+
+ startGridsMultiThreaded(3);
+
+ clientMode(true);
+
+ startGridsMultiThreaded(3, 3);
+
+ waitForTopology(6);
+
+ for (Ignite node : G.allGrids()) {
+ ClusterNode locNode0 = node.cluster().localNode();
+
+ assertNotNull(locNode0.consistentId());
+
+ for (ClusterNode node0 : node.cluster().nodes())
+ assertNotNull(node0.consistentId());
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testClientNodesStatus() throws Exception {
+ startGrid(0);
+
+ for (Ignite node : G.allGrids()) {
+ assertEquals(0, node.cluster().forClients().nodes().size());
+ assertEquals(1, node.cluster().forServers().nodes().size());
+ }
+
+ clientMode(true);
+
+ startGrid(1);
+
+ for (Ignite node : G.allGrids()) {
+ assertEquals(1, node.cluster().forClients().nodes().size());
+ assertEquals(1, node.cluster().forServers().nodes().size());
+ }
+
+ clientMode(false);
+
+ startGrid(2);
+
+ clientMode(true);
+
+ startGrid(3);
+
+ for (Ignite node : G.allGrids()) {
+ assertEquals(2, node.cluster().forClients().nodes().size());
+ assertEquals(2, node.cluster().forServers().nodes().size());
+ }
+
+ stopGrid(1);
+
+ waitForTopology(3);
+
+ for (Ignite node : G.allGrids()) {
+ assertEquals(1, node.cluster().forClients().nodes().size());
+ assertEquals(2, node.cluster().forServers().nodes().size());
+ }
+
+ stopGrid(2);
+
+ waitForTopology(2);
+
+ for (Ignite node : G.allGrids()) {
+ assertEquals(1, node.cluster().forClients().nodes().size());
+ assertEquals(1, node.cluster().forServers().nodes().size());
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void _testLocalAuthenticationFails() throws Exception {
+ auth = ZkTestNodeAuthenticator.factory(getTestIgniteInstanceName(0));
+
+ Throwable err = GridTestUtils.assertThrows(log, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ startGrid(0);
+
+ return null;
+ }
+ }, IgniteCheckedException.class, null);
+
+ IgniteSpiException spiErr = X.cause(err, IgniteSpiException.class);
+
+ assertNotNull(spiErr);
+ assertTrue(spiErr.getMessage().contains("Authentication failed for local node"));
+
+ startGrid(1);
+ startGrid(2);
+
+ checkTestSecuritySubject(2);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testAuthentication() throws Exception {
+ auth = ZkTestNodeAuthenticator.factory(getTestIgniteInstanceName(1),
+ getTestIgniteInstanceName(5));
+
+ startGrid(0);
+
+ checkTestSecuritySubject(1);
+
+ {
+ clientMode(false);
+ checkStartFail(1);
+
+ clientMode(true);
+ checkStartFail(1);
+
+ clientMode(false);
+ }
+
+ startGrid(2);
+
+ checkTestSecuritySubject(2);
+
+ stopGrid(2);
+
+ checkTestSecuritySubject(1);
+
+ startGrid(2);
+
+ checkTestSecuritySubject(2);
+
+ stopGrid(0);
+
+ checkTestSecuritySubject(1);
+
+ checkStartFail(1);
+
+ clientMode(false);
+
+ startGrid(3);
+
+ clientMode(true);
+
+ startGrid(4);
+
+ clientMode(false);
+
+ startGrid(0);
+
+ checkTestSecuritySubject(4);
+
+ checkStartFail(1);
+ checkStartFail(5);
+
+ clientMode(true);
+
+ checkStartFail(1);
+ checkStartFail(5);
+ }
+
+ /**
+ * @param nodeIdx Node index.
+ */
+ private void checkStartFail(final int nodeIdx) {
+ Throwable err = GridTestUtils.assertThrows(log, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ startGrid(nodeIdx);
+
+ return null;
+ }
+ }, IgniteCheckedException.class, null);
+
+ IgniteSpiException spiErr = X.cause(err, IgniteSpiException.class);
+
+ assertNotNull(spiErr);
+ assertTrue(spiErr.getMessage().contains("Authentication failed"));
+ }
+
+ /**
+ * @param expNodes Expected nodes number.
+ * @throws Exception If failed.
+ */
+ private void checkTestSecuritySubject(int expNodes) throws Exception {
+ waitForTopology(expNodes);
+
+ List<Ignite> nodes = G.allGrids();
+
+ JdkMarshaller marsh = new JdkMarshaller();
+
+ for (Ignite ignite : nodes) {
+ Collection<ClusterNode> nodes0 = ignite.cluster().nodes();
+
+ assertEquals(nodes.size(), nodes0.size());
+
+ for (ClusterNode node : nodes0) {
+ byte[] secSubj = node.attribute(ATTR_SECURITY_SUBJECT_V2);
+
+ assertNotNull(secSubj);
+
+ ZkTestNodeAuthenticator.TestSecurityContext secCtx = marsh.unmarshal(secSubj, null);
+
+ assertEquals(node.attribute(ATTR_IGNITE_INSTANCE_NAME), secCtx.nodeName);
+ }
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testStopNode_1() throws Exception {
+ startGrids(5);
+
+ waitForTopology(5);
+
+ stopGrid(3);
+
+ waitForTopology(4);
+
+ startGrid(3);
+
+ waitForTopology(5);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCustomEventsSimple1_SingleNode() throws Exception {
+ ackEveryEventSystemProperty();
+
+ Ignite srv0 = startGrid(0);
+
+ srv0.createCache(new CacheConfiguration<>("c1"));
+
+ waitForEventsAcks(srv0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCustomEventsSimple1_5_Nodes() throws Exception {
+ ackEveryEventSystemProperty();
+
+ Ignite srv0 = startGrids(5);
+
+ srv0.createCache(new CacheConfiguration<>("c1"));
+
+ awaitPartitionMapExchange();
+
+ waitForEventsAcks(srv0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCustomEvents_FastStopProcess_1() throws Exception {
+ customEvents_FastStopProcess(1, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCustomEvents_FastStopProcess_2() throws Exception {
+ customEvents_FastStopProcess(5, 5);
+ }
+
+ /**
+ * @param srvs Servers number.
+ * @param clients Clients number.
+ * @throws Exception If failed.
+ */
+ private void customEvents_FastStopProcess(int srvs, int clients) throws Exception {
+ ackEveryEventSystemProperty();
+
+ Map<UUID, List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>>> rcvdMsgs =
+ new ConcurrentHashMap<>();
+
+ Ignite crd = startGrid(0);
+
+ UUID crdId = crd.cluster().localNode().id();
+
+ if (srvs > 1)
+ startGridsMultiThreaded(1, srvs - 1);
+
+ if (clients > 0) {
+ client = true;
+
+ startGridsMultiThreaded(srvs, clients);
+ }
+
+ awaitPartitionMapExchange();
+
+ List<Ignite> nodes = G.allGrids();
+
+ assertEquals(srvs + clients, nodes.size());
+
+ for (Ignite node : nodes)
+ registerTestEventListeners(node, rcvdMsgs);
+
+ int payload = 0;
+
+ AffinityTopologyVersion topVer = ((IgniteKernal)crd).context().discovery().topologyVersionEx();
+
+ for (Ignite node : nodes) {
+ UUID sndId = node.cluster().localNode().id();
+
+ info("Send from node: " + sndId);
+
+ GridDiscoveryManager discoveryMgr = ((IgniteKernal)node).context().discovery();
+
+ {
+ List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> expCrdMsgs = new ArrayList<>();
+ List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> expNodesMsgs = Collections.emptyList();
+
+ TestFastStopProcessCustomMessage msg = new TestFastStopProcessCustomMessage(false, payload++);
+
+ expCrdMsgs.add(new T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>(topVer, sndId, msg));
+
+ discoveryMgr.sendCustomEvent(msg);
+
+ doSleep(200); // Wait some time to check extra messages are not received.
+
+ checkEvents(crd, rcvdMsgs, expCrdMsgs);
+
+ for (Ignite node0 : nodes) {
+ if (node0 != crd)
+ checkEvents(node0, rcvdMsgs, expNodesMsgs);
+ }
+
+ rcvdMsgs.clear();
+ }
+ {
+ List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> expCrdMsgs = new ArrayList<>();
+ List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> expNodesMsgs = new ArrayList<>();
+
+ TestFastStopProcessCustomMessage msg = new TestFastStopProcessCustomMessage(true, payload++);
+
+ expCrdMsgs.add(new T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>(topVer, sndId, msg));
+
+ discoveryMgr.sendCustomEvent(msg);
+
+ TestFastStopProcessCustomMessageAck ackMsg = new TestFastStopProcessCustomMessageAck(msg.payload);
+
+ expCrdMsgs.add(new T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>(topVer, crdId, ackMsg));
+ expNodesMsgs.add(new T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>(topVer, crdId, ackMsg));
+
+ doSleep(200); // Wait some time to check extra messages are not received.
+
+ checkEvents(crd, rcvdMsgs, expCrdMsgs);
+
+ for (Ignite node0 : nodes) {
+ if (node0 != crd)
+ checkEvents(node0, rcvdMsgs, expNodesMsgs);
+ }
+
+ rcvdMsgs.clear();
+ }
+
+ waitForEventsAcks(crd);
+ }
+ }
+
+ /**
+ * @param node Node to check.
+ * @param rcvdMsgs Received messages.
+ * @param expMsgs Expected messages.
+ * @throws Exception If failed.
+ */
+ private void checkEvents(
+ Ignite node,
+ final Map<UUID, List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>>> rcvdMsgs,
+ final List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> expMsgs) throws Exception {
+ final UUID nodeId = node.cluster().localNode().id();
+
+ assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> msgs = rcvdMsgs.get(nodeId);
+
+ int size = msgs == null ? 0 : msgs.size();
+
+ return size >= expMsgs.size();
+ }
+ }, 5000));
+
+ List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> msgs = rcvdMsgs.get(nodeId);
+
+ if (msgs == null)
+ msgs = Collections.emptyList();
+
+ assertEqualsCollections(expMsgs, msgs);
+ }
+
+ /**
+ * @param node Node.
+ * @param rcvdMsgs Map to store received events.
+ */
+ private void registerTestEventListeners(Ignite node,
+ final Map<UUID, List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>>> rcvdMsgs) {
+ GridDiscoveryManager discoveryMgr = ((IgniteKernal)node).context().discovery();
+
+ final UUID nodeId = node.cluster().localNode().id();
+
+ discoveryMgr.setCustomEventListener(TestFastStopProcessCustomMessage.class,
+ new CustomEventListener<TestFastStopProcessCustomMessage>() {
+ @Override public void onCustomEvent(AffinityTopologyVersion topVer, ClusterNode snd, TestFastStopProcessCustomMessage msg) {
+ List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> list = rcvdMsgs.get(nodeId);
+
+ if (list == null)
+ rcvdMsgs.put(nodeId, list = new ArrayList<>());
+
+ list.add(new T3<>(topVer, snd.id(), (DiscoveryCustomMessage)msg));
+ }
+ }
+ );
+ discoveryMgr.setCustomEventListener(TestFastStopProcessCustomMessageAck.class,
+ new CustomEventListener<TestFastStopProcessCustomMessageAck>() {
+ @Override public void onCustomEvent(AffinityTopologyVersion topVer, ClusterNode snd, TestFastStopProcessCustomMessageAck msg) {
+ List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> list = rcvdMsgs.get(nodeId);
+
+ if (list == null)
+ rcvdMsgs.put(nodeId, list = new ArrayList<>());
+
+ list.add(new T3<>(topVer, snd.id(), (DiscoveryCustomMessage)msg));
+ }
+ }
+ );
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSegmentation1() throws Exception {
+ sesTimeout = 2000;
+ testSockNio = true;
+
+ Ignite node0 = startGrid(0);
+
+ final CountDownLatch l = new CountDownLatch(1);
+
+ node0.events().localListen(new IgnitePredicate<Event>() {
+ @Override public boolean apply(Event evt) {
+ l.countDown();
+
+ return false;
+ }
+ }, EventType.EVT_NODE_SEGMENTED);
+
+ ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0);
+
+ c0.closeSocket(true);
+
+ for (int i = 0; i < 10; i++) {
+ Thread.sleep(1_000);
+
+ if (l.getCount() == 0)
+ break;
+ }
+
+ info("Allow connect");
+
+ c0.allowConnect();
+
+ assertTrue(l.await(10, TimeUnit.SECONDS));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSegmentation2() throws Exception {
+ sesTimeout = 2000;
+
+ Ignite node0 = startGrid(0);
+
+ final CountDownLatch l = new CountDownLatch(1);
+
+ node0.events().localListen(new IgnitePredicate<Event>() {
+ @Override public boolean apply(Event evt) {
+ l.countDown();
+
+ return false;
+ }
+ }, EventType.EVT_NODE_SEGMENTED);
+
+ try {
+ zkCluster.close();
+
+ assertTrue(l.await(10, TimeUnit.SECONDS));
+ }
+ finally {
+ zkCluster = ZookeeperDiscoverySpiTestSuite2.createTestingCluster(ZK_SRVS);
+
+ zkCluster.start();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSegmentation3() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8183");
+
+ sesTimeout = 5000;
+
+ Ignite node0 = startGrid(0);
+
+ final CountDownLatch l = new CountDownLatch(1);
+
+ node0.events().localListen(new IgnitePredicate<Event>() {
+ @Override public boolean apply(Event evt) {
+ l.countDown();
+
+ return false;
+ }
+ }, EventType.EVT_NODE_SEGMENTED);
+
+ List<TestingZooKeeperServer> srvs = zkCluster.getServers();
+
+ assertEquals(3, srvs.size());
+
+ try {
+ srvs.get(0).stop();
+ srvs.get(1).stop();
+
+ assertTrue(l.await(20, TimeUnit.SECONDS));
+ }
+ finally {
+ zkCluster.close();
+
+ zkCluster = ZookeeperDiscoverySpiTestSuite2.createTestingCluster(ZK_SRVS);
+
+ zkCluster.start();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testQuorumRestore() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8180");
+
+ sesTimeout = 15_000;
+
+ startGrids(3);
+
+ waitForTopology(3);
+
+ List<TestingZooKeeperServer> srvs = zkCluster.getServers();
+
+ assertEquals(3, srvs.size());
+
+ try {
+ srvs.get(0).stop();
+ srvs.get(1).stop();
+
+ U.sleep(2000);
+
+ srvs.get(1).restart();
+
+ U.sleep(4000);
+
+ startGrid(4);
+
+ waitForTopology(4);
+ }
+ finally {
+ zkCluster.close();
+
+ zkCluster = ZookeeperDiscoverySpiTestSuite2.createTestingCluster(ZK_SRVS);
+
+ zkCluster.start();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionRestore1() throws Exception {
+ testSockNio = true;
+
+ Ignite node0 = startGrid(0);
+
+ ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0);
+
+ c0.closeSocket(false);
+
+ startGrid(1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionRestore2() throws Exception {
+ testSockNio = true;
+
+ Ignite node0 = startGrid(0);
+
+ ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0);
+
+ c0.closeSocket(false);
+
+ startGridsMultiThreaded(1, 5);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionRestore_NonCoordinator1() throws Exception {
+ connectionRestore_NonCoordinator(false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionRestore_NonCoordinator2() throws Exception {
+ connectionRestore_NonCoordinator(true);
+ }
+
+ /**
+ * @param failWhenDisconnected {@code True} if fail node while another node is disconnected.
+ * @throws Exception If failed.
+ */
+ private void connectionRestore_NonCoordinator(boolean failWhenDisconnected) throws Exception {
+ testSockNio = true;
+
+ Ignite node0 = startGrid(0);
+ Ignite node1 = startGrid(1);
+
+ ZkTestClientCnxnSocketNIO c1 = ZkTestClientCnxnSocketNIO.forNode(node1);
+
+ c1.closeSocket(true);
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() {
+ try {
+ startGrid(2);
+ }
+ catch (Exception e) {
+ info("Start error: " + e);
+ }
+
+ return null;
+ }
+ }, "start-node");
+
+ checkEvents(node0, joinEvent(3));
+
+ if (failWhenDisconnected) {
+ ZookeeperDiscoverySpi spi = spis.get(getTestIgniteInstanceName(2));
+
+ closeZkClient(spi);
+
+ checkEvents(node0, failEvent(4));
+ }
+
+ c1.allowConnect();
+
+ checkEvents(ignite(1), joinEvent(3));
+
+ if (failWhenDisconnected) {
+ checkEvents(ignite(1), failEvent(4));
+
+ IgnitionEx.stop(getTestIgniteInstanceName(2), true, true);
+ }
+
+ fut.get();
+
+ waitForTopology(failWhenDisconnected ? 2 : 3);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionRestore_Coordinator1() throws Exception {
+ connectionRestore_Coordinator(1, 1, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionRestore_Coordinator1_1() throws Exception {
+ connectionRestore_Coordinator(1, 1, 1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionRestore_Coordinator2() throws Exception {
+ connectionRestore_Coordinator(1, 3, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionRestore_Coordinator3() throws Exception {
+ connectionRestore_Coordinator(3, 3, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionRestore_Coordinator4() throws Exception {
+ connectionRestore_Coordinator(3, 3, 1);
+ }
+
+ /**
+ * @param initNodes Number of initially started nodes.
+ * @param startNodes Number of nodes to start after coordinator loose connection.
+ * @param failCnt Number of nodes to stop after coordinator loose connection.
+ * @throws Exception If failed.
+ */
+ private void connectionRestore_Coordinator(final int initNodes, int startNodes, int failCnt) throws Exception {
+ sesTimeout = 30_000;
+ testSockNio = true;
+
+ Ignite node0 = startGrids(initNodes);
+
+ ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0);
+
+ c0.closeSocket(true);
+
+ final AtomicInteger nodeIdx = new AtomicInteger(initNodes);
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+ @Override public Void call() {
+ try {
+ startGrid(nodeIdx.getAndIncrement());
+ }
+ catch (Exception e) {
+ error("Start failed: " + e);
+ }
+
+ return null;
+ }
+ }, startNodes, "start-node");
+
+ int cnt = 0;
+
+ DiscoveryEvent[] expEvts = new DiscoveryEvent[startNodes - failCnt];
+
+ int expEvtCnt = 0;
+
+ sesTimeout = 1000;
+
+ List<ZkTestClientCnxnSocketNIO> blockedC = new ArrayList<>();
+
+ final List<String> failedZkNodes = new ArrayList<>(failCnt);
+
+ for (int i = initNodes; i < initNodes + startNodes; i++) {
+ final ZookeeperDiscoverySpi spi = waitSpi(getTestIgniteInstanceName(i));
+
+ assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ Object spiImpl = GridTestUtils.getFieldValue(spi, "impl");
+
+ if (spiImpl == null)
+ return false;
+
+ long internalOrder = GridTestUtils.getFieldValue(spiImpl, "rtState", "internalOrder");
+
+ return internalOrder > 0;
+ }
+ }, 10_000));
+
+ if (cnt++ < failCnt) {
+ ZkTestClientCnxnSocketNIO c = ZkTestClientCnxnSocketNIO.forNode(getTestIgniteInstanceName(i));
+
+ c.closeSocket(true);
+
+ blockedC.add(c);
+
+ failedZkNodes.add(aliveZkNodePath(spi));
+ }
+ else {
+ expEvts[expEvtCnt] = joinEvent(initNodes + expEvtCnt + 1);
+
+ expEvtCnt++;
+ }
+ }
+
+ waitNoAliveZkNodes(log, zkCluster.getConnectString(), failedZkNodes, 30_000);
+
+ c0.allowConnect();
+
+ for (ZkTestClientCnxnSocketNIO c : blockedC)
+ c.allowConnect();
+
+ if (expEvts.length > 0) {
+ for (int i = 0; i < initNodes; i++)
+ checkEvents(ignite(i), expEvts);
+ }
+
+ fut.get();
+
+ waitForTopology(initNodes + startNodes - failCnt);
+ }
+
+ /**
+ * @param node Node.
+ * @return Corresponding znode.
+ */
+ private static String aliveZkNodePath(Ignite node) {
+ return aliveZkNodePath(node.configuration().getDiscoverySpi());
+ }
+
+ /**
+ * @param spi SPI.
+ * @return Znode related to given SPI.
+ */
+ private static String aliveZkNodePath(DiscoverySpi spi) {
+ String path = GridTestUtils.getFieldValue(spi, "impl", "rtState", "locNodeZkPath");
+
+ return path.substring(path.lastIndexOf('/') + 1);
+ }
+
+ /**
+ * @param log Logger.
+ * @param connectString Zookeeper connect string.
+ * @param failedZkNodes Znodes which should be removed.
+ * @param timeout Timeout.
+ * @throws Exception If failed.
+ */
+ private static void waitNoAliveZkNodes(final IgniteLogger log,
+ String connectString,
+ final List<String> failedZkNodes,
+ long timeout)
+ throws Exception
+ {
+ final ZookeeperClient zkClient = new ZookeeperClient(log, connectString, 10_000, null);
+
+ try {
+ assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ try {
+ List<String> c = zkClient.getChildren(IGNITE_ZK_ROOT + "/" + ZkIgnitePaths.ALIVE_NODES_DIR);
+
+ for (String failedZkNode : failedZkNodes) {
+ if (c.contains(failedZkNode)) {
+ log.info("Alive node is not removed [node=" + failedZkNode + ", all=" + c + ']');
+
+ return false;
+ }
+ }
+
+ return true;
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+
+ fail();
+
+ return true;
+ }
+ }
+ }, timeout));
+ }
+ finally {
+ zkClient.close();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConcurrentStartWithClient() throws Exception {
+ final int NODES = 20;
+
+ for (int i = 0; i < 3; i++) {
+ info("Iteration: " + i);
+
+ final int srvIdx = ThreadLocalRandom.current().nextInt(NODES);
+
+ final AtomicInteger idx = new AtomicInteger();
+
+ GridTestUtils.runMultiThreaded(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ int threadIdx = idx.getAndIncrement();
+
+ clientModeThreadLocal(threadIdx == srvIdx || ThreadLocalRandom.current().nextBoolean());
+
+ startGrid(threadIdx);
+
+ return null;
+ }
+ }, NODES, "start-node");
+
+ waitForTopology(NODES);
+
+ stopAllGrids();
+
+ checkEventsConsistency();
+
+ evts.clear();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConcurrentStart() throws Exception {
+ final int NODES = 20;
+
+ for (int i = 0; i < 3; i++) {
+ info("Iteration: " + i);
+
+ final AtomicInteger idx = new AtomicInteger();
+
+ final CyclicBarrier b = new CyclicBarrier(NODES);
+
+ GridTestUtils.runMultiThreaded(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ b.await();
+
+ int threadIdx = idx.getAndIncrement();
+
+ startGrid(threadIdx);
+
+ return null;
+ }
+ }, NODES, "start-node");
+
+ waitForTopology(NODES);
+
+ stopAllGrids();
+
+ checkEventsConsistency();
+
+ evts.clear();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConcurrentStartStop1() throws Exception {
+ concurrentStartStop(1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConcurrentStartStop2() throws Exception {
+ concurrentStartStop(5);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConcurrentStartStop2_EventsThrottle() throws Exception {
+ System.setProperty(ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS, "1");
+
+ try {
+ concurrentStartStop(5);
+ }
+ finally {
+ System.clearProperty(ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS);
+ }
+ }
+
+ /**
+ * @param initNodes Number of initially started nnodes.
+ * @throws Exception If failed.
+ */
+ private void concurrentStartStop(final int initNodes) throws Exception {
+ startGrids(initNodes);
+
+ final int NODES = 5;
+
+ long topVer = initNodes;
+
+ for (int i = 0; i < 10; i++) {
+ info("Iteration: " + i);
+
+ DiscoveryEvent[] expEvts = new DiscoveryEvent[NODES];
+
+ startGridsMultiThreaded(initNodes, NODES);
+
+ for (int j = 0; j < NODES; j++)
+ expEvts[j] = joinEvent(++topVer);
+
+ checkEvents(ignite(0), expEvts);
+
+ checkEventsConsistency();
+
+ final CyclicBarrier b = new CyclicBarrier(NODES);
+
+ GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+ @Override public void apply(Integer idx) {
+ try {
+ b.await();
+
+ stopGrid(initNodes + idx);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+
+ fail();
+ }
+ }
+ }, NODES, "stop-node");
+
+ for (int j = 0; j < NODES; j++)
+ expEvts[j] = failEvent(++topVer);
+
+ checkEventsConsistency();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testClusterRestart() throws Exception {
+ startGridsMultiThreaded(3, false);
+
+ stopAllGrids();
+
+ evts.clear();
+
+ startGridsMultiThreaded(3, false);
+
+ waitForTopology(3);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConnectionRestore4() throws Exception {
+ testSockNio = true;
+
+ Ignite node0 = startGrid(0);
+
+ ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0);
+
+ c0.closeSocket(false);
+
+ startGrid(1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testStartStop_1_Node() throws Exception {
+ startGrid(0);
+
+ waitForTopology(1);
+
+ stopGrid(0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRestarts_2_Nodes() throws Exception {
+ startGrid(0);
+
+ for (int i = 0; i < 10; i++) {
+ info("Iteration: " + i);
+
+ startGrid(1);
+
+ waitForTopology(2);
+
+ stopGrid(1);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testStartStop_2_Nodes_WithCache() throws Exception {
+ startGrids(2);
+
+ for (Ignite node : G.allGrids()) {
+ IgniteCache<Object, Object> cache = node.cache(DEFAULT_CACHE_NAME);
+
+ assertNotNull(cache);
+
+ for (int i = 0; i < 100; i++) {
+ cache.put(i, node.name());
+
+ assertEquals(node.name(), cache.get(i));
+ }
+ }
+
+ awaitPartitionMapExchange();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testStartStop_2_Nodes() throws Exception {
+ ackEveryEventSystemProperty();
+
+ startGrid(0);
+
+ waitForTopology(1);
+
+ startGrid(1);
+
+ waitForTopology(2);
+
+ for (Ignite node : G.allGrids())
+ node.compute().broadcast(new DummyCallable(null));
+
+ awaitPartitionMapExchange();
+
+ waitForEventsAcks(ignite(0));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMultipleClusters() throws Exception {
+ Ignite c0 = startGrid(0);
+
+ zkRootPath = "/cluster2";
+
+ Ignite c1 = startGridsMultiThreaded(1, 5);
+
+ zkRootPath = "/cluster3";
+
+ Ignite c2 = startGridsMultiThreaded(6, 3);
+
+ checkNodesNumber(c0, 1);
+ checkNodesNumber(c1, 5);
+ checkNodesNumber(c2, 3);
+
+ stopGrid(2);
+
+ checkNodesNumber(c0, 1);
+ checkNodesNumber(c1, 4);
+ checkNodesNumber(c2, 3);
+
+ for (int i = 0; i < 3; i++)
+ stopGrid(i + 6);
+
+ checkNodesNumber(c0, 1);
+ checkNodesNumber(c1, 4);
+
+ c2 = startGridsMultiThreaded(6, 2);
+
+ checkNodesNumber(c0, 1);
+ checkNodesNumber(c1, 4);
+ checkNodesNumber(c2, 2);
+
+ evts.clear();
+ }
+
+ /**
+ * @param node Node.
+ * @param expNodes Expected node in cluster.
+ * @throws Exception If failed.
+ */
+ private void checkNodesNumber(final Ignite node, final int expNodes) throws Exception {
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return node.cluster().nodes().size() == expNodes;
+ }
+ }, 5000);
+
+ assertEquals(expNodes, node.cluster().nodes().size());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testStartStop1() throws Exception {
+ ackEveryEventSystemProperty();
+
+ startGridsMultiThreaded(5, false);
+
+ waitForTopology(5);
+
+ awaitPartitionMapExchange();
+
+ waitForEventsAcks(ignite(0));
+
+ stopGrid(0);
+
+ waitForTopology(4);
+
+ for (Ignite node : G.allGrids())
+ node.compute().broadcast(new DummyCallable(null));
+
+ startGrid(0);
+
+ waitForTopology(5);
+
+ awaitPartitionMapExchange();
+
+ waitForEventsAcks(grid(CU.oldest(ignite(1).cluster().nodes())));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testStartStop3() throws Exception {
+ startGrids(4);
+
+ awaitPartitionMapExchange();
+
+ stopGrid(0);
+
+ startGrid(5);
+
+ awaitPartitionMapExchange();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testStartStop4() throws Exception {
+ startGrids(6);
+
+ awaitPartitionMapExchange();
+
+ stopGrid(2);
+
+ if (ThreadLocalRandom.current().nextBoolean())
+ awaitPartitionMapExchange();
+
+ stopGrid(1);
+
+ if (ThreadLocalRandom.current().nextBoolean())
+ awaitPartitionMapExchange();
+
+ stopGrid(0);
+
+ if (ThreadLocalRandom.current().nextBoolean())
+ awaitPartitionMapExchange();
+
+ startGrid(7);
+
+ awaitPartitionMapExchange();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testStartStop2() throws Exception {
+ startGridsMultiThreaded(10, false);
+
+ GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+ @Override public void apply(Integer idx) {
+ stopGrid(idx);
+ }
+ }, 3, "stop-node-thread");
+
+ waitForTopology(7);
+
+ startGridsMultiThreaded(0, 3);
+
+ waitForTopology(10);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testStartStopWithClients() throws Exception {
+ final int SRVS = 3;
+
+ startGrids(SRVS);
+
+ clientMode(true);
+
+ final int THREADS = 30;
+
+ for (int i = 0; i < 5; i++) {
+ info("Iteration: " + i);
+
+ startGridsMultiThreaded(SRVS, THREADS);
+
+ waitForTopology(SRVS + THREADS);
+
+ GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+ @Override public void apply(Integer idx) {
+ stopGrid(idx + SRVS);
+ }
+ }, THREADS, "stop-node");
+
+ waitForTopology(SRVS);
+
+ checkEventsConsistency();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testTopologyChangeMultithreaded() throws Exception {
+ topologyChangeWithRestarts(false, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testTopologyChangeMultithreaded_RestartZk() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8184");
+
+ try {
+ topologyChangeWithRestarts(true, false);
+ }
+ finally {
+ zkCluster.stop();
+
+ zkCluster = null;
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testTopologyChangeMultithreaded_RestartZk_CloseClients() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8184");
+
+ try {
+ topologyChangeWithRestarts(true, true);
+ }
+ finally {
+ zkCluster.stop();
+
+ zkCluster = null;
+ }
+ }
+
+ /**
+ * @param restartZk If {@code true} in background restarts on of ZK servers.
+ * @param closeClientSock If {@code true} in background closes zk clients' sockets.
+ * @throws Exception If failed.
+ */
+ private void topologyChangeWithRestarts(boolean restartZk, boolean closeClientSock) throws Exception {
+ sesTimeout = 30_000;
+
+ if (closeClientSock)
+ testSockNio = true;
+
+ long stopTime = System.currentTimeMillis() + 60_000;
+
+ AtomicBoolean stop = new AtomicBoolean();
+
+ IgniteInternalFuture<?> fut1 = null;
+
+ IgniteInternalFuture<?> fut2 = null;
+
+ try {
+ fut1 = restartZk ? startRestartZkServers(stopTime, stop) : null;
+ fut2 = closeClientSock ? startCloseZkClientSocket(stopTime, stop) : null;
+
+ int INIT_NODES = 10;
+
+ startGridsMultiThreaded(INIT_NODES);
+
+ final int MAX_NODES = 20;
+
+ final List<Integer> startedNodes = new ArrayList<>();
+
+ for (int i = 0; i < INIT_NODES; i++)
+ startedNodes.add(i);
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ final AtomicInteger startIdx = new AtomicInteger(INIT_NODES);
+
+ while (System.currentTimeMillis() < stopTime) {
+ if (startedNodes.size() >= MAX_NODES) {
+ int stopNodes = rnd.nextInt(5) + 1;
+
+ log.info("Next, stop nodes: " + stopNodes);
+
+ final List<Integer> idxs = new ArrayList<>();
+
+ while (idxs.size() < stopNodes) {
+ Integer stopIdx = rnd.nextInt(startedNodes.size());
+
+ if (!idxs.contains(stopIdx))
+ idxs.add(startedNodes.get(stopIdx));
+ }
+
+ GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+ @Override public void apply(Integer threadIdx) {
+ int stopNodeIdx = idxs.get(threadIdx);
+
+ info("Stop node: " + stopNodeIdx);
+
+ stopGrid(stopNodeIdx);
+ }
+ }, stopNodes, "stop-node");
+
+ startedNodes.removeAll(idxs);
+ }
+ else {
+ int startNodes = rnd.nextInt(5) + 1;
+
+ log.info("Next, start nodes: " + startNodes);
+
+ GridTestUtils.runMultiThreaded(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ int idx = startIdx.incrementAndGet();
+
+ log.info("Start node: " + idx);
+
+ startGrid(idx);
+
+ synchronized (startedNodes) {
+ startedNodes.add(idx);
+ }
+
+ return null;
+ }
+ }, startNodes, "start-node");
+ }
+
+ U.sleep(rnd.nextInt(100) + 1);
+ }
+ }
+ finally {
+ stop.set(true);
+ }
+
+ if (fut1 != null)
+ fut1.get();
+
+ if (fut2 != null)
+ fut2.get();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRandomTopologyChanges() throws Exception {
+ randomTopologyChanges(false, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void checkZkNodesCleanup() throws Exception {
+ final ZookeeperClient zkClient = new ZookeeperClient(getTestResources().getLogger(),
+ zkCluster.getConnectString(),
+ 30_000,
+ null);
+
+ final String basePath = IGNITE_ZK_ROOT + "/";
+
+ final String aliveDir = basePath + ZkIgnitePaths.ALIVE_NODES_DIR + "/";
+
+ try {
+ List<String> znodes = listSubTree(zkClient.zk(), IGNITE_ZK_ROOT);
+
+ boolean foundAlive = false;
+
+ for (String znode : znodes) {
+ if (znode.startsWith(aliveDir)) {
+ foundAlive = true;
+
+ break;
+ }
+ }
+
+ assertTrue(foundAlive); // Sanity check to make sure we check correct directory.
+
+ assertTrue("Failed to wait for unused znodes cleanup", GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ try {
+ List<String> znodes = listSubTree(zkClient.zk(), IGNITE_ZK_ROOT);
+
+ for (String znode : znodes) {
+ if (znode.startsWith(aliveDir) || znode.length() < basePath.length())
+ continue;
+
+ znode = znode.substring(basePath.length());
+
+ if (!znode.contains("/")) // Ignore roots.
+ continue;
+
+ // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8193
+ if (znode.startsWith("jd/"))
+ continue;
+
+ log.info("Found unexpected znode: " + znode);
+
+ return false;
+ }
+
+ return true;
+ }
+ catch (Exception e) {
+ error("Unexpected error: " + e, e);
+
+ fail("Unexpected error: " + e);
+ }
+
+ return false;
+ }
+ }, 10_000));
+ }
+ finally {
+ zkClient.close();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRandomTopologyChanges_RestartZk() throws Exception {
+ randomTopologyChanges(true, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRandomTopologyChanges_CloseClients() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8182");
+
+ randomTopologyChanges(false, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeployService1() throws Exception {
+ startGridsMultiThreaded(3);
+
+ grid(0).services(grid(0).cluster()).deployNodeSingleton("test", new GridCacheAbstractFullApiSelfTest.DummyServiceImpl());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeployService2() throws Exception {
+ clientMode(false);
+
+ startGrid(0);
+
+ clientMode(true);
+
+ startGrid(1);
+
+ grid(0).services(grid(0).cluster()).deployNodeSingleton("test", new GridCacheAbstractFullApiSelfTest.DummyServiceImpl());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeployService3() throws Exception {
+ IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() {
+ @Override public Object call() throws Exception {
+ clientModeThreadLocal(true);
+
+ startGrid(0);
+
+ return null;
+ }
+ }, "start-node");
+
+ clientModeThreadLocal(false);
+
+ startGrid(1);
+
+ fut.get();
+
+ grid(0).services(grid(0).cluster()).deployNodeSingleton("test", new GridCacheAbstractFullApiSelfTest.DummyServiceImpl());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testLargeUserAttribute1() throws Exception {
+ initLargeAttribute();
+
+ startGrid(0);
+
+ checkZkNodesCleanup();
+
+ userAttrs = null;
+
+ startGrid(1);
+
+ waitForEventsAcks(ignite(0));
+
+ waitForTopology(2);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testLargeUserAttribute2() throws Exception {
+ startGrid(0);
+
+ initLargeAttribute();
+
+ startGrid(1);
+
+ waitForEventsAcks(ignite(0));
+
+ checkZkNodesCleanup();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testLargeUserAttribute3() throws Exception {
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ long stopTime = System.currentTimeMillis() + 60_000;
+
+ int nodes = 0;
+
+ for (int i = 0; i < 25; i++) {
+ info("Iteration: " + i);
+
+ if (rnd.nextBoolean())
+ initLargeAttribute();
+ else
+ userAttrs = null;
+
+ clientMode(i > 5);
+
+ startGrid(i);
+
+ nodes++;
+
+ if (System.currentTimeMillis() >= stopTime)
+ break;
+ }
+
+ waitForTopology(nodes);
+ }
+
+ /**
+ *
+ */
+ private void initLargeAttribute() {
+ userAttrs = new HashMap<>();
+
+ int[] attr = new int[1024 * 1024 + ThreadLocalRandom.current().nextInt(1024)];
+
+ for (int i = 0; i < attr.length; i++)
+ attr[i] = i;
+
+ userAttrs.put("testAttr", attr);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testLargeCustomEvent() throws Exception {
+ Ignite srv0 = startGrid(0);
+
+ // Send large message, single node in topology.
+ IgniteCache<Object, Object> cache = srv0.createCache(largeCacheConfiguration("c1"));
+
+ for (int i = 0; i < 100; i++)
+ cache.put(i, i);
+
+ assertEquals(1, cache.get(1));
+
+ waitForEventsAcks(ignite(0));
+
+ startGridsMultiThreaded(1, 3);
+
+ srv0.destroyCache("c1");
+
+ // Send large message, multiple nodes in topology.
+ cache = srv0.createCache(largeCacheConfiguration("c1"));
+
+ for (int i = 0; i < 100; i++)
+ cache.put(i, i);
+
+ waitForTopology(4);
+
+ ignite(3).createCache(largeCacheConfiguration("c2"));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testClientReconnectSessionExpire1_1() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8131");
+
+ clientReconnectSessionExpire(false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testClientReconnectSessionExpire1_2() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8131");
+
+ clientReconnectSessionExpire(true);
+ }
+
+ /**
+ * @param closeSock Test mode flag.
+ * @throws Exception If failed.
+ */
+ private void clientReconnectSessionExpire(boolean closeSock) throws Exception {
+ startGrid(0);
+
+ sesTimeout = 2000;
+ clientMode(true);
+ testSockNio = true;
+
+ Ignite client = startGrid(1);
+
+ client.cache(DEFAULT_CACHE_NAME).put(1, 1);
+
+ reconnectClientNodes(log, Collections.singletonList(client), closeSock);
+
+ assertEquals(1, client.cache(DEFAULT_CACHE_NAME).get(1));
+
+ client.compute().broadcast(new DummyCallable(null));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testForceClientReconnect() throws Exception {
+ final int SRVS = 3;
+
+ startGrids(SRVS);
+
+ clientMode(true);
+
+ startGrid(SRVS);
+
+ reconnectClientNodes(Collections.singletonList(ignite(SRVS)), new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ ZookeeperDiscoverySpi spi = waitSpi(getTestIgniteInstanceName(SRVS));
+
+ spi.clientReconnect();
+
+ return null;
+ }
+ });
+
+ waitForTopology(SRVS + 1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testForcibleClientFail() throws Exception {
+ final int SRVS = 3;
+
+ startGrids(SRVS);
+
+ clientMode(true);
+
+ startGrid(SRVS);
+
+ reconnectClientNodes(Collections.singletonList(ignite(SRVS)), new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ ZookeeperDiscoverySpi spi = waitSpi(getTestIgniteInstanceName(0));
+
+ spi.failNode(ignite(SRVS).cluster().localNode().id(), "Test forcible node fail");
+
+ return null;
+ }
+ });
+
+ waitForTopology(SRVS + 1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDuplicatedNodeId() throws Exception {
+ UUID nodeId0 = nodeId = UUID.randomUUID();
+
+ startGrid(0);
+
+ int failingNodeIdx = 100;
+
+ for (int i = 0; i < 5; i++) {
+ final int idx = failingNodeIdx++;
+
+ nodeId = nodeId0;
+
+ info("Start node with duplicated ID [iter=" + i + ", nodeId=" + nodeId + ']');
+
+ Throwable err = GridTestUtils.assertThrows(log, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ startGrid(idx);
+
+ return null;
+ }
+ }, IgniteCheckedException.class, null);
+
+ IgniteSpiException spiErr = X.cause(err, IgniteSpiException.class);
+
+ assertNotNull(spiErr);
+ assertTrue(spiErr.getMessage().contains("Node with the same ID already exists"));
+
+ nodeId = null;
+
+ info("Start node with unique ID [iter=" + i + ']');
+
+ Ignite ignite = startGrid(idx);
+
+ nodeId0 = ignite.cluster().localNode().id();
+
+ waitForTopology(i + 2);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testPing() throws Exception {
+ sesTimeout = 5000;
+
+ startGrids(3);
+
+ final ZookeeperDiscoverySpi spi = waitSpi(getTestIgniteInstanceName(1));
+
+ final UUID nodeId = ignite(2).cluster().localNode().id();
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+ @Override public void run() {
+ assertTrue(spi.pingNode(nodeId));
+ }
+ }, 32, "ping");
+
+ fut.get();
+
+ fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+ @Override public void run() {
+ spi.pingNode(nodeId);
+ }
+ }, 32, "ping");
+
+ U.sleep(100);
+
+ stopGrid(2);
+
+ fut.get();
+
+ fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+ @Override public void run() {
+ assertFalse(spi.pingNode(nodeId));
+ }
+ }, 32, "ping");
+
+ fut.get();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testWithPersistence1() throws Exception {
+ startWithPersistence(false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testWithPersistence2() throws Exception {
+ startWithPersistence(true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testNoOpCommunicationFailureResolve_1() throws Exception {
+ communicationFailureResolve_Simple(2);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testNoOpCommunicationErrorResolve_2() throws Exception {
+ communicationFailureResolve_Simple(10);
+ }
+
+ /**
+ * @param nodes Nodes number.
+ * @throws Exception If failed.
+ */
+ private void communicationFailureResolve_Simple(int nodes) throws Exception {
+ assert nodes > 1;
+
+ sesTimeout = 2000;
+ commFailureRslvr = NoOpCommunicationFailureResolver.FACTORY;
+
+ startGridsMultiThreaded(nodes);
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 3; i++) {
+ info("Iteration: " + i);
+
+ int idx1 = rnd.nextInt(nodes);
+
+ int idx2;
+
+ do {
+ idx2 = rnd.nextInt(nodes);
+ }
+ while (idx1 == idx2);
+
+ ZookeeperDiscoverySpi spi = spi(ignite(idx1));
+
+ spi.resolveCommunicationFailure(ignite(idx2).cluster().localNode(), new Exception("test"));
+
+ checkInternalStructuresCleanup();
+ }
+ }
+
+ /**
+ * Tests case when one node fails before sending communication status.
+ *
+ * @throws Exception If failed.
+ */
+ public void testNoOpCommunicationErrorResolve_3() throws Exception {
+ sesTimeout = 2000;
+ commFailureRslvr = NoOpCommunicationFailureResolver.FACTORY;
+
+ startGridsMultiThreaded(3);
+
+ sesTimeout = 10_000;
+
+ testSockNio = true;
+ sesTimeout = 5000;
+
+ startGrid(3);
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Object>() {
+ @Override public Object call() {
+ ZookeeperDiscoverySpi spi = spi(ignite(0));
+
+ spi.resolveCommunicationFailure(ignite(1).cluster().localNode(), new Exception("test"));
+
+ return null;
+ }
+ });
+
+ U.sleep(1000);
+
+ ZkTestClientCnxnSocketNIO nio = ZkTestClientCnxnSocketNIO.forNode(ignite(3));
+
+ nio.closeSocket(true);
+
+ try {
+ stopGrid(3);
+
+ fut.get();
+ }
+ finally {
+ nio.allowConnect();
+ }
+
+ waitForTopology(3);
+ }
+
+ /**
+ * Tests case when Coordinator fails while resolve process is in progress.
+ *
+ * @throws Exception If failed.
+ */
+ public void testNoOpCommunicationErrorResolve_4() throws Exception {
+ testCommSpi = true;
+
+ sesTimeout = 2000;
+ commFailureRslvr = NoOpCommunicationFailureResolver.FACTORY;
+
+ startGrid(0);
+
+ startGridsMultiThreaded(1, 3);
+
+ ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.testSpi(ignite(3));
+
+ commSpi.pingLatch = new CountDownLatch(1);
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Object>() {
+ @Override public Object call() {
+ ZookeeperDiscoverySpi spi = spi(ignite(1));
+
+ spi.resolveCommunicationFailure(ignite(2).cluster().localNode(), new Exception("test"));
+
+ return null;
+ }
+ });
+
+ U.sleep(1000);
+
+ assertFalse(fut.isDone());
+
+ stopGrid(0);
+
+ commSpi.pingLatch.countDown();
+
+ fut.get();
+
+ waitForTopology(3);
+ }
+
+ /**
+ * Tests that nodes join is delayed while resolve is in progress.
+ *
+ * @throws Exception If failed.
+ */
+ public void testNoOpCommunicationErrorResolve_5() throws Exception {
+ testCommSpi = true;
+
+ sesTimeout = 2000;
+ commFailureRslvr = NoOpCommunicationFailureResolver.FACTORY;
+
+ startGrid(0);
+
+ startGridsMultiThreaded(1, 3);
+
+ ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.testSpi(ignite(3));
+
+ commSpi.pingStartLatch = new CountDownLatch(1);
+ commSpi.pingLatch = new CountDownLatch(1);
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Object>() {
+ @Override public Object call() {
+ ZookeeperDiscoverySpi spi = spi(ignite(1));
+
+ spi.resolveCommunicationFailure(ignite(2).cluster().localNode(), new Exception("test"));
+
+ return null;
+ }
+ });
+
+ assertTrue(commSpi.pingStartLatch.await(10, SECONDS));
+
+ try {
+ assertFalse(fut.isDone());
+
+ final AtomicInteger nodeIdx = new AtomicInteger(3);
+
+ IgniteInternalFuture<?> startFut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ startGrid(nodeIdx.incrementAndGet());
+
+ return null;
+ }
+ }, 3, "start-node");
+
+ U.sleep(1000);
+
+ assertFalse(startFut.isDone());
+
+ assertEquals(4, ignite(0).cluster().nodes().size());
+
+ commSpi.pingLatch.countDown();
+
+ startFut.get();
+ fut.get();
+
+ waitForTopology(7);
+ }
+ finally {
+ commSpi.pingLatch.countDown();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillNode_1() throws Exception {
+ communicationFailureResolve_KillNodes(2, Collections.singleton(2L));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillNode_2() throws Exception {
+ communicationFailureResolve_KillNodes(3, Collections.singleton(2L));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillNode_3() throws Exception {
+ communicationFailureResolve_KillNodes(10, Arrays.asList(2L, 4L, 6L));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillCoordinator_1() throws Exception {
+ communicationFailureResolve_KillNodes(2, Collections.singleton(1L));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillCoordinator_2() throws Exception {
+ communicationFailureResolve_KillNodes(3, Collections.singleton(1L));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillCoordinator_3() throws Exception {
+ communicationFailureResolve_KillNodes(10, Arrays.asList(1L, 4L, 6L));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillCoordinator_4() throws Exception {
+ communicationFailureResolve_KillNodes(10, Arrays.asList(1L, 2L, 3L));
+ }
+
+ /**
+ * @param startNodes Number of nodes to start.
+ * @param killNodes Nodes to kill by resolve process.
+ * @throws Exception If failed.
+ */
+ private void communicationFailureResolve_KillNodes(int startNodes, Collection<Long> killNodes) throws Exception {
+ testCommSpi = true;
+
+ commFailureRslvr = TestNodeKillCommunicationFailureResolver.factory(killNodes);
+
+ startGrids(startNodes);
+
+ ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.testSpi(ignite(0));
+
+ commSpi.checkRes = new BitSet(startNodes);
+
+ ZookeeperDiscoverySpi spi = null;
+ UUID killNodeId = null;
+
+ for (Ignite node : G.allGrids()) {
+ ZookeeperDiscoverySpi spi0 = spi(node);
+
+ if (!killNodes.contains(node.cluster().localNode().order()))
+ spi = spi0;
+ else
+ killNodeId = node.cluster().localNode().id();
+ }
+
+ assertNotNull(spi);
+ assertNotNull(killNodeId);
+
+ try {
+ spi.resolveCommunicationFailure(spi.getNode(killNodeId), new Exception("test"));
+
+ fail("Exception is not thrown");
+ }
+ catch (IgniteSpiException e) {
+ assertTrue("Unexpected exception: " + e, e.getCause() instanceof ClusterTopologyCheckedException);
+ }
+
+ int expNodes = startNodes - killNodes.size();
+
+ waitForTopology(expNodes);
+
+ for (Ignite node : G.allGrids())
+ assertFalse(killNodes.contains(node.cluster().localNode().order()));
+
+ startGrid(startNodes);
+
+ waitForTopology(expNodes + 1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationFailureResolve_KillCoordinator_5() throws Exception {
+ sesTimeout = 2000;
+
+ testCommSpi = true;
+ commFailureRslvr = KillCoordinatorCommunicationFailureResolver.FACTORY;
+
+ startGrids(10);
+
+ int crd = 0;
+
+ int nodeIdx = 10;
+
+ for (int i = 0; i < 10; i++) {
+ info("Iteration: " + i);
+
+ for (Ignite node : G.allGrids())
+ ZkTestCommunicationSpi.testSpi(node).initCheckResult(10);
+
+ UUID crdId = ignite(crd).cluster().localNode().id();
+
+ ZookeeperDiscoverySpi spi = spi(ignite(crd + 1));
+
+ try {
+ spi.resolveCommunicationFailure(spi.getNode(crdId), new Exception("test"));
+
+ fail("Exception is not thrown");
+ }
+ catch (IgniteSpiException e) {
+ assertTrue("Unexpected exception: " + e, e.getCause() instanceof ClusterTopologyCheckedException);
+ }
+
+ waitForTopology(9);
+
+ startGrid(nodeIdx++);
+
+ waitForTopology(10);
+
+ crd++;
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationFailureResolve_KillRandom() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8179");
+
+ sesTimeout = 2000;
+
+ testCommSpi = true;
+ commFailureRslvr = KillRandomCommunicationFailureResolver.FACTORY;
+
+ startGridsMultiThreaded(10);
+
+ clientMode(true);
+
+ startGridsMultiThreaded(10, 5);
+
+ int nodeIdx = 15;
+
+ for (int i = 0; i < 10; i++) {
+ info("Iteration: " + i);
+
+ ZookeeperDiscoverySpi spi = null;
+
+ for (Ignite node : G.allGrids()) {
+ ZkTestCommunicationSpi.testSpi(node).initCheckResult(100);
+
+ spi = spi(node);
+ }
+
+ assert spi != null;
+
+ try {
+ spi.resolveCommunicationFailure(spi.getRemoteNodes().iterator().next(), new Exception("test"));
+ }
+ catch (IgniteSpiException ignore) {
+ // No-op.
+ }
+
+ clientMode(ThreadLocalRandom.current().nextBoolean());
+
+ startGrid(nodeIdx++);
+
+ awaitPartitionMapExchange();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDefaultCommunicationFailureResolver1() throws Exception {
+ testCommSpi = true;
+ sesTimeout = 5000;
+
+ startGrids(3);
+
+ ZkTestCommunicationSpi.testSpi(ignite(0)).initCheckResult(3, 0, 1);
+ ZkTestCommunicationSpi.testSpi(ignite(1)).initCheckResult(3, 0, 1);
+ ZkTestCommunicationSpi.testSpi(ignite(2)).initCheckResult(3, 2);
+
+ UUID killedId = nodeId(2);
+
+ assertNotNull(ignite(0).cluster().node(killedId));
+
+ ZookeeperDiscoverySpi spi = spi(ignite(0));
+
+ spi.resolveCommunicationFailure(spi.getNode(ignite(1).cluster().localNode().id()), new Exception("test"));
+
+ waitForTopology(2);
+
+ assertNull(ignite(0).cluster().node(killedId));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDefaultCommunicationFailureResolver2() throws Exception {
+ testCommSpi = true;
+ sesTimeout = 5000;
+
+ startGrids(3);
+
+ clientMode(true);
+
+ startGridsMultiThreaded(3, 2);
+
+ ZkTestCommunicationSpi.testSpi(ignite(0)).initCheckResult(5, 0, 1);
+ ZkTestCommunicationSpi.testSpi(ignite(1)).initCheckResult(5, 0, 1);
+ ZkTestCommunicationSpi.testSpi(ignite(2)).initCheckResult(5, 2, 3, 4);
+ ZkTestCommunicationSpi.testSpi(ignite(3)).initCheckResult(5, 2, 3, 4);
+ ZkTestCommunicationSpi.testSpi(ignite(4)).initCheckResult(5, 2, 3, 4);
+
+ ZookeeperDiscoverySpi spi = spi(ignite(0));
+
+ spi.resolveCommunicationFailure(spi.getNode(ignite(1).cluster().localNode().id()), new Exception("test"));
+
+ waitForTopology(2);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDefaultCommunicationFailureResolver3() throws Exception {
+ defaultCommunicationFailureResolver_BreakCommunication(3, 1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDefaultCommunicationFailureResolver4() throws Exception {
+ defaultCommunicationFailureResolver_BreakCommunication(3, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDefaultCommunicationFailureResolver5() throws Exception {
+ defaultCommunicationFailureResolver_BreakCommunication(10, 1, 3, 6);
+ }
+
+ /**
+ * @param startNodes Initial nodes number.
+ * @param breakNodes Node indices where communication server is closed.
+ * @throws Exception If failed.
+ */
+ private void defaultCommunicationFailureResolver_BreakCommunication(int startNodes, final int...breakNodes) throws Exception {
+ sesTimeout = 5000;
+
+ startGridsMultiThreaded(startNodes);
+
+ final CyclicBarrier b = new CyclicBarrier(breakNodes.length);
+
+ GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+ @Override public void apply(Integer threadIdx) {
+ try {
+ b.await();
+
+ int nodeIdx = breakNodes[threadIdx];
+
+ info("Close communication: " + nodeIdx);
+
+ ((TcpCommunicationSpi)ignite(nodeIdx).configuration().getCommunicationSpi()).simulateNodeFailure();
+ }
+ catch (Exception e) {
+ fail("Unexpected error: " + e);
+ }
+ }
+ }, breakNodes.length, "break-communication");
+
+ waitForTopology(startNodes - breakNodes.length);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationFailureResolve_CachesInfo1() throws Exception {
+ testCommSpi = true;
+ sesTimeout = 5000;
+
+ final CacheInfoCommunicationFailureResolver rslvr = new CacheInfoCommunicationFailureResolver();
+
+ commFailureRslvr = new IgniteOutClosure<CommunicationFailureResolver>() {
+ @Override public CommunicationFailureResolver apply() {
+ return rslvr;
+ }
+ };
+
+ startGrids(2);
+
+ awaitPartitionMapExchange();
+
+ Map<String, T3<Integer, Integer, Integer>> expCaches = new HashMap<>();
+
+ expCaches.put(DEFAULT_CACHE_NAME, new T3<>(RendezvousAffinityFunction.DFLT_PARTITION_COUNT, 0, 1));
+
+ checkResolverCachesInfo(ignite(0), expCaches);
+
+ List<CacheConfiguration> caches = new ArrayList<>();
+
+ CacheConfiguration c1 = new CacheConfiguration("c1");
+ c1.setBackups(1);
+ c1.setAffinity(new RendezvousAffinityFunction(false, 64));
+ caches.add(c1);
+
+ CacheConfiguration c2 = new CacheConfiguration("c2");
+ c2.setBackups(2);
+ c2.setAffinity(new RendezvousAffinityFunction(false, 128));
+ caches.add(c2);
+
+ CacheConfiguration c3 = new CacheConfiguration("c3");
+ c3.setCacheMode(CacheMode.REPLICATED);
+ c3.setAffinity(new RendezvousAffinityFunction(false, 256));
+ caches.add(c3);
+
+ ignite(0).createCaches(caches);
+
+ expCaches.put("c1", new T3<>(64, 1, 2));
+ expCaches.put("c2", new T3<>(128, 2, 2));
+ expCaches.put("c3", new T3<>(256, 1, 2));
+
+ checkResolverCachesInfo(ignite(0), expCaches);
+
+ startGrid(2);
+ startGrid(3);
+
+ awaitPartitionMapExchange();
+
+ expCaches.put("c2", new T3<>(128, 2, 3));
+ expCaches.put("c3", new T3<>(256, 1, 4));
+
+ checkResolverCachesInfo(ignite(0), expCaches);
+
+ CacheConfiguration c4 = new CacheConfiguration("c4");
+ c4.setCacheMode(CacheMode.PARTITIONED);
+ c4.setBackups(0);
+ c4.setAffinity(new RendezvousAffinityFunction(false, 256));
+ c4.setNodeFilter(new TestCacheNodeExcludingFilter(getTestIgniteInstanceName(0), getTestIgniteInstanceName(1)));
+
+ ignite(2).createCache(c4);
+
+ expCaches.put("c4", new T3<>(256, 0, 1));
+
+ checkResolverCachesInfo(ignite(0), expCaches);
+
+ stopGrid(0); // Stop current coordinator, check new coordinator will initialize required caches information.
+
+ awaitPartitionMapExchange();
+
+ expCaches.put("c3", new T3<>(256, 1, 3));
+
+ checkResolverCachesInfo(ignite(1), expCaches);
+
+ startGrid(0);
+
+ expCaches.put("c3", new T3<>(256, 1, 4));
+
+ checkResolverCachesInfo(ignite(1), expCaches);
+
+ stopGrid(1);
+
+ expCaches.put("c3", new T3<>(256, 1, 3));
+
+ checkResolverCachesInfo(ignite(3), expCaches);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationFailureResolve_CachesInfo2() throws Exception {
+ testCommSpi = true;
+ sesTimeout = 5000;
+
+ final CacheInfoCommunicationFailureResolver rslvr = new CacheInfoCommunicationFailureResolver();
+
+ commFailureRslvr = new IgniteOutClosure<CommunicationFailureResolver>() {
+ @Override public CommunicationFailureResolver apply() {
+ return rslvr;
+ }
+ };
+
+ Ignite srv0 = startGrid(0);
+
+ CacheConfiguration ccfg = new CacheConfiguration("c1
<TRUNCATED>
[35/54] [abbrv] ignite git commit: IGNITE-8106 Collect suppressed
exceptions from causes. - Fixes #3735.
Posted by ag...@apache.org.
IGNITE-8106 Collect suppressed exceptions from causes. - Fixes #3735.
Signed-off-by: Alexey Kuznetsov <ak...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/98ef9259
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/98ef9259
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/98ef9259
Branch: refs/heads/ignite-6083
Commit: 98ef925933f392d419f70b2fcf51e3655b08b290
Parents: a3eb1f5
Author: Ilya Kasnacheev <il...@gmail.com>
Authored: Wed Apr 11 19:32:52 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Apr 11 19:32:52 2018 +0700
----------------------------------------------------------------------
.../cluster/GridChangeStateCommandHandler.java | 3 +-
.../apache/ignite/internal/util/typedef/X.java | 37 +++++++++++++++-----
.../visor/util/VisorExceptionWrapper.java | 11 +++---
.../communication/tcp/TcpCommunicationSpi.java | 2 +-
.../ignite/GridSuppressedExceptionSelfTest.java | 23 +++++++++++-
5 files changed, 59 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ef9259/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
index 7bb13d9..619be34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
@@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.rest.handlers.GridRestCommandHandle
import org.apache.ignite.internal.processors.rest.request.GridRestChangeStateRequest;
import org.apache.ignite.internal.processors.rest.request.GridRestRequest;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -78,7 +79,7 @@ public class GridChangeStateCommandHandler extends GridRestCommandHandlerAdapter
sb.a(e.getMessage()).a("\n").a("suppressed: \n");
- for (Throwable t:e.getSuppressed())
+ for (Throwable t : X.getSuppressedList(e))
sb.a(t.getMessage()).a("\n");
res.setError(sb.toString());
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ef9259/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java b/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
index 395de23..1a43daa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
@@ -469,14 +469,12 @@ public final class X {
if (t == null || cls == null)
return false;
- if (t.getSuppressed() != null) {
- for (Throwable th : t.getSuppressed()) {
- if (cls.isAssignableFrom(th.getClass()))
- return true;
+ for (Throwable th : t.getSuppressed()) {
+ if (cls.isAssignableFrom(th.getClass()))
+ return true;
- if (hasSuppressed(th, cls))
- return true;
- }
+ if (hasSuppressed(th, cls))
+ return true;
}
return false;
@@ -749,6 +747,29 @@ public final class X {
}
/**
+ * Collects suppressed exceptions from throwable and all it causes.
+ *
+ * @param t Throwable.
+ * @return List of suppressed throwables.
+ */
+ public static List<Throwable> getSuppressedList(@Nullable Throwable t) {
+ List<Throwable> result = new ArrayList<>();
+
+ if (t == null)
+ return result;
+
+ do {
+ for (Throwable suppressed : t.getSuppressed()) {
+ result.add(suppressed);
+
+ result.addAll(getSuppressedList(suppressed));
+ }
+ } while ((t = t.getCause()) != null);
+
+ return result;
+ }
+
+ /**
* A way to get the entire nested stack-trace of an throwable.
*
* The result of this method is highly dependent on the JDK version
@@ -889,4 +910,4 @@ public final class X {
return dflt;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ef9259/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorExceptionWrapper.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorExceptionWrapper.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorExceptionWrapper.java
index 15e9557..ba52c5f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorExceptionWrapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorExceptionWrapper.java
@@ -17,7 +17,8 @@
package org.apache.ignite.internal.visor.util;
-import org.apache.ignite.internal.util.typedef.F;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.X;
/**
* Exception wrapper for safe for transferring to Visor.
@@ -56,12 +57,10 @@ public class VisorExceptionWrapper extends Throwable {
if (cause.getCause() != null)
initCause(new VisorExceptionWrapper(cause.getCause()));
- Throwable[] suppressed = cause.getSuppressed();
+ List<Throwable> suppressed = X.getSuppressedList(cause);
- if (!F.isEmpty(suppressed)) {
- for (Throwable sup : suppressed)
- addSuppressed(new VisorExceptionWrapper(sup));
- }
+ for (Throwable sup : suppressed)
+ addSuppressed(new VisorExceptionWrapper(sup));
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ef9259/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 9e7b592..df37dff 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3476,7 +3476,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
ctx.failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
"rmtNode=" + node +
", errs=" + errs +
- ", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
+ ", connectErrs=" + X.getSuppressedList(errs) + ']');
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ef9259/modules/core/src/test/java/org/apache/ignite/GridSuppressedExceptionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/GridSuppressedExceptionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/GridSuppressedExceptionSelfTest.java
index 6e32249..55e54fb 100644
--- a/modules/core/src/test/java/org/apache/ignite/GridSuppressedExceptionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/GridSuppressedExceptionSelfTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite;
import java.io.IOException;
+import java.util.List;
import junit.framework.TestCase;
import org.apache.ignite.internal.util.typedef.X;
@@ -70,6 +71,26 @@ public class GridSuppressedExceptionSelfTest extends TestCase {
/**
* @throws Exception If failed.
*/
+ public void testXGetSuppressedList() throws Exception {
+ IgniteCheckedException me = prepareMultiException();
+
+ assertEquals(3, X.getSuppressedList(me).size());
+
+ RuntimeException e = new RuntimeException();
+ e.addSuppressed(me);
+
+ List<Throwable> suppresseds = X.getSuppressedList(e);
+
+ assertEquals(4, suppresseds.size());
+
+ assertEquals("Test message.", suppresseds.get(0).getMessage());
+ for (int i = 1; i <= 3; i++)
+ assertEquals("Demo exception.", suppresseds.get(1).getMessage());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testXCause() throws Exception {
IgniteCheckedException me = prepareMultiException();
@@ -116,4 +137,4 @@ public class GridSuppressedExceptionSelfTest extends TestCase {
else
generateException(calls - 1, cause);
}
-}
\ No newline at end of file
+}
[37/54] [abbrv] ignite git commit: IGNITE-8221: Security for thin
clients.
Posted by ag...@apache.org.
IGNITE-8221: Security for thin clients.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5a292763
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5a292763
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5a292763
Branch: refs/heads/ignite-6083
Commit: 5a29276355c4eb8966e5825883e1232ee2a80509
Parents: 747e6c5
Author: Alexey Kukushkin <al...@yahoo.com>
Authored: Wed Apr 11 16:29:07 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Apr 11 16:38:12 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 6 +++
.../client/ClientAuthenticationException.java | 2 +-
.../client/ClientAuthorizationException.java | 46 ++++++++++++++++++++
.../internal/client/thin/ClientChannel.java | 3 +-
.../internal/client/thin/TcpClientChannel.java | 39 ++++++++---------
.../IgniteAuthenticationProcessor.java | 5 ++-
.../processors/cache/GridCacheProcessor.java | 32 ++++++++++++++
.../processors/cache/GridCacheUtils.java | 5 +++
.../client/ClientConnectionContext.java | 45 ++++++++++++++++++-
.../platform/client/ClientRequest.java | 29 ++++++++++++
.../platform/client/ClientStatus.java | 3 ++
.../cache/ClientCacheClearKeyRequest.java | 3 ++
.../cache/ClientCacheClearKeysRequest.java | 3 ++
.../client/cache/ClientCacheClearRequest.java | 3 ++
.../cache/ClientCacheContainsKeyRequest.java | 3 ++
.../cache/ClientCacheContainsKeysRequest.java | 3 ++
...ientCacheCreateWithConfigurationRequest.java | 6 ++-
.../cache/ClientCacheCreateWithNameRequest.java | 3 ++
.../client/cache/ClientCacheDestroyRequest.java | 3 ++
.../client/cache/ClientCacheGetAllRequest.java | 3 ++
.../ClientCacheGetAndPutIfAbsentRequest.java | 3 ++
.../cache/ClientCacheGetAndPutRequest.java | 3 ++
.../cache/ClientCacheGetAndRemoveRequest.java | 3 ++
.../cache/ClientCacheGetAndReplaceRequest.java | 3 ++
...acheGetOrCreateWithConfigurationRequest.java | 6 ++-
.../ClientCacheGetOrCreateWithNameRequest.java | 3 ++
.../client/cache/ClientCacheGetRequest.java | 3 ++
.../client/cache/ClientCacheGetSizeRequest.java | 3 ++
.../client/cache/ClientCachePutAllRequest.java | 3 ++
.../cache/ClientCachePutIfAbsentRequest.java | 3 ++
.../client/cache/ClientCachePutRequest.java | 3 ++
.../cache/ClientCacheRemoveAllRequest.java | 3 ++
.../cache/ClientCacheRemoveIfEqualsRequest.java | 3 ++
.../cache/ClientCacheRemoveKeyRequest.java | 3 ++
.../cache/ClientCacheRemoveKeysRequest.java | 3 ++
.../ClientCacheReplaceIfEqualsRequest.java | 3 ++
.../client/cache/ClientCacheReplaceRequest.java | 3 ++
.../client/cache/ClientCacheRequest.java | 32 ++++++++++++++
.../cache/ClientCacheScanQueryRequest.java | 3 ++
.../cache/ClientCacheSqlFieldsQueryRequest.java | 1 +
.../cache/ClientCacheSqlQueryRequest.java | 1 +
.../plugin/security/AuthenticationContext.java | 40 +++++++++++++++++
.../plugin/security/SecurityPermission.java | 11 ++++-
.../ignite/spi/discovery/tcp/ServerImpl.java | 12 ++++-
44 files changed, 371 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 04eb425..662338c 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -835,6 +835,12 @@ public final class IgniteSystemProperties {
public static final String IGNITE_WAL_FSYNC_WITH_DEDICATED_WORKER = "IGNITE_WAL_FSYNC_WITH_DEDICATED_WORKER";
/**
+ * When set to {@code true}, on-heap cache cannot be enabled - see
+ * {@link CacheConfiguration#setOnheapCacheEnabled(boolean)}.
+ * Default is {@code false}.
+ */
+ public static final String IGNITE_DISABLE_ONHEAP_CACHE = "IGNITE_DISABLE_ONHEAP_CACHE";
+ /**
* When set to {@code false}, loaded pages implementation is switched to previous version of implementation,
* FullPageIdTable. {@code True} value enables 'Robin Hood hashing: backward shift deletion'.
* Default is {@code true}.
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java b/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
index dc39c7a..0c24db8 100644
--- a/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
+++ b/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
@@ -18,7 +18,7 @@
package org.apache.ignite.client;
/**
- * Indicates Ignite server the client is connected to closed the connection and no longer available.
+ * Indicates user name or password is invalid.
*/
public class ClientAuthenticationException extends ClientException {
/** Serial version uid. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/client/ClientAuthorizationException.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/client/ClientAuthorizationException.java b/modules/core/src/main/java/org/apache/ignite/client/ClientAuthorizationException.java
new file mode 100644
index 0000000..cacede6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/client/ClientAuthorizationException.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.client;
+
+/**
+ * Indicates user has no permission to perform operation.
+ */
+public class ClientAuthorizationException extends ClientException {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 0L;
+
+ /** Message. */
+ private static final String MSG = "User is not authorized to perform this operation";
+
+ /**
+ * Default constructor.
+ */
+ public ClientAuthorizationException() {
+ super(MSG);
+ }
+
+ /**
+ * Constructs a new exception with the specified cause and a detail
+ * message of <tt>(cause==null ? null : cause.toString())</tt>.
+ *
+ * @param cause the cause.
+ */
+ public ClientAuthorizationException(Throwable cause) {
+ super(MSG, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientChannel.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientChannel.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientChannel.java
index 71502a4..eb62c80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientChannel.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientChannel.java
@@ -22,6 +22,7 @@ import java.util.function.Function;
import org.apache.ignite.internal.binary.streams.BinaryInputStream;
import org.apache.ignite.internal.binary.streams.BinaryOutputStream;
import org.apache.ignite.client.ClientConnectionException;
+import org.apache.ignite.client.ClientAuthorizationException;
/**
* Processing thin client requests and responses.
@@ -41,5 +42,5 @@ interface ClientChannel extends AutoCloseable {
* @return Received operation payload or {@code null} if response has no payload.
*/
public <T> T receive(ClientOperation op, long reqId, Function<BinaryInputStream, T> payloadReader)
- throws ClientConnectionException;
+ throws ClientConnectionException, ClientAuthorizationException;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
index 404793a..8e8294f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
@@ -50,6 +50,7 @@ import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import org.apache.ignite.client.ClientAuthenticationException;
+import org.apache.ignite.client.ClientAuthorizationException;
import org.apache.ignite.client.ClientConnectionException;
import org.apache.ignite.client.SslMode;
import org.apache.ignite.client.SslProtocol;
@@ -62,6 +63,7 @@ import org.apache.ignite.internal.binary.streams.BinaryHeapOutputStream;
import org.apache.ignite.internal.binary.streams.BinaryInputStream;
import org.apache.ignite.internal.binary.streams.BinaryOffheapOutputStream;
import org.apache.ignite.internal.binary.streams.BinaryOutputStream;
+import org.apache.ignite.internal.processors.platform.client.ClientStatus;
/**
* Implements {@link ClientChannel} over TCP.
@@ -138,7 +140,8 @@ class TcpClientChannel implements ClientChannel {
/** {@inheritDoc} */
public <T> T receive(ClientOperation op, long reqId, Function<BinaryInputStream, T> payloadReader)
- throws ClientConnectionException {
+ throws ClientConnectionException, ClientAuthorizationException {
+
final int MIN_RES_SIZE = 8 + 4; // minimal response size: long (8 bytes) ID + int (4 bytes) status
int resSize = new BinaryHeapInputStream(read(4)).readInt();
@@ -163,7 +166,12 @@ class TcpClientChannel implements ClientChannel {
String err = new BinaryReaderExImpl(null, resIn, null, true).readString();
- throw new ClientServerError(err, status, reqId);
+ switch (status) {
+ case ClientStatus.SECURITY_VIOLATION:
+ throw new ClientAuthorizationException();
+ default:
+ throw new ClientServerError(err, status, reqId);
+ }
}
if (resSize <= MIN_RES_SIZE || payloadReader == null)
@@ -539,16 +547,10 @@ class TcpClientChannel implements ClientChannel {
/** */
private static KeyStore loadKeyStore(String lb, String path, String type, char[] pwd) {
- InputStream in = null;
+ KeyStore store;
try {
- KeyStore store = KeyStore.getInstance(type);
-
- in = new FileInputStream(new File(path));
-
- store.load(in, pwd);
-
- return store;
+ store = KeyStore.getInstance(type);
}
catch (KeyStoreException e) {
throw new ClientError(
@@ -556,6 +558,13 @@ class TcpClientChannel implements ClientChannel {
e
);
}
+
+ try (InputStream in = new FileInputStream(new File(path))) {
+
+ store.load(in, pwd);
+
+ return store;
+ }
catch (FileNotFoundException e) {
throw new ClientError(String.format("%s key store file [%s] does not exist", lb, path), e);
}
@@ -571,16 +580,6 @@ class TcpClientChannel implements ClientChannel {
catch (IOException e) {
throw new ClientError(String.format("Could not read %s key store", lb), e);
}
- finally {
- if (in != null) {
- try {
- in.close();
- }
- catch (IOException ignored) {
- // Fail silently
- }
- }
- }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
index c025e8c..93b3a5b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
@@ -897,7 +897,10 @@ public class IgniteAuthenticationProcessor extends GridProcessorAdapter implemen
// Can be empty on initial start of PDS cluster (default user will be created and stored after activate)
if (!F.isEmpty(initUsrs.usrs)) {
- users.clear();
+ if (users == null)
+ users = new ConcurrentHashMap<>();
+ else
+ users.clear();
for (User u : initUsrs.usrs)
users.put(u.name(), u);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 7edac73..3aa6603 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -149,6 +149,8 @@ import org.apache.ignite.marshaller.MarshallerUtils;
import org.apache.ignite.marshaller.jdk.JdkMarshaller;
import org.apache.ignite.mxbean.CacheGroupMetricsMXBean;
import org.apache.ignite.mxbean.IgniteMBeanAware;
+import org.apache.ignite.plugin.security.SecurityException;
+import org.apache.ignite.plugin.security.SecurityPermission;
import org.apache.ignite.spi.IgniteNodeValidationResult;
import org.apache.ignite.spi.discovery.DiscoveryDataBag;
import org.apache.ignite.spi.discovery.DiscoveryDataBag.GridDiscoveryData;
@@ -1126,6 +1128,9 @@ public class GridCacheProcessor extends GridProcessorAdapter {
CacheConfiguration cfg = cacheCtx.config();
+ if (cacheCtx.userCache())
+ authorizeCacheCreate(cacheCtx.name(), cfg);
+
// Intentionally compare Boolean references using '!=' below to check if the flag has been explicitly set.
if (cfg.isStoreKeepBinary() && cfg.isStoreKeepBinary() != CacheConfiguration.DFLT_STORE_KEEP_BINARY
&& !(ctx.config().getMarshaller() instanceof BinaryMarshaller))
@@ -3151,6 +3156,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
Collection<DynamicCacheChangeRequest> sndReqs = new ArrayList<>(reqs.size());
for (DynamicCacheChangeRequest req : reqs) {
+ authorizeCacheChange(req);
+
DynamicCacheStartFuture fut = new DynamicCacheStartFuture(req.requestId());
try {
@@ -3216,6 +3223,31 @@ public class GridCacheProcessor extends GridProcessorAdapter {
}
/**
+ * Authorize dynamic cache management.
+ */
+ private void authorizeCacheChange(DynamicCacheChangeRequest req) {
+ if (req.cacheType() == null || req.cacheType() == CacheType.USER) {
+ if (req.stop())
+ ctx.security().authorize(req.cacheName(), SecurityPermission.CACHE_DESTROY, null);
+ else
+ authorizeCacheCreate(req.cacheName(), req.startCacheConfiguration());
+ }
+ }
+
+ /**
+ * Authorize start/create cache operation.
+ */
+ private void authorizeCacheCreate(String cacheName, CacheConfiguration cacheCfg) {
+ ctx.security().authorize(cacheName, SecurityPermission.CACHE_CREATE, null);
+
+ if (cacheCfg != null && cacheCfg.isOnheapCacheEnabled() &&
+ System.getProperty(IgniteSystemProperties.IGNITE_DISABLE_ONHEAP_CACHE, "false")
+ .toUpperCase().equals("TRUE")
+ )
+ throw new SecurityException("Authorization failed for enabling on-heap cache.");
+ }
+
+ /**
* @return Non null exception if node is stopping or disconnected.
*/
@Nullable private IgniteCheckedException checkNodeState() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index d672420..e244c75 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -98,6 +98,8 @@ import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.lang.IgniteReducer;
import org.apache.ignite.lifecycle.LifecycleAware;
import org.apache.ignite.plugin.CachePluginConfiguration;
+import org.apache.ignite.plugin.security.SecurityException;
+import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
import org.apache.ignite.transactions.Transaction;
import org.apache.ignite.transactions.TransactionConcurrency;
import org.apache.ignite.transactions.TransactionIsolation;
@@ -1290,6 +1292,9 @@ public class GridCacheUtils {
if (e.getCause() instanceof NullPointerException)
return (NullPointerException)e.getCause();
+ if (e.getCause() instanceof SecurityException)
+ return (SecurityException)e.getCause();
+
C1<IgniteCheckedException, IgniteException> converter = U.getExceptionConverter(e.getClass());
return converter != null ? new CacheException(converter.apply(e)) : new CacheException(e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
index 7ab2d33..061aab3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
@@ -20,16 +20,24 @@ package org.apache.ignite.internal.processors.platform.client;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
+import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.processors.authentication.AuthorizationContext;
+import org.apache.ignite.internal.processors.authentication.IgniteAccessControlException;
import org.apache.ignite.internal.processors.odbc.ClientListenerConnectionContext;
import org.apache.ignite.internal.processors.odbc.ClientListenerMessageParser;
import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.processors.odbc.ClientListenerRequestHandler;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.plugin.security.AuthenticationContext;
+import org.apache.ignite.plugin.security.SecurityCredentials;
+
+import static org.apache.ignite.plugin.security.SecuritySubjectType.REMOTE_CLIENT;
/**
* Thin Client connection context.
@@ -62,6 +70,9 @@ public class ClientConnectionContext implements ClientListenerConnectionContext
/** Cursor counter. */
private final AtomicLong curCnt = new AtomicLong();
+ /** Security context or {@code null} if security is disabled. */
+ private SecurityContext secCtx = null;
+
/**
* Ctor.
*
@@ -129,7 +140,9 @@ public class ClientConnectionContext implements ClientListenerConnectionContext
}
}
- if (kernalCtx.authentication().enabled()) {
+ if (kernalCtx.security().enabled())
+ authCtx = thirdPartyAuthentication(user, pwd).authorizationContext();
+ else if (kernalCtx.authentication().enabled()) {
if (user == null || user.length() == 0)
throw new IgniteCheckedException("Unauthenticated sessions are prohibited.");
@@ -179,4 +192,34 @@ public class ClientConnectionContext implements ClientListenerConnectionContext
public void decrementCursors() {
curCnt.decrementAndGet();
}
+
+ /**
+ * @return Security context or {@code null} if security is disabled.
+ */
+ public SecurityContext securityContext() {
+ return secCtx;
+ }
+
+ /**
+ * Do 3-rd party authentication.
+ */
+ private AuthenticationContext thirdPartyAuthentication(String user, String pwd) throws IgniteCheckedException {
+ SecurityCredentials cred = new SecurityCredentials(user, pwd);
+
+ AuthenticationContext authCtx = new AuthenticationContext();
+
+ authCtx.subjectType(REMOTE_CLIENT);
+ authCtx.subjectId(UUID.randomUUID());
+ authCtx.nodeAttributes(Collections.emptyMap());
+ authCtx.credentials(cred);
+
+ secCtx = kernalCtx.security().authenticate(authCtx);
+
+ if (secCtx == null)
+ throw new IgniteAccessControlException(
+ String.format("The user name or password is incorrect [userName=%s]", user)
+ );
+
+ return authCtx;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java
index 76823b5..799b3e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java
@@ -19,6 +19,9 @@ package org.apache.ignite.internal.processors.platform.client;
import org.apache.ignite.binary.BinaryRawReader;
import org.apache.ignite.internal.processors.odbc.ClientListenerRequest;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.plugin.security.SecurityException;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Thin client request.
@@ -58,4 +61,30 @@ public class ClientRequest implements ClientListenerRequest {
public ClientResponse process(ClientConnectionContext ctx) {
return new ClientResponse(reqId);
}
+
+ /**
+ * Run the code with converting {@link SecurityException} to {@link IgniteClientException}.
+ */
+ protected static void runWithSecurityExceptionHandler(Runnable runnable) {
+ try {
+ runnable.run();
+ }
+ catch (SecurityException ex) {
+ throw new IgniteClientException(
+ ClientStatus.SECURITY_VIOLATION,
+ "Client is not authorized to perform this operation",
+ ex
+ );
+ }
+ }
+
+ /**
+ * Authorize for specified permission.
+ */
+ protected void authorize(ClientConnectionContext ctx, SecurityPermission perm) {
+ SecurityContext secCtx = ctx.securityContext();
+
+ if (secCtx != null)
+ runWithSecurityExceptionHandler(() -> ctx.kernalContext().security().authorize(null, perm, secCtx));
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
index e0049b4..b8dfb1f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
@@ -48,4 +48,7 @@ public final class ClientStatus {
/** Resource does not exist. */
public static final int RESOURCE_DOES_NOT_EXIST = 1011;
+
+ /** Resource does not exist. */
+ public static final int SECURITY_VIOLATION = 1012;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java
index 6bcbbe8..5f8e952 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.platform.client.cache;
import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Clear key request.
@@ -37,6 +38,8 @@ public class ClientCacheClearKeyRequest extends ClientCacheKeyRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_REMOVE);
+
cache(ctx).clear(key());
return super.process(ctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java
index 04eb7f6..d803f69 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.platform.client.cache;
import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Clear keys request.
@@ -37,6 +38,8 @@ public class ClientCacheClearKeysRequest extends ClientCacheKeysRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_REMOVE);
+
cache(ctx).clearAll(keys());
return super.process(ctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java
index 0e5f20d..7b84522 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.platform.client.cache;
import org.apache.ignite.binary.BinaryRawReader;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache clear request.
@@ -37,6 +38,8 @@ public class ClientCacheClearRequest extends ClientCacheRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_REMOVE);
+
cache(ctx).clear();
return super.process(ctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java
index 8470828..386f448 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* ContainsKey request.
@@ -38,6 +39,8 @@ public class ClientCacheContainsKeyRequest extends ClientCacheKeyRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ);
+
boolean val = cache(ctx).containsKey(key());
return new ClientBooleanResponse(requestId(), val);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java
index 41e1306..b5184bf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* ContainsKeys request.
@@ -38,6 +39,8 @@ public class ClientCacheContainsKeysRequest extends ClientCacheKeysRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ);
+
boolean val = cache(ctx).containsKeys(keys());
return new ClientBooleanResponse(requestId(), val);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java
index 4b4dcec..65f9784 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java
@@ -25,6 +25,7 @@ import org.apache.ignite.internal.processors.platform.client.ClientRequest;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
import org.apache.ignite.internal.processors.platform.client.ClientStatus;
import org.apache.ignite.internal.processors.platform.client.IgniteClientException;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache create with configuration request.
@@ -47,8 +48,11 @@ public class ClientCacheCreateWithConfigurationRequest extends ClientRequest {
/** {@inheritDoc} */
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_CREATE);
+
try {
- ctx.kernalContext().grid().createCache(cacheCfg);
+ // Use security exception handler since the code authorizes "enable on-heap cache" permission
+ runWithSecurityExceptionHandler(() -> ctx.kernalContext().grid().createCache(cacheCfg));
} catch (CacheExistsException e) {
throw new IgniteClientException(ClientStatus.CACHE_EXISTS, e.getMessage());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java
index 9155d76..cacf099 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java
@@ -24,6 +24,7 @@ import org.apache.ignite.internal.processors.platform.client.ClientRequest;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
import org.apache.ignite.internal.processors.platform.client.ClientStatus;
import org.apache.ignite.internal.processors.platform.client.IgniteClientException;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache create with name request.
@@ -45,6 +46,8 @@ public class ClientCacheCreateWithNameRequest extends ClientRequest {
/** {@inheritDoc} */
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_CREATE);
+
try {
ctx.kernalContext().grid().createCache(cacheName);
} catch (CacheExistsException e) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java
index 6645a03..b6f85ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.binary.BinaryRawReader;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientRequest;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache destroy request.
@@ -42,6 +43,8 @@ public class ClientCacheDestroyRequest extends ClientRequest {
/** {@inheritDoc} */
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_DESTROY);
+
String cacheName = ClientCacheRequest.cacheDescriptor(ctx, cacheId).cacheName();
ctx.kernalContext().grid().destroyCache(cacheName);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java
index 2b33af1..a07305c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java
@@ -22,6 +22,7 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionCon
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
import java.util.Map;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* GetAll request.
@@ -39,6 +40,8 @@ public class ClientCacheGetAllRequest extends ClientCacheKeysRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ);
+
Map val = cache(ctx).getAll(keys());
return new ClientCacheGetAllResponse(requestId(), val);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java
index 8360213..8713a21 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache get and put if absent request.
@@ -38,6 +39,8 @@ public class ClientCacheGetAndPutIfAbsentRequest extends ClientCacheKeyValueRequ
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT);
+
Object res = cache(ctx).getAndPutIfAbsent(key(), val());
return new ClientObjectResponse(requestId(), res);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java
index 7a540e8..dde5181 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache get and put request.
@@ -38,6 +39,8 @@ public class ClientCacheGetAndPutRequest extends ClientCacheKeyValueRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT);
+
Object res = cache(ctx).getAndPut(key(), val());
return new ClientObjectResponse(requestId(), res);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java
index e4fd735..3b9dd4b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache get and remove request.
@@ -38,6 +39,8 @@ public class ClientCacheGetAndRemoveRequest extends ClientCacheKeyRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_REMOVE);
+
Object val = cache(ctx).getAndRemove(key());
return new ClientObjectResponse(requestId(), val);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java
index dba8639..8ba157a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache get and replace request.
@@ -38,6 +39,8 @@ public class ClientCacheGetAndReplaceRequest extends ClientCacheKeyValueRequest
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT);
+
Object res = cache(ctx).getAndReplace(key(), val());
return new ClientObjectResponse(requestId(), res);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java
index 267318a..48569b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java
@@ -25,6 +25,7 @@ import org.apache.ignite.internal.processors.platform.client.ClientRequest;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
import org.apache.ignite.internal.processors.platform.client.ClientStatus;
import org.apache.ignite.internal.processors.platform.client.IgniteClientException;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache get or create with configuration request.
@@ -47,8 +48,11 @@ public class ClientCacheGetOrCreateWithConfigurationRequest extends ClientReques
/** {@inheritDoc} */
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_CREATE);
+
try {
- ctx.kernalContext().grid().getOrCreateCache(cacheCfg);
+ // Use security exception handler since the code authorizes "enable on-heap cache" permission
+ runWithSecurityExceptionHandler(() -> ctx.kernalContext().grid().getOrCreateCache(cacheCfg));
} catch (CacheExistsException e) {
throw new IgniteClientException(ClientStatus.CACHE_EXISTS, e.getMessage());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java
index 94dd115..3c4ce7b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.binary.BinaryRawReader;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientRequest;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache create with name request.
@@ -42,6 +43,8 @@ public class ClientCacheGetOrCreateWithNameRequest extends ClientRequest {
/** {@inheritDoc} */
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_CREATE);
+
ctx.kernalContext().grid().getOrCreateCache(cacheName);
return super.process(ctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java
index 41558c2..dc17cbf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache get request.
@@ -38,6 +39,8 @@ public class ClientCacheGetRequest extends ClientCacheKeyRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ);
+
Object val = cache(ctx).get(key());
return new ClientObjectResponse(requestId(), val);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java
index ba185bf..474c206 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java
@@ -22,6 +22,7 @@ import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientLongResponse;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache size request.
@@ -50,6 +51,8 @@ public class ClientCacheGetSizeRequest extends ClientCacheRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ);
+
long res = cache(ctx).sizeLong(modes);
return new ClientLongResponse(requestId(), res);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java
index 28a7fa5..57e3144 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java
@@ -23,6 +23,7 @@ import org.apache.ignite.internal.processors.platform.client.ClientResponse;
import java.util.LinkedHashMap;
import java.util.Map;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* PutAll request.
@@ -50,6 +51,8 @@ public class ClientCachePutAllRequest extends ClientCacheRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_PUT);
+
cache(ctx).putAll(map);
return super.process(ctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java
index 4dd2cde..ec81bc0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache put if absent request.
@@ -38,6 +39,8 @@ public class ClientCachePutIfAbsentRequest extends ClientCacheKeyValueRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT);
+
boolean res = cache(ctx).putIfAbsent(key(), val());
return new ClientBooleanResponse(requestId(), res);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java
index 2c396b7..116460e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.platform.client.cache;
import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache put request.
@@ -37,6 +38,8 @@ public class ClientCachePutRequest extends ClientCacheKeyValueRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_PUT);
+
cache(ctx).put(key(), val());
return super.process(ctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java
index f5adc63..d90d873 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.platform.client.cache;
import org.apache.ignite.binary.BinaryRawReader;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache removeAll request.
@@ -37,6 +38,8 @@ public class ClientCacheRemoveAllRequest extends ClientCacheRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_REMOVE);
+
cache(ctx).removeAll();
return super.process(ctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java
index b86f2f8..26c191f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache remove request with value.
@@ -38,6 +39,8 @@ public class ClientCacheRemoveIfEqualsRequest extends ClientCacheKeyValueRequest
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_REMOVE);
+
boolean res = cache(ctx).remove(key(), val());
return new ClientBooleanResponse(requestId(), res);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java
index a68c327..5af9743 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Remove request.
@@ -38,6 +39,8 @@ public class ClientCacheRemoveKeyRequest extends ClientCacheKeyRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_REMOVE);
+
boolean val = cache(ctx).remove(key());
return new ClientBooleanResponse(requestId(), val);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java
index 043b568..62dea00 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.platform.client.cache;
import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Remove keys request.
@@ -37,6 +38,8 @@ public class ClientCacheRemoveKeysRequest extends ClientCacheKeysRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_REMOVE);
+
cache(ctx).removeAll(keys());
return super.process(ctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java
index 8645fbb..056367d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache replace request.
@@ -43,6 +44,8 @@ public class ClientCacheReplaceIfEqualsRequest extends ClientCacheKeyValueReques
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT);
+
boolean res = cache(ctx).replace(key(), val(), newVal);
return new ClientBooleanResponse(requestId(), res);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java
index bd7a642..ea04593 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx;
import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache replace request.
@@ -38,6 +39,8 @@ public class ClientCacheReplaceRequest extends ClientCacheKeyValueRequest {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT);
+
boolean res = cache(ctx).replace(key(), val());
return new ClientBooleanResponse(requestId(), res);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java
index 52b799f..9e2d1f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java
@@ -24,6 +24,8 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionCon
import org.apache.ignite.internal.processors.platform.client.ClientRequest;
import org.apache.ignite.internal.processors.platform.client.ClientStatus;
import org.apache.ignite.internal.processors.platform.client.IgniteClientException;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Cache get request.
@@ -119,4 +121,34 @@ class ClientCacheRequest extends ClientRequest {
protected int cacheId() {
return cacheId;
}
+
+ /** {@inheritDoc} */
+ protected void authorize(ClientConnectionContext ctx, SecurityPermission perm) {
+ SecurityContext secCtx = ctx.securityContext();
+
+ if (secCtx != null) {
+ DynamicCacheDescriptor cacheDesc = cacheDescriptor(ctx, cacheId);
+
+ runWithSecurityExceptionHandler(() -> {
+ ctx.kernalContext().security().authorize(cacheDesc.cacheName(), perm, secCtx);
+ });
+ }
+ }
+
+ /**
+ * Authorize for multiple permissions.
+ */
+ protected void authorize(ClientConnectionContext ctx, SecurityPermission... perm)
+ throws IgniteClientException {
+ SecurityContext secCtx = ctx.securityContext();
+
+ if (secCtx != null) {
+ DynamicCacheDescriptor cacheDesc = cacheDescriptor(ctx, cacheId);
+
+ runWithSecurityExceptionHandler(() -> {
+ for (SecurityPermission p : perm)
+ ctx.kernalContext().security().authorize(cacheDesc.cacheName(), p, secCtx);
+ });
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java
index 26ab236..70b6966 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java
@@ -28,6 +28,7 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionCon
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
import org.apache.ignite.internal.processors.platform.utils.PlatformUtils;
import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Scan query request.
@@ -80,6 +81,8 @@ public class ClientCacheScanQueryRequest extends ClientCacheRequest {
/** {@inheritDoc} */
@Override public ClientResponse process(ClientConnectionContext ctx) {
+ authorize(ctx, SecurityPermission.CACHE_READ);
+
IgniteCache cache = filterPlatform == FILTER_PLATFORM_JAVA && !isKeepBinary() ? rawCache(ctx) : cache(ctx);
ScanQuery qry = new ScanQuery()
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
index cfd4498..3aa95bf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
@@ -29,6 +29,7 @@ import org.apache.ignite.internal.processors.platform.cache.PlatformCache;
import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Sql query request.
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java
index 8c21be1..40693e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java
@@ -26,6 +26,7 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionCon
import org.apache.ignite.internal.processors.platform.client.ClientResponse;
import java.util.concurrent.TimeUnit;
+import org.apache.ignite.plugin.security.SecurityPermission;
/**
* Sql query request.
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java
index 91f3379..9f2cfe2 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java
@@ -21,6 +21,7 @@ import java.net.InetSocketAddress;
import java.util.Collections;
import java.util.Map;
import java.util.UUID;
+import org.apache.ignite.internal.processors.authentication.AuthorizationContext;
/**
* Authentication context.
@@ -41,6 +42,12 @@ public class AuthenticationContext {
/** */
private Map<String, Object> nodeAttrs;
+ /** Authorization context. */
+ private AuthorizationContext athrCtx;
+
+ /** True if this is a client node context. */
+ private boolean client;
+
/**
* Gets subject type.
*
@@ -130,4 +137,37 @@ public class AuthenticationContext {
public void nodeAttributes(Map<String, Object> nodeAttrs) {
this.nodeAttrs = nodeAttrs;
}
+
+ /**
+ * @return Native Apache Ignite authorization context acquired after authentication or {@code null} if native
+ * Ignite authentication is not used.
+ */
+ public AuthorizationContext authorizationContext(){
+ return athrCtx;
+ }
+
+ /**
+ * Set authorization context acquired after native Apache Ignite authentication.
+ */
+ public AuthenticationContext authorizationContext(AuthorizationContext newVal) {
+ athrCtx = newVal;
+
+ return this;
+ }
+
+ /**
+ * @return {@code true} if this is a client node context.
+ */
+ public boolean isClient() {
+ return client;
+ }
+
+ /**
+ * Sets flag indicating if this is client node context.
+ */
+ public AuthenticationContext setClient(boolean newVal) {
+ client = newVal;
+
+ return this;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
index 5436161..bca667d 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
@@ -64,7 +64,16 @@ public enum SecurityPermission {
SERVICE_CANCEL,
/** Service invoke permission. */
- SERVICE_INVOKE;
+ SERVICE_INVOKE,
+
+ /** Cache create permission. */
+ CACHE_CREATE,
+
+ /** Cache create permission. */
+ CACHE_DESTROY,
+
+ /** Join as server node permission. */
+ JOIN_AS_SERVER;
/** Enumerated values. */
private static final SecurityPermission[] VALS = values();
http://git-wip-us.apache.org/repos/asf/ignite/blob/5a292763/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 7bf37e1..6d3864e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -99,6 +99,7 @@ import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgniteProductVersion;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.plugin.security.SecurityCredentials;
+import org.apache.ignite.plugin.security.SecurityPermission;
import org.apache.ignite.plugin.security.SecurityPermissionSet;
import org.apache.ignite.spi.IgniteNodeValidationResult;
import org.apache.ignite.spi.IgniteSpiContext;
@@ -3559,6 +3560,8 @@ class ServerImpl extends TcpDiscoveryImpl {
return;
}
else {
+ String authFailedMsg = null;
+
if (!(subj instanceof Serializable)) {
// Node has not pass authentication.
LT.warn(log, "Authentication subject is not Serializable [nodeId=" + node.id() +
@@ -3567,9 +3570,16 @@ class ServerImpl extends TcpDiscoveryImpl {
", addrs=" +
U.addressesAsString(node) + ']');
+ authFailedMsg = "Authentication subject is not serializable";
+ }
+ else if (!node.isClient() &&
+ !subj.systemOperationAllowed(SecurityPermission.JOIN_AS_SERVER))
+ authFailedMsg = "Node is not authorised to join as a server node";
+
+ if (authFailedMsg != null) {
// Always output in debug.
if (log.isDebugEnabled())
- log.debug("Authentication subject is not serializable [nodeId=" + node.id() +
+ log.debug(authFailedMsg + " [nodeId=" + node.id() +
", addrs=" + U.addressesAsString(node));
try {
[26/54] [abbrv] ignite git commit: IGNITE-7871 Implemented additional
synchronization phase for correct partition counters update
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java
new file mode 100644
index 0000000..bad1b61
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ * Message is used to send acks for {@link Latch} instances management.
+ */
+public class LatchAckMessage implements Message {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Latch id. */
+ private String latchId;
+
+ /** Latch topology version. */
+ private AffinityTopologyVersion topVer;
+
+ /** Flag indicates that ack is final. */
+ private boolean isFinal;
+
+ /**
+ * Constructor.
+ *
+ * @param latchId Latch id.
+ * @param topVer Latch topology version.
+ * @param isFinal Final acknowledgement flag.
+ */
+ public LatchAckMessage(String latchId, AffinityTopologyVersion topVer, boolean isFinal) {
+ this.latchId = latchId;
+ this.topVer = topVer;
+ this.isFinal = isFinal;
+ }
+
+ /**
+ * Empty constructor for marshalling purposes.
+ */
+ public LatchAckMessage() {
+ }
+
+ /**
+ * @return Latch id.
+ */
+ public String latchId() {
+ return latchId;
+ }
+
+ /**
+ * @return Latch topology version.
+ */
+ public AffinityTopologyVersion topVer() {
+ return topVer;
+ }
+
+ /**
+ * @return {@code} if ack is final.
+ */
+ public boolean isFinal() {
+ return isFinal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeBoolean("isFinal", isFinal))
+ return false;
+
+ writer.incrementState();
+
+ case 1:
+ if (!writer.writeString("latchId", latchId))
+ return false;
+
+ writer.incrementState();
+
+ case 2:
+ if (!writer.writeMessage("topVer", topVer))
+ return false;
+
+ writer.incrementState();
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ isFinal = reader.readBoolean("isFinal");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 1:
+ latchId = reader.readString("latchId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 2:
+ topVer = reader.readMessage("topVer");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+ }
+
+ return reader.afterMessageRead(LatchAckMessage.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return 135;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 3;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index 7785605..33f84f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -3525,6 +3525,11 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
U.error(log, "Failed to prepare transaction: " + this, e);
}
+ catch (Throwable t) {
+ fut.onDone(t);
+
+ throw t;
+ }
if (err != null)
fut.rollbackOnError(err);
@@ -3544,6 +3549,11 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
U.error(log, "Failed to prepare transaction: " + this, e);
}
+ catch (Throwable t) {
+ fut.onDone(t);
+
+ throw t;
+ }
if (err != null)
fut.rollbackOnError(err);
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 5cfd92d..68ec83d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -189,7 +189,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
freeList.saveMetadata();
long updCntr = store.updateCounter();
- int size = store.fullSize();
+ long size = store.fullSize();
long rmvId = globalRemoveId().get();
PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
@@ -318,7 +318,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
partMetaId,
updCntr,
rmvId,
- size,
+ (int)size, // TODO: Partition size may be long
cntrsPageId,
state == null ? -1 : (byte)state.ordinal(),
pageCnt
@@ -549,7 +549,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
final int grpId,
final int partId,
final int currAllocatedPageCnt,
- final int partSize
+ final long partSize
) {
if (part != null) {
boolean reserved = part.reserve();
@@ -1301,7 +1301,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
}
/** {@inheritDoc} */
- @Override public int fullSize() {
+ @Override public long fullSize() {
try {
CacheDataStore delegate0 = init0(true);
@@ -1313,7 +1313,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
}
/** {@inheritDoc} */
- @Override public int cacheSize(int cacheId) {
+ @Override public long cacheSize(int cacheId) {
try {
CacheDataStore delegate0 = init0(true);
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 9bfaaf3..945ef48 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -490,7 +490,7 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
@Override public AffinityTopologyVersion topologyVersion() {
AffinityTopologyVersion res = topVer;
- if (res.equals(AffinityTopologyVersion.NONE)) {
+ if (res == null || res.equals(AffinityTopologyVersion.NONE)) {
if (system()) {
AffinityTopologyVersion topVer = cctx.tm().lockedTopologyVersion(Thread.currentThread().getId(), this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index fbdeca1..9fb8777 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -545,10 +545,10 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
* @param topVer Topology version.
* @return Future that will be completed when all ongoing transactions are finished.
*/
- public IgniteInternalFuture<Boolean> finishTxs(AffinityTopologyVersion topVer) {
+ public IgniteInternalFuture<Boolean> finishLocalTxs(AffinityTopologyVersion topVer) {
GridCompoundFuture<IgniteInternalTx, Boolean> res =
new CacheObjectsReleaseFuture<>(
- "Tx",
+ "LocalTx",
topVer,
new IgniteReducer<IgniteInternalTx, Boolean>() {
@Override public boolean collect(IgniteInternalTx e) {
@@ -561,8 +561,9 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
});
for (IgniteInternalTx tx : txs()) {
- if (needWaitTransaction(tx, topVer))
+ if (needWaitTransaction(tx, topVer)) {
res.add(tx.finishFuture());
+ }
}
res.markInitialized();
@@ -571,6 +572,29 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
}
/**
+ * Creates a future that will wait for finishing all tx updates on backups after all local transactions are finished.
+ *
+ * NOTE:
+ * As we send finish request to backup nodes after transaction successfully completed on primary node
+ * it's important to ensure that all updates from primary to backup are finished or at least remote transaction has created on backup node.
+ *
+ * @param finishLocalTxsFuture Local transactions finish future.
+ * @param topVer Topology version.
+ * @return Future that will be completed when all ongoing transactions are finished.
+ */
+ public IgniteInternalFuture<?> finishAllTxs(IgniteInternalFuture<?> finishLocalTxsFuture, AffinityTopologyVersion topVer) {
+ final GridCompoundFuture finishAllTxsFuture = new CacheObjectsReleaseFuture("AllTx", topVer);
+
+ // After finishing all local updates, wait for finishing all tx updates on backups.
+ finishLocalTxsFuture.listen(future -> {
+ finishAllTxsFuture.add(cctx.mvcc().finishRemoteTxs(topVer));
+ finishAllTxsFuture.markInitialized();
+ });
+
+ return finishAllTxsFuture;
+ }
+
+ /**
* @param tx Transaction.
* @param topVer Exchange version.
* @return {@code True} if need wait transaction for exchange.
@@ -1834,12 +1858,12 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
* @return Finish future for related remote transactions.
*/
@SuppressWarnings("unchecked")
- public IgniteInternalFuture<?> remoteTxFinishFuture(GridCacheVersion nearVer) {
- GridCompoundFuture<Void, Void> fut = new GridCompoundFuture<>();
+ public IgniteInternalFuture<IgniteInternalTx> remoteTxFinishFuture(GridCacheVersion nearVer) {
+ GridCompoundFuture<IgniteInternalTx, IgniteInternalTx> fut = new GridCompoundFuture<>();
for (final IgniteInternalTx tx : txs()) {
if (!tx.local() && nearVer.equals(tx.nearXidVersion()))
- fut.add((IgniteInternalFuture) tx.finishFuture());
+ fut.add(tx.finishFuture());
}
fut.markInitialized();
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java
index 7263656..702b188 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java
@@ -76,7 +76,7 @@ public class CacheDhtLocalPartitionAfterRemoveSelfTest extends GridCommonAbstrac
cache = grid(g).cache(DEFAULT_CACHE_NAME);
for (GridDhtLocalPartition p : dht(cache).topology().localPartitions()) {
- int size = p.dataStore().fullSize();
+ long size = p.dataStore().fullSize();
assertTrue("Unexpected size: " + size, size <= 32);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
index 468bbc8..6c570d7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
@@ -86,6 +86,7 @@ import org.apache.ignite.internal.util.lang.GridIterator;
import org.apache.ignite.internal.util.lang.GridPlainCallable;
import org.apache.ignite.internal.util.lang.gridfunc.ContainsPredicate;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.PA;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.CU;
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteExchangeLatchManagerCoordinatorFailTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteExchangeLatchManagerCoordinatorFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteExchangeLatchManagerCoordinatorFailTest.java
new file mode 100644
index 0000000..52cd033
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteExchangeLatchManagerCoordinatorFailTest.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.datastructures;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.collect.Lists;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
+import org.apache.ignite.lang.IgniteBiClosure;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ * Tests for {@link ExchangeLatchManager} functionality when latch coordinator is failed.
+ */
+public class IgniteExchangeLatchManagerCoordinatorFailTest extends GridCommonAbstractTest {
+ /** */
+ private static final String LATCH_NAME = "test";
+
+ /** 5 nodes. */
+ private final AffinityTopologyVersion latchTopVer = new AffinityTopologyVersion(5, 0);
+
+ /** Wait before latch creation. */
+ private final IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean> beforeCreate = (mgr, syncLatch) -> {
+ try {
+ syncLatch.countDown();
+ syncLatch.await();
+
+ Latch distributedLatch = mgr.getOrCreate(LATCH_NAME, latchTopVer);
+
+ distributedLatch.countDown();
+
+ distributedLatch.await();
+ } catch (Exception e) {
+ log.error("Unexpected exception", e);
+
+ return false;
+ }
+
+ return true;
+ };
+
+ /** Wait before latch count down. */
+ private final IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean> beforeCountDown = (mgr, syncLatch) -> {
+ try {
+ Latch distributedLatch = mgr.getOrCreate(LATCH_NAME, latchTopVer);
+
+ syncLatch.countDown();
+ syncLatch.await();
+
+ distributedLatch.countDown();
+
+ distributedLatch.await();
+ } catch (Exception e) {
+ log.error("Unexpected exception ", e);
+
+ return false;
+ }
+
+ return true;
+ };
+
+ /** Wait after all operations are successful. */
+ private final IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean> all = (mgr, syncLatch) -> {
+ try {
+ Latch distributedLatch = mgr.getOrCreate(LATCH_NAME, latchTopVer);
+
+ distributedLatch.countDown();
+
+ syncLatch.countDown();
+
+ distributedLatch.await();
+
+ syncLatch.await();
+ } catch (Exception e) {
+ log.error("Unexpected exception ", e);
+
+ return false;
+ }
+
+ return true;
+ };
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ * Test scenarios description:
+ *
+ * We have existing coordinator and 4 other nodes.
+ * Each node do following operations:
+ * 1) Create latch
+ * 2) Countdown latch
+ * 3) Await latch
+ *
+ * While nodes do the operations we shutdown coordinator and next oldest node become new coordinator.
+ * We should check that new coordinator properly restored latch and all nodes finished latch completion successfully after that.
+ *
+ * Each node before coordinator shutdown can be in 3 different states:
+ *
+ * State {@link #beforeCreate} - Node didn't create latch yet.
+ * State {@link #beforeCountDown} - Node created latch but didn't count down it yet.
+ * State {@link #all} - Node created latch and count downed it.
+ *
+ * We should check important cases when future coordinator is in one of these states, and other 3 nodes have 3 different states.
+ */
+
+ /**
+ * Scenario 1:
+ *
+ * Node 1 state -> {@link #beforeCreate}
+ * Node 2 state -> {@link #beforeCountDown}
+ * Node 3 state -> {@link #all}
+ * Node 4 state -> {@link #beforeCreate}
+ */
+ public void testCoordinatorFail1() throws Exception {
+ List<IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean>> nodeStates = Lists.newArrayList(
+ beforeCreate,
+ beforeCountDown,
+ all,
+ beforeCreate
+ );
+
+ doTestCoordinatorFail(nodeStates);
+ }
+
+ /**
+ * Scenario 2:
+ *
+ * Node 1 state -> {@link #beforeCountDown}
+ * Node 2 state -> {@link #beforeCountDown}
+ * Node 3 state -> {@link #all}
+ * Node 4 state -> {@link #beforeCreate}
+ */
+ public void testCoordinatorFail2() throws Exception {
+ List<IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean>> nodeStates = Lists.newArrayList(
+ beforeCountDown,
+ beforeCountDown,
+ all,
+ beforeCreate
+ );
+
+ doTestCoordinatorFail(nodeStates);
+ }
+
+ /**
+ * Scenario 3:
+ *
+ * Node 1 state -> {@link #all}
+ * Node 2 state -> {@link #beforeCountDown}
+ * Node 3 state -> {@link #all}
+ * Node 4 state -> {@link #beforeCreate}
+ */
+ public void testCoordinatorFail3() throws Exception {
+ List<IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean>> nodeStates = Lists.newArrayList(
+ all,
+ beforeCountDown,
+ all,
+ beforeCreate
+ );
+
+ doTestCoordinatorFail(nodeStates);
+ }
+
+ /**
+ * Test latch coordinator fail with specified scenarios.
+ *
+ * @param nodeScenarios Node scenarios.
+ * @throws Exception If failed.
+ */
+ private void doTestCoordinatorFail(List<IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean>> nodeScenarios) throws Exception {
+ IgniteEx crd = (IgniteEx) startGridsMultiThreaded(5);
+ crd.cluster().active(true);
+
+ // Latch to synchronize node states.
+ CountDownLatch syncLatch = new CountDownLatch(5);
+
+ GridCompoundFuture finishAllLatches = new GridCompoundFuture();
+
+ AtomicBoolean hasErrors = new AtomicBoolean();
+
+ for (int node = 1; node < 5; node++) {
+ IgniteEx grid = grid(node);
+ ExchangeLatchManager latchMgr = grid.context().cache().context().exchange().latch();
+ final int stateIdx = node - 1;
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(() -> {
+ boolean success = nodeScenarios.get(stateIdx).apply(latchMgr, syncLatch);
+ if (!success)
+ hasErrors.set(true);
+ }, 1, "latch-runner-" + node);
+
+ finishAllLatches.add(fut);
+ }
+
+ finishAllLatches.markInitialized();
+
+ // Wait while all nodes reaches their states.
+ while (syncLatch.getCount() != 1) {
+ Thread.sleep(10);
+
+ if (hasErrors.get())
+ throw new Exception("All nodes should complete latches without errors");
+ }
+
+ crd.close();
+
+ // Resume progress for all nodes.
+ syncLatch.countDown();
+
+ // Wait for distributed latch completion.
+ finishAllLatches.get(5000);
+
+ Assert.assertFalse("All nodes should complete latches without errors", hasErrors.get());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
new file mode 100644
index 0000000..63d772a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
@@ -0,0 +1,316 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+/**
+ *
+ */
+public class GridCachePartitionsStateValidationTest extends GridCommonAbstractTest {
+ /** Cache name. */
+ private static final String CACHE_NAME = "cache";
+
+ /** */
+ private boolean clientMode;
+
+ /** {@inheritDoc */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setConsistentId(igniteInstanceName);
+
+ cfg.setCacheConfiguration(new CacheConfiguration(CACHE_NAME)
+ .setBackups(1)
+ .setAffinity(new RendezvousAffinityFunction(false, 32))
+ );
+
+ cfg.setCommunicationSpi(new SingleMessageInterceptorCommunicationSpi(2));
+
+ if (clientMode)
+ cfg.setClientMode(true);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /** {@inheritDoc */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ clientMode = false;
+ }
+
+ /**
+ * Test that partitions state validation works correctly.
+ *
+ * @throws Exception If failed.
+ */
+ public void testValidationIfPartitionCountersAreInconsistent() throws Exception {
+ IgniteEx ignite = (IgniteEx) startGrids(2);
+ ignite.cluster().active(true);
+
+ awaitPartitionMapExchange();
+
+ // Modify update counter for some partition.
+ for (GridDhtLocalPartition partition : ignite.cachex(CACHE_NAME).context().topology().localPartitions()) {
+ partition.updateCounter(100500L);
+ break;
+ }
+
+ // Trigger exchange.
+ startGrid(2);
+
+ awaitPartitionMapExchange();
+
+ // Nothing should happen (just log error message) and we're still able to put data to corrupted cache.
+ ignite.cache(CACHE_NAME).put(0, 0);
+
+ stopAllGrids();
+ }
+
+ /**
+ * Test that all nodes send correct {@link GridDhtPartitionsSingleMessage} with consistent update counters.
+ *
+ * @throws Exception If failed.
+ */
+ public void testPartitionCountersConsistencyOnExchange() throws Exception {
+ IgniteEx ignite = (IgniteEx) startGrids(4);
+ ignite.cluster().active(true);
+
+ awaitPartitionMapExchange();
+
+ final String atomicCacheName = "atomic-cache";
+ final String txCacheName = "tx-cache";
+
+ clientMode = true;
+
+ Ignite client = startGrid(4);
+
+ clientMode = false;
+
+ IgniteCache atomicCache = client.getOrCreateCache(new CacheConfiguration<>(atomicCacheName)
+ .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+ .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+ .setBackups(2)
+ .setAffinity(new RendezvousAffinityFunction(false, 32))
+ );
+
+ IgniteCache txCache = client.getOrCreateCache(new CacheConfiguration<>(txCacheName)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+ .setBackups(2)
+ .setAffinity(new RendezvousAffinityFunction(false, 32))
+ );
+
+ for (int it = 0; it < 10; it++) {
+ SingleMessageInterceptorCommunicationSpi spi = (SingleMessageInterceptorCommunicationSpi) ignite.configuration().getCommunicationSpi();
+ spi.clear();
+
+ // Stop load future.
+ final AtomicBoolean stop = new AtomicBoolean();
+
+ // Run atomic load.
+ IgniteInternalFuture atomicLoadFuture = GridTestUtils.runMultiThreadedAsync(() -> {
+ int k = 0;
+
+ while (!stop.get()) {
+ k++;
+ try {
+ atomicCache.put(k, k);
+ } catch (Exception ignored) {}
+ }
+ }, 1, "atomic-load");
+
+ // Run tx load.
+ IgniteInternalFuture txLoadFuture = GridTestUtils.runMultiThreadedAsync(() -> {
+ final int txOps = 5;
+
+ while (!stop.get()) {
+ List<Integer> randomKeys = Stream.generate(() -> ThreadLocalRandom.current().nextInt(5))
+ .limit(txOps)
+ .sorted()
+ .collect(Collectors.toList());
+
+ try (Transaction tx = ignite.transactions().txStart(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED)) {
+ for (Integer key : randomKeys)
+ txCache.put(key, key);
+
+ tx.commit();
+ }
+ catch (Exception ignored) { }
+ }
+ }, 4, "tx-load");
+
+ // Wait for some data.
+ Thread.sleep(1000);
+
+ // Prevent sending full message.
+ spi.blockFullMessage();
+
+ // Trigger exchange.
+ IgniteInternalFuture nodeStopFuture = GridTestUtils.runAsync(() -> stopGrid(3));
+
+ try {
+ spi.waitUntilAllSingleMessagesAreSent();
+
+ List<GridDhtPartitionsSingleMessage> interceptedMessages = spi.getMessages();
+
+ // Associate each message with existing node UUID.
+ Map<UUID, GridDhtPartitionsSingleMessage> messagesMap = new HashMap<>();
+ for (int i = 0; i < interceptedMessages.size(); i++)
+ messagesMap.put(grid(i + 1).context().localNodeId(), interceptedMessages.get(i));
+
+ GridDhtPartitionsStateValidator validator = new GridDhtPartitionsStateValidator(ignite.context().cache().context());
+
+ // Validate partition update counters. If counters are not consistent, exception will be thrown.
+ validator.validatePartitionsUpdateCounters(ignite.cachex(atomicCacheName).context().topology(), messagesMap, Collections.emptySet());
+ validator.validatePartitionsUpdateCounters(ignite.cachex(txCacheName).context().topology(), messagesMap, Collections.emptySet());
+
+ } finally {
+ // Stop load and resume exchange.
+ spi.unblockFullMessage();
+
+ stop.set(true);
+
+ atomicLoadFuture.get();
+ txLoadFuture.get();
+ nodeStopFuture.get();
+ }
+
+ // Return grid to initial state.
+ startGrid(3);
+
+ awaitPartitionMapExchange();
+ }
+ }
+
+ /**
+ * SPI which intercepts single messages during exchange.
+ */
+ private static class SingleMessageInterceptorCommunicationSpi extends TcpCommunicationSpi {
+ /** */
+ private static final List<GridDhtPartitionsSingleMessage> messages = new CopyOnWriteArrayList<>();
+
+ /** Future completes when {@link #singleMessagesThreshold} messages are sent to coordinator. */
+ private static final GridFutureAdapter allSingleMessagesSent = new GridFutureAdapter();
+
+ /** A number of single messages we're waiting for send. */
+ private final int singleMessagesThreshold;
+
+ /** Latch which blocks full message sending. */
+ private volatile CountDownLatch blockFullMsgLatch;
+
+ /**
+ * Constructor.
+ */
+ private SingleMessageInterceptorCommunicationSpi(int singleMessagesThreshold) {
+ this.singleMessagesThreshold = singleMessagesThreshold;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+ if (((GridIoMessage) msg).message() instanceof GridDhtPartitionsSingleMessage) {
+ GridDhtPartitionsSingleMessage singleMsg = (GridDhtPartitionsSingleMessage) ((GridIoMessage) msg).message();
+
+ // We're interesting for only exchange messages and when node is stopped.
+ if (singleMsg.exchangeId() != null && singleMsg.exchangeId().isLeft() && !singleMsg.client()) {
+ messages.add(singleMsg);
+
+ if (messages.size() == singleMessagesThreshold)
+ allSingleMessagesSent.onDone();
+ }
+ }
+
+ try {
+ if (((GridIoMessage) msg).message() instanceof GridDhtPartitionsFullMessage) {
+ if (blockFullMsgLatch != null)
+ blockFullMsgLatch.await();
+ }
+ }
+ catch (Exception ignored) { }
+
+ super.sendMessage(node, msg, ackC);
+ }
+
+ /** */
+ public void clear() {
+ messages.clear();
+ allSingleMessagesSent.reset();
+ }
+
+ /** */
+ public List<GridDhtPartitionsSingleMessage> getMessages() {
+ return Collections.unmodifiableList(messages);
+ }
+
+ /** */
+ public void blockFullMessage() {
+ blockFullMsgLatch = new CountDownLatch(1);
+ }
+
+ /** */
+ public void unblockFullMessage() {
+ blockFullMsgLatch.countDown();
+ }
+
+ /** */
+ public void waitUntilAllSingleMessagesAreSent() throws IgniteCheckedException {
+ allSingleMessagesSent.get();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
new file mode 100644
index 0000000..9ed8d54
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+
+/**
+ * Test correct behaviour of {@link GridDhtPartitionsStateValidator} class.
+ */
+public class GridCachePartitionsStateValidatorSelfTest extends GridCommonAbstractTest {
+ /** Mocks and stubs. */
+ private final UUID localNodeId = UUID.randomUUID();
+ /** */
+ private GridCacheSharedContext cctxMock;
+ /** */
+ private GridDhtPartitionTopology topologyMock;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ // Prepare mocks.
+ cctxMock = Mockito.mock(GridCacheSharedContext.class);
+ Mockito.when(cctxMock.localNodeId()).thenReturn(localNodeId);
+
+ topologyMock = Mockito.mock(GridDhtPartitionTopology.class);
+ Mockito.when(topologyMock.partitionState(Matchers.any(), Matchers.anyInt())).thenReturn(GridDhtPartitionState.OWNING);
+ Mockito.when(topologyMock.groupId()).thenReturn(0);
+ Mockito.when(topologyMock.partitions()).thenReturn(3);
+
+ List<GridDhtLocalPartition> localPartitions = Lists.newArrayList(
+ partitionMock(0, 1, 1),
+ partitionMock(1, 2, 2),
+ partitionMock(2, 3, 3)
+ );
+ Mockito.when(topologyMock.localPartitions()).thenReturn(localPartitions);
+ Mockito.when(topologyMock.currentLocalPartitions()).thenReturn(localPartitions);
+ }
+
+ /**
+ * @return Partition mock with specified {@code id}, {@code updateCounter} and {@code size}.
+ */
+ private GridDhtLocalPartition partitionMock(int id, long updateCounter, long size) {
+ GridDhtLocalPartition partitionMock = Mockito.mock(GridDhtLocalPartition.class);
+ Mockito.when(partitionMock.id()).thenReturn(id);
+ Mockito.when(partitionMock.updateCounter()).thenReturn(updateCounter);
+ Mockito.when(partitionMock.fullSize()).thenReturn(size);
+ return partitionMock;
+ }
+
+ /**
+ * @return Message containing specified {@code countersMap}.
+ */
+ private GridDhtPartitionsSingleMessage fromUpdateCounters(Map<Integer, T2<Long, Long>> countersMap) {
+ GridDhtPartitionsSingleMessage msg = new GridDhtPartitionsSingleMessage();
+ msg.addPartitionUpdateCounters(0, countersMap);
+ return msg;
+ }
+
+ /**
+ * @return Message containing specified {@code sizesMap}.
+ */
+ private GridDhtPartitionsSingleMessage fromCacheSizes(Map<Integer, Long> sizesMap) {
+ GridDhtPartitionsSingleMessage msg = new GridDhtPartitionsSingleMessage();
+ msg.addPartitionSizes(0, sizesMap);
+ return msg;
+ }
+
+ /**
+ * Test partition update counters validation.
+ */
+ public void testPartitionCountersValidation() {
+ UUID remoteNode = UUID.randomUUID();
+ UUID ignoreNode = UUID.randomUUID();
+
+ // For partitions 0 and 2 (zero counter) we have inconsistent update counters.
+ Map<Integer, T2<Long, Long>> updateCountersMap = new HashMap<>();
+ updateCountersMap.put(0, new T2<>(2L, 2L));
+ updateCountersMap.put(1, new T2<>(2L, 2L));
+
+ // Form single messages map.
+ Map<UUID, GridDhtPartitionsSingleMessage> messages = new HashMap<>();
+ messages.put(remoteNode, fromUpdateCounters(updateCountersMap));
+ messages.put(ignoreNode, fromUpdateCounters(updateCountersMap));
+
+ GridDhtPartitionsStateValidator validator = new GridDhtPartitionsStateValidator(cctxMock);
+
+ // (partId, (nodeId, updateCounter))
+ Map<Integer, Map<UUID, Long>> result = validator.validatePartitionsUpdateCounters(topologyMock, messages, Sets.newHashSet(ignoreNode));
+
+ // Check that validation result contains all necessary information.
+ Assert.assertEquals(2, result.size());
+ Assert.assertTrue(result.containsKey(0));
+ Assert.assertTrue(result.containsKey(2));
+ Assert.assertTrue(result.get(0).get(localNodeId) == 1L);
+ Assert.assertTrue(result.get(0).get(remoteNode) == 2L);
+ Assert.assertTrue(result.get(2).get(localNodeId) == 3L);
+ Assert.assertTrue(result.get(2).get(remoteNode) == 0L);
+ }
+
+ /**
+ * Test partition cache sizes validation.
+ */
+ public void testPartitionCacheSizesValidation() {
+ UUID remoteNode = UUID.randomUUID();
+ UUID ignoreNode = UUID.randomUUID();
+
+ // For partitions 0 and 2 we have inconsistent cache sizes.
+ Map<Integer, Long> cacheSizesMap = new HashMap<>();
+ cacheSizesMap.put(0, 2L);
+ cacheSizesMap.put(1, 2L);
+ cacheSizesMap.put(2, 2L);
+
+ // Form single messages map.
+ Map<UUID, GridDhtPartitionsSingleMessage> messages = new HashMap<>();
+ messages.put(remoteNode, fromCacheSizes(cacheSizesMap));
+ messages.put(ignoreNode, fromCacheSizes(cacheSizesMap));
+
+ GridDhtPartitionsStateValidator validator = new GridDhtPartitionsStateValidator(cctxMock);
+
+ // (partId, (nodeId, cacheSize))
+ Map<Integer, Map<UUID, Long>> result = validator.validatePartitionsSizes(topologyMock, messages, Sets.newHashSet(ignoreNode));
+
+ // Check that validation result contains all necessary information.
+ Assert.assertEquals(2, result.size());
+ Assert.assertTrue(result.containsKey(0));
+ Assert.assertTrue(result.containsKey(2));
+ Assert.assertTrue(result.get(0).get(localNodeId) == 1L);
+ Assert.assertTrue(result.get(0).get(remoteNode) == 2L);
+ Assert.assertTrue(result.get(2).get(localNodeId) == 3L);
+ Assert.assertTrue(result.get(2).get(remoteNode) == 2L);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticOnPartitionExchangeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticOnPartitionExchangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticOnPartitionExchangeTest.java
new file mode 100644
index 0000000..03ea0f7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticOnPartitionExchangeTest.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.GridCacheAffinityManager;
+import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRequest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareRequest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.T1;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
+
+/**
+ *
+ */
+public class TxOptimisticOnPartitionExchangeTest extends GridCommonAbstractTest {
+ /** Nodes count. */
+ private static final int NODES_CNT = 3;
+
+ /** Tx size. */
+ private static final int TX_SIZE = 20 * NODES_CNT;
+
+ /** Cache name. */
+ private static final String CACHE_NAME = "cache";
+
+ /** Logger started. */
+ private static volatile boolean msgInterception;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ startGrids(NODES_CNT);
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ stopAllGrids();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setCommunicationSpi(new TestCommunicationSpi(log()));
+
+ cfg.setCacheConfiguration(defaultCacheConfiguration()
+ .setName(CACHE_NAME)
+ .setAtomicityMode(TRANSACTIONAL)
+ .setWriteSynchronizationMode(FULL_SYNC)
+ .setCacheMode(PARTITIONED)
+ .setBackups(1));
+
+ return cfg;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testConsistencyOnPartitionExchange() throws Exception {
+ doTest(SERIALIZABLE, true);
+ doTest(READ_COMMITTED, true);
+ doTest(SERIALIZABLE, false);
+ doTest(READ_COMMITTED, false);
+ }
+
+ /**
+ * @param isolation {@link TransactionIsolation}.
+ * @param txInitiatorPrimary False If the transaction does not use the keys of the node that initiated it.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ public void doTest(final TransactionIsolation isolation, boolean txInitiatorPrimary) throws Exception {
+ final CountDownLatch txStarted = new CountDownLatch(1);
+
+ final IgniteCache cache = ignite(0).cache(CACHE_NAME);
+
+ final Map<Integer, Integer> txValues = new TreeMap<>();
+
+ ClusterNode node = ignite(0).cluster().node();
+
+ GridCacheAffinityManager affinity = ((IgniteCacheProxy)cache).context().affinity();
+
+ for (int i = 0; txValues.size() < TX_SIZE; i++) {
+ if (!txInitiatorPrimary && node.equals(affinity.primaryByKey(i, NONE)))
+ continue;
+
+ txValues.put(i, i);
+ }
+
+ TestCommunicationSpi.init();
+
+ msgInterception = true;
+
+ IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+ @Override public Object call() {
+ try (Transaction tx = ignite(0).transactions().txStart(OPTIMISTIC, isolation)) {
+ info(">>> TX started.");
+
+ txStarted.countDown();
+
+ cache.putAll(txValues);
+
+ tx.commit();
+
+ info(">>> TX committed.");
+ }
+
+ return null;
+ }
+ });
+
+ txStarted.await();
+
+ try {
+ info(">>> Grid starting.");
+
+ IgniteEx ignite = startGrid(NODES_CNT);
+
+ info(">>> Grid started.");
+
+ fut.get();
+
+ awaitPartitionMapExchange();
+
+ msgInterception = false;
+
+ IgniteCache<Object, Object> cacheStartedNode = ignite.cache(CACHE_NAME);
+
+ try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+ Set<Object> keys = cacheStartedNode.getAll(txValues.keySet()).keySet();
+
+ assertEquals(txValues.keySet(), new TreeSet<>(keys));
+
+ tx.commit();
+ }
+ }
+ finally {
+ msgInterception = false;
+
+ stopGrid(NODES_CNT);
+ }
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ConstantConditions")
+ private static class TestCommunicationSpi extends TcpCommunicationSpi {
+ /** Partition single message sent from added node. */
+ private static volatile CountDownLatch partSingleMsgSentFromAddedNode;
+
+ /** Partition supply message sent count. */
+ private static final AtomicInteger partSupplyMsgSentCnt = new AtomicInteger();
+
+ /** Logger. */
+ private IgniteLogger log;
+
+ /**
+ * @param log Logger.
+ */
+ public TestCommunicationSpi(IgniteLogger log) {
+ this.log = log;
+ }
+
+ /**
+ *
+ */
+ public static void init() {
+ partSingleMsgSentFromAddedNode = new CountDownLatch(1);
+
+ partSupplyMsgSentCnt.set(0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void sendMessage(
+ final ClusterNode node,
+ final Message msg,
+ final IgniteInClosure<IgniteException> ackC
+ ) throws IgniteSpiException {
+ if (msgInterception) {
+ if (msg instanceof GridIoMessage) {
+ final Message msg0 = ((GridIoMessage)msg).message();
+
+ String locNodeId = ((IgniteEx)ignite).context().localNodeId().toString();
+
+ int nodeIdx = Integer.parseInt(locNodeId.substring(locNodeId.length() - 3));
+
+ if (nodeIdx == 0) {
+ if (msg0 instanceof GridNearTxPrepareRequest || msg0 instanceof GridDhtTxPrepareRequest) {
+ GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ partSingleMsgSentFromAddedNode.await();
+
+ sendMessage(node, msg, ackC, true);
+
+ return null;
+ }
+ });
+
+ return;
+
+ }
+ else if (msg0 instanceof GridNearTxFinishRequest || msg0 instanceof GridDhtTxFinishRequest) {
+ GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ final T1<Integer> i = new T1<>(0);
+
+ while (waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return partSupplyMsgSentCnt.get() > i.get();
+ }
+ }, i.get() == 0 ? 5_000 : 500))
+ i.set(partSupplyMsgSentCnt.get());
+
+ sendMessage(node, msg, ackC, true);
+
+ return null;
+ }
+ });
+
+ return;
+ }
+ }
+ else if (nodeIdx == NODES_CNT && msg0 instanceof GridDhtPartitionsSingleMessage)
+ partSingleMsgSentFromAddedNode.countDown();
+
+ if (msg0 instanceof GridDhtPartitionSupplyMessage)
+ partSupplyMsgSentCnt.incrementAndGet();
+ }
+ }
+
+ sendMessage(node, msg, ackC, msgInterception);
+ }
+
+ /**
+ * @param node Node.
+ * @param msg Message.
+ * @param ackC Ack closure.
+ * @param logMsg Log Messages.
+ */
+ private void sendMessage(
+ final ClusterNode node,
+ final Message msg,
+ final IgniteInClosure<IgniteException> ackC,
+ boolean logMsg
+ ) throws IgniteSpiException {
+ if (logMsg) {
+ String id = node.id().toString();
+ String locNodeId = ((IgniteEx)ignite).context().localNodeId().toString();
+
+ Message msg0 = ((GridIoMessage)msg).message();
+
+ log.info(
+ String.format(">>> Output msg[type=%s, fromNode= %s, toNode=%s]",
+ msg0.getClass().getSimpleName(),
+ locNodeId.charAt(locNodeId.length() - 1),
+ id.charAt(id.length() - 1)
+ )
+ );
+ }
+
+ super.sendMessage(node, msg, ackC);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index bb397f7..0612615 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -133,6 +133,8 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteCrossCacheT
import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheAtomicNearCacheSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedTxExceptionSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheGlobalLoadTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionsStateValidationTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionsStateValidatorSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheGetStoreErrorSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearTxExceptionSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxExceptionSelfTest;
@@ -292,6 +294,8 @@ public class IgniteCacheTestSuite extends TestSuite {
suite.addTestSuite(IgniteCacheSystemTransactionsSelfTest.class);
suite.addTestSuite(CacheDeferredDeleteSanitySelfTest.class);
suite.addTestSuite(CacheDeferredDeleteQueueTest.class);
+ suite.addTestSuite(GridCachePartitionsStateValidatorSelfTest.class);
+ suite.addTestSuite(GridCachePartitionsStateValidationTest.class);
suite.addTest(IgniteCacheTcpClientDiscoveryTestSuite.suite());
http://git-wip-us.apache.org/repos/asf/ignite/blob/da77b981/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index f8add30..415479d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -31,6 +31,7 @@ import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalPessim
import org.apache.ignite.internal.processors.cache.WalModeChangeAdvancedSelfTest;
import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAffinityNodeSelfTest;
import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
+import org.apache.ignite.internal.processors.cache.datastructures.IgniteExchangeLatchManagerCoordinatorFailTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheExchangeMergeTest;
import org.apache.ignite.internal.processors.cache.distributed.CachePartitionStateTest;
import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionEvictionDuringReadThroughSelfTest;
@@ -40,6 +41,7 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticT
import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeTest;
import org.apache.ignite.internal.processors.cache.distributed.IgnitePessimisticTxSuspendResumeTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticOnPartitionExchangeTest;
import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNearCacheTest;
import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNoDeadlockDetectionTest;
import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest;
@@ -93,6 +95,10 @@ public class IgniteCacheTestSuite6 extends TestSuite {
suite.addTestSuite(PartitionedTransactionalOptimisticCacheGetsDistributionTest.class);
suite.addTestSuite(PartitionedTransactionalPessimisticCacheGetsDistributionTest.class);
+ suite.addTestSuite(TxOptimisticOnPartitionExchangeTest.class);
+
+ suite.addTestSuite(IgniteExchangeLatchManagerCoordinatorFailTest.class);
+
return suite;
}
}