You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/06/01 08:52:58 UTC

[01/12] ignite git commit: IGNITE-5210 - If enabled security authentication, server is unable to restart if client tries to reconnect

Repository: ignite
Updated Branches:
  refs/heads/ignite-5075 70fc1a019 -> eeba611c8


IGNITE-5210 - If enabled security authentication, server is unable to restart if client tries to reconnect


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a7e56606
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a7e56606
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a7e56606

Branch: refs/heads/ignite-5075
Commit: a7e566068c143a5c33ea0f583157bd197068b321
Parents: 738451c
Author: dkarachentsev <dk...@gridgain.com>
Authored: Wed May 31 15:35:34 2017 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Wed May 31 15:35:34 2017 +0300

----------------------------------------------------------------------
 .../ignite/spi/discovery/tcp/ClientImpl.java    | 14 +++-
 .../discovery/AuthenticationRestartTest.java    | 88 ++++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java        |  2 +
 3 files changed, 100 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/a7e56606/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 619b4cc..d5bacdb 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -89,8 +89,8 @@ import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryCustomEventMessa
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryDuplicateIdMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryHandshakeRequest;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryHandshakeResponse;
-import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryMetricsUpdateMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryJoinRequestMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryMetricsUpdateMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddedMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeFailedMessage;
@@ -639,9 +639,12 @@ class ClientImpl extends TcpDiscoveryImpl {
                 if (!recon) {
                     TcpDiscoveryNode node = locNode;
 
-                    if (locNode.order() > 0)
+                    if (locNode.order() > 0) {
                         node = locNode.clientReconnectNode(spi.spiCtx.nodeAttributes());
 
+                        marshalCredentials(node);
+                    }
+
                     msg = new TcpDiscoveryJoinRequestMessage(
                             node,
                             spi.collectExchangeData(new DiscoveryDataPacket(getLocalNodeId())));
@@ -736,8 +739,11 @@ class ClientImpl extends TcpDiscoveryImpl {
             // Use security-unsafe getter.
             Map<String, Object> attrs = new HashMap<>(node.getAttributes());
 
-            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS,
-                U.marshal(spi.marshaller(), attrs.get(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS)));
+            Object creds = attrs.get(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS);
+
+            assert !(creds instanceof byte[]);
+
+            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS, U.marshal(spi.marshaller(), creds));
 
             node.setAttributes(attrs);
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/a7e56606/modules/core/src/test/java/org/apache/ignite/spi/discovery/AuthenticationRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/AuthenticationRestartTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/AuthenticationRestartTest.java
new file mode 100644
index 0000000..5841094
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/AuthenticationRestartTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TestReconnectPluginProvider;
+import org.apache.ignite.spi.discovery.tcp.TestReconnectProcessor;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
+/**
+ * Checks whether client is able to reconnect to restarted cluster with
+ * enabled security.
+ */
+public class AuthenticationRestartTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setClientMode(igniteInstanceName.contains("client"));
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setJoinTimeout(1120_000);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        TestReconnectPluginProvider.enabled = true;
+        TestReconnectProcessor.enabled = true;
+
+        startGrid("server");
+        startGrid("client");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        TestReconnectPluginProvider.enabled = false;
+        TestReconnectProcessor.enabled = false;
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientReconnect() throws Exception {
+        stopGrid("server");
+
+        final IgniteEx client = grid("client");
+
+        waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return client.cluster().clientReconnectFuture() != null;
+            }
+        }, 10_000);
+
+        startGrid("server");
+
+        IgniteFuture<?> fut = client.cluster().clientReconnectFuture();
+
+        assertNotNull(fut);
+
+        fut.get();
+
+        assertEquals(2, client.cluster().nodes().size());
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/a7e56606/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index e6b39f7..1287149 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -19,6 +19,7 @@ package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
 import org.apache.ignite.spi.GridTcpSpiForwardingSelfTest;
+import org.apache.ignite.spi.discovery.AuthenticationRestartTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoveryMarshallerCheckSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiMulticastTest;
@@ -87,6 +88,7 @@ public class IgniteSpiDiscoverySelfTestSuite extends TestSuite {
         suite.addTest(new TestSuite(TcpDiscoveryMultiThreadedTest.class));
 
         suite.addTest(new TestSuite(TcpDiscoveryNodeAttributesUpdateOnReconnectTest.class));
+        suite.addTest(new TestSuite(AuthenticationRestartTest.class));
 
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));


[11/12] ignite git commit: Merge remote-tracking branch 'remotes/origin/master' into ignite-5075

Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-5075

# Conflicts:
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5ef69009
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5ef69009
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5ef69009

Branch: refs/heads/ignite-5075
Commit: 5ef69009894cc2a6d33e31d923daed2f00bc45a1
Parents: 056b28f c488311
Author: sboikov <sb...@gridgain.com>
Authored: Thu Jun 1 11:52:35 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Jun 1 11:52:35 2017 +0300

----------------------------------------------------------------------
 DEVNOTES.txt                                    |  74 ++--
 assembly/dependencies-fabric-lgpl.xml           |   2 +
 assembly/dependencies-fabric.xml                |   2 +
 assembly/release-fabric-lgpl.xml                |   1 +
 assembly/release-fabric.xml                     |   1 +
 assembly/release-yardstick.xml                  |  98 +++++
 .../affinity/GridAffinityProcessor.java         |   2 +-
 .../cache/CacheAffinitySharedManager.java       |  35 +-
 .../processors/cache/GridCacheAdapter.java      |   3 +-
 .../processors/cache/GridCacheProcessor.java    |  39 +-
 .../CacheDataStructuresManager.java             |   4 +-
 .../GridDhtPartitionsExchangeFuture.java        |   2 +-
 .../closure/GridClosureProcessor.java           |  11 +-
 .../service/GridServiceProcessor.java           |   4 +-
 .../processors/service/GridServiceProxy.java    |   4 +-
 .../processors/task/GridTaskProcessor.java      |   7 +-
 .../task/GridTaskThreadContextKey.java          |   5 +-
 .../ignite/spi/discovery/tcp/ClientImpl.java    |  14 +-
 .../IgniteComputeTopologyExceptionTest.java     |   2 +-
 ...IgniteClientCacheInitializationFailTest.java | 367 +++++++++++++++++++
 .../discovery/AuthenticationRestartTest.java    |  88 +++++
 .../testsuites/IgniteCacheTestSuite4.java       |   2 +
 .../IgniteSpiDiscoverySelfTestSuite.java        |   2 +
 .../query/h2/sql/GridSqlQuerySplitter.java      |  22 +-
 ...ClientReconnectCacheQueriesFailoverTest.java |   2 +-
 .../query/IgniteSqlSplitterSelfTest.java        | 256 ++++++++++---
 modules/ml/pom.xml                              |  13 +
 .../main/java/org/apache/ignite/ml/Model.java   |  39 ++
 .../ml/clustering/BaseKMeansClusterer.java      |  98 +++++
 .../apache/ignite/ml/clustering/Clusterer.java  |  32 ++
 .../ml/clustering/ClusterizationModel.java      |  29 ++
 .../clustering/KMeansDistributedClusterer.java  | 298 +++++++++++++++
 .../ml/clustering/KMeansLocalClusterer.java     | 174 +++++++++
 .../ignite/ml/clustering/KMeansModel.java       |  79 ++++
 .../ignite/ml/clustering/WeightedClusterer.java |  38 ++
 .../ignite/ml/clustering/package-info.java      |  22 ++
 .../apache/ignite/ml/math/DistanceMeasure.java  |  39 ++
 .../ignite/ml/math/EuclideanDistance.java       |  48 +++
 .../org/apache/ignite/ml/math/MathUtils.java    |  31 ++
 .../java/org/apache/ignite/ml/math/Matrix.java  |   2 +
 .../apache/ignite/ml/math/StorageConstants.java |   3 +
 .../org/apache/ignite/ml/math/VectorUtils.java  |  41 +++
 .../math/exceptions/ConvergenceException.java   |  48 +++
 .../exceptions/MathArithmeticException.java     |   6 +-
 .../exceptions/MathIllegalNumberException.java  |  51 +++
 .../exceptions/MathIllegalStateException.java   |  49 +++
 .../exceptions/NumberIsTooSmallException.java   |  79 ++++
 .../ignite/ml/math/functions/Functions.java     |  39 ++
 .../apache/ignite/ml/math/impls/CacheUtils.java |  47 +++
 .../ml/math/impls/matrix/AbstractMatrix.java    |   2 +
 .../impls/matrix/SparseDistributedMatrix.java   |   2 +-
 .../impls/storage/matrix/MapWrapperStorage.java |  93 +++++
 .../vector/SparseLocalOnHeapVectorStorage.java  |  32 ++
 .../ml/math/impls/vector/DelegatingVector.java  |   5 +
 .../ml/math/impls/vector/MapWrapperVector.java  |  32 ++
 .../ml/math/impls/vector/SparseLocalVector.java |   9 +
 .../ignite/ml/math/impls/vector/VectorView.java |   1 +
 .../ignite/ml/math/statistics/Variance.java     |  53 +++
 .../org/apache/ignite/ml/math/util/MapUtil.java |  38 ++
 .../apache/ignite/ml/math/util/MatrixUtil.java  |  38 +-
 .../org/apache/ignite/ml/IgniteMLTestSuite.java |   4 +-
 .../ml/clustering/ClusteringTesetSuite.java     |  15 +
 .../KMeansDistributedClustererTest.java         | 184 ++++++++++
 .../ml/clustering/KMeansLocalClustererTest.java |  46 +++
 .../apache/ignite/ml/clustering/KMeansUtil.java |  33 ++
 .../ignite/ml/math/MathImplLocalTestSuite.java  |   1 +
 modules/platforms/cpp/examples/Makefile.am      |   1 +
 .../cpp/examples/compute-example/Makefile.am    |  59 +++
 .../compute-example/config/compute-example.xml  |  52 +++
 .../project/vs/compute-example.vcxproj          | 102 ++++++
 .../project/vs/compute-example.vcxproj.filters  |  27 ++
 .../compute-example/src/compute_example.cpp     | 177 +++++++++
 modules/platforms/cpp/examples/configure.ac     |   1 +
 .../project/vs/continuous-query-example.vcxproj |   4 -
 .../src/continuous_query_example.cpp            | 130 ++++++-
 .../examples/include/ignite/examples/address.h  |   2 +-
 .../include/ignite/examples/organization.h      |   4 +-
 .../cpp/examples/project/vs/ignite-examples.sln |   6 +
 .../Apache.Ignite.Core.nuspec                   |   5 +-
 modules/platforms/dotnet/build.ps1              |   5 +-
 modules/web-console/web-agent/pom.xml           |   8 +
 parent/pom.xml                                  |   8 +-
 pom.xml                                         | 254 ++++---------
 83 files changed, 3452 insertions(+), 335 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/5ef69009/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 1c71dab,9063a12..c563b81
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@@ -378,37 -392,51 +378,44 @@@ public class CacheAffinitySharedManager
                  nearCfg = req.nearCacheConfiguration();
              }
              else {
 -                startCache = cctx.cacheContext(action.descriptor().cacheId()) == null &&
 -                    CU.affinityNode(cctx.localNode(), req.startCacheConfiguration().getNodeFilter());
 +                startCache = cctx.cacheContext(cacheDesc.cacheId()) == null &&
 +                    CU.affinityNode(cctx.localNode(), cacheDesc.groupDescriptor().config().getNodeFilter());
              }
  
-             if (startCache) {
-                 cctx.cache().prepareCacheStart(cacheDesc, nearCfg, fut.topologyVersion());
+             try {
+                 if (startCache) {
+                     cctx.cache().prepareCacheStart(cacheDesc, nearCfg, fut.topologyVersion());
  
-                 if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) {
-                     if (fut.discoCache().cacheGroupAffinityNodes(cacheDesc.groupId()).isEmpty())
-                         U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+                     if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) {
 -                        if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
++                        if (fut.discoCache().cacheGroupAffinityNodes(cacheDesc.groupId()).isEmpty())
+                             U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+                     }
                  }
 +            }
++            catch (IgniteCheckedException e) {
++                U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
++                    "[cacheName=" + req.cacheName() + ']', e);
+ 
 -                if (!crd || !lateAffAssign) {
 -                    GridCacheContext cacheCtx = cctx.cacheContext(cacheDesc.cacheId());
 -
 -                    if (cacheCtx != null && !cacheCtx.isLocal()) {
 -                        boolean clientCacheStarted =
 -                            req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
++                cctx.cache().forceCloseCache(fut.topologyVersion(), action, e);
++            }
 +        }
  
-         Set<Integer> gprs = new HashSet<>();
 -                        if (clientCacheStarted)
 -                            initAffinity(cacheDesc, cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
 -                        else if (!req.clientStartOnly()) {
 -                            assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
++            Set<Integer> gprs = new HashSet<>();
  
-         for (ExchangeActions.ActionData action : exchActions.newAndClientCachesStartRequests()) {
-             Integer grpId = action.descriptor().groupId();
 -                            GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
++                for (ExchangeActions.ActionData action : exchActions.newAndClientCachesStartRequests()) {
++                    Integer grpId = action.descriptor().groupId();
  
-             if (gprs.add(grpId)) {
-                 if (crd && lateAffAssign)
-                     initStartedGroupOnCoordinator(fut, action.descriptor().groupDescriptor());
-                 else {
-                     CacheGroupInfrastructure grp = cctx.cache().cacheGroup(grpId);
 -                            assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
++                    if (gprs.add(grpId)) {
++                        if (crd && lateAffAssign)
++                    initStartedGroupOnCoordinator(fut, action.descriptor().groupDescriptor());else  {
++                        CacheGroupInfrastructure grp = cctx.cache().cacheGroup(grpId);
  
-                     if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.topologyVersion())) {
 -                            List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
 -                                fut.discoveryEvent(), fut.discoCache());
++                        if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.topologyVersion())) {
 +                        assert grp.affinity().lastVersion().equals(AffinityTopologyVersion.NONE) : grp.affinity().lastVersion();
  
 -                            aff.initialize(fut.topologyVersion(), assignment);
 -                        }
 +                        initAffinity(registeredGrps.get(grp.groupId()), grp.affinity(), fut);
                      }
                  }
 -                else
 -                    initStartedCacheOnCoordinator(fut, cacheDesc.cacheId());
 -            }
 -            catch (IgniteCheckedException e) {
 -                U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
 -                    "[cacheName=" + req.cacheName() + ']', e);
 -
 -                cctx.cache().forceCloseCache(fut.topologyVersion(), action, e);
              }
          }
  

http://git-wip-us.apache.org/repos/asf/ignite/blob/5ef69009/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/5ef69009/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index cac8420,c7e8670..afb578b
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@@ -2031,10 -1889,9 +2031,10 @@@ public class GridCacheProcessor extend
  
      /**
       * @param req Stop request.
 +     * @return Cache group for stopped cache.
       */
-     private CacheGroupInfrastructure prepareCacheStop(DynamicCacheChangeRequest req) {
-         assert req.stop() || req.close() : req;
 -    private void prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose) {
++    private CacheGroupInfrastructure prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose) {
+         assert req.stop() || req.close() || forceClose : req;
  
          GridCacheAdapter<?, ?> cache = caches.remove(req.cacheName());
  
@@@ -2081,14 -1956,11 +2103,14 @@@
              for (ExchangeActions.ActionData action : exchActions.cacheStopRequests()) {
                  stopGateway(action.request());
  
-                 prepareCacheStop(action.request());
+                 prepareCacheStop(action.request(), forceClose);
              }
  
 -            for (DynamicCacheChangeRequest req : exchActions.closeRequests(ctx.localNodeId())) {
 -                String cacheName = req.cacheName();
 +            for (CacheGroupDescriptor grpDesc : exchActions.cacheGroupsToStop())
 +                stopCacheGroup(grpDesc.groupId());
 +
 +            for (ExchangeActions.ActionData req : exchActions.closeRequests(ctx.localNodeId())) {
 +                String cacheName = req.request().cacheName();
  
                  IgniteCacheProxy<?, ?> proxy = jCacheProxies.get(cacheName);
  
@@@ -2105,12 -1977,12 +2127,15 @@@
  
                          proxy.context().gate().onStopped();
  
-                         CacheGroupInfrastructure grp = prepareCacheStop(req.request());
 -                        prepareCacheStop(req, forceClose);
++                        CacheGroupInfrastructure grp = prepareCacheStop(req.request(), forceClose);
 +
 +                        if (grp != null && !grp.hasCaches())
 +                            stopCacheGroup(grp.groupId());
                      }
                  }
+ 
+                 if (forceClose)
+                     completeCacheStartFuture(req, false, err);
              }
          }
      }

http://git-wip-us.apache.org/repos/asf/ignite/blob/5ef69009/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 65a20eb,2cd5629..d000e26
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@@ -1197,16 -1161,16 +1197,16 @@@ public class GridDhtPartitionsExchangeF
  
                  boolean valid = true;
  
 -                if (cacheCtx.config().getTopologyValidator() != null && !CU.isSystemCache(cacheCtx.name()))
 -                    valid = cacheCtx.config().getTopologyValidator().validate(discoEvt.topologyNodes());
 +                if (grp.topologyValidator() != null && !grp.systemCache())
 +                    valid = grp.topologyValidator().validate(discoEvt.topologyNodes());
  
 -                m.put(cacheCtx.cacheId(), new CacheValidation(valid, lostParts));
 +                m.put(grp.groupId(), new CacheValidation(valid, lostParts));
              }
  
 -            cacheValidRes = m;
 +            grpValidRes = m;
          }
  
-         cctx.cache().onExchangeDone(exchId.topologyVersion(), exchActions, err);
+         cctx.cache().onExchangeDone(exchId.topologyVersion(), exchActions, err, false);
  
          cctx.exchange().onExchangeDone(this, err);
  


[05/12] ignite git commit: IGNITE-5161: Implemented C++ compute example.

Posted by sb...@apache.org.
IGNITE-5161: Implemented C++ compute example.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d0bddfa5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d0bddfa5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d0bddfa5

Branch: refs/heads/ignite-5075
Commit: d0bddfa527af6c5cf4554e484ce04c078bfd69e9
Parents: 41dcce7
Author: Igor Sapego <is...@gridgain.com>
Authored: Wed May 31 18:36:52 2017 +0300
Committer: Igor Sapego <is...@gridgain.com>
Committed: Wed May 31 18:36:52 2017 +0300

----------------------------------------------------------------------
 modules/platforms/cpp/examples/Makefile.am      |   1 +
 .../cpp/examples/compute-example/Makefile.am    |  59 +++++++
 .../compute-example/config/compute-example.xml  |  52 ++++++
 .../project/vs/compute-example.vcxproj          | 102 +++++++++++
 .../project/vs/compute-example.vcxproj.filters  |  27 +++
 .../compute-example/src/compute_example.cpp     | 177 +++++++++++++++++++
 modules/platforms/cpp/examples/configure.ac     |   1 +
 .../project/vs/continuous-query-example.vcxproj |   4 -
 .../src/continuous_query_example.cpp            |   5 +-
 .../examples/include/ignite/examples/address.h  |   2 +-
 .../include/ignite/examples/organization.h      |   4 +-
 .../cpp/examples/project/vs/ignite-examples.sln |   6 +
 12 files changed, 430 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/Makefile.am
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/Makefile.am b/modules/platforms/cpp/examples/Makefile.am
index 8cfb999..29b140b 100644
--- a/modules/platforms/cpp/examples/Makefile.am
+++ b/modules/platforms/cpp/examples/Makefile.am
@@ -22,5 +22,6 @@ SUBDIRS = \
     odbc-example \
     query-example \
 	continuous-query-example \
+	compute-example \
     include
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/compute-example/Makefile.am
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/compute-example/Makefile.am b/modules/platforms/cpp/examples/compute-example/Makefile.am
new file mode 100644
index 0000000..d757bf7
--- /dev/null
+++ b/modules/platforms/cpp/examples/compute-example/Makefile.am
@@ -0,0 +1,59 @@
+##
+## Licensed to the Apache Software Foundation (ASF) under one or more
+## contributor license agreements.  See the NOTICE file distributed with
+## this work for additional information regarding copyright ownership.
+## The ASF licenses this file to You under the Apache License, Version 2.0
+## (the "License"); you may not use this file except in compliance with
+## the License.  You may obtain a copy of the License at
+##
+##      http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+
+ACLOCAL_AMFLAGS =-I m4
+
+noinst_PROGRAMS = ignite-compute-example
+
+AM_CPPFLAGS = \
+    -I@top_srcdir@/include \
+    -I@top_srcdir@/../core/include \
+    -I@top_srcdir@/../core/os/linux/include \
+    -I@top_srcdir@/../common/include \
+    -I@top_srcdir@/../common/os/linux/include \
+    -I@top_srcdir@/../binary/include \
+    -I@top_srcdir@/../jni/include \
+    -I@top_srcdir@/../jni/os/linux/include \
+    -I$(JAVA_HOME)/include \
+    -I$(JAVA_HOME)/include/linux \
+    -DIGNITE_IMPL \
+    -D__STDC_LIMIT_MACROS \
+    -D__STDC_CONSTANT_MACROS
+
+AM_CXXFLAGS = \
+    -Wall \
+    -std=c++03
+
+ignite_compute_example_LDADD = \
+    @top_srcdir@/../core/libignite.la \
+    -lodbc \
+    -lpthread
+
+ignite_compute_example_LDFLAGS = \
+    -static-libtool-libs
+
+ignite_compute_example_SOURCES = \
+    src/compute_example.cpp
+
+run-check: check
+	./ignite-compute-example -p
+
+clean-local: clean-check
+	$(RM) *.gcno *.gcda
+
+clean-check:
+	$(RM) $(ignite_compute_example_OBJECTS)

http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/compute-example/config/compute-example.xml
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/compute-example/config/compute-example.xml b/modules/platforms/cpp/examples/compute-example/config/compute-example.xml
new file mode 100644
index 0000000..1348b1c
--- /dev/null
+++ b/modules/platforms/cpp/examples/compute-example/config/compute-example.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:util="http://www.springframework.org/schema/util"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util
+        http://www.springframework.org/schema/util/spring-util.xsd">
+    <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+
+        <!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. -->
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <!--
+                        Ignite provides several options for automatic discovery that can be used
+                        instead os static IP based discovery.
+                    -->
+                    <!-- Uncomment static IP finder to enable static-based discovery of initial nodes. -->
+                    <!--<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">-->
+                    <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">
+                        <property name="addresses">
+                            <list>
+                                <!-- In distributed environment, replace with actual host IP address. -->
+                                <value>127.0.0.1:47550..47551</value>
+                            </list>
+                        </property>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+    </bean>
+</beans>

http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj b/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj
new file mode 100644
index 0000000..1b84a9c
--- /dev/null
+++ b/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj
@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <RootNamespace>igniteexamples</RootNamespace>
+    <ProjectGuid>{18BB0A18-8213-472A-81A0-9D9753697135}</ProjectGuid>
+    <ProjectName>compute-example</ProjectName>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v100</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v100</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;..\..\..\include;..\..\..\..\jni\os\win\include;..\..\..\..\jni\include;..\..\..\..\common\os\win\include;..\..\..\..\common\include;..\..\..\..\binary\include;..\..\..\..\core\os\win\include;..\..\..\..\core\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalDependencies>jvm.lib;ignite.jni.lib;ignite.binary.lib;ignite.core.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>..\..\..\..\project\vs\$(Platform)\$(Configuration)\;$(JAVA_HOME)\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+    </Link>
+    <PostBuildEvent>
+      <Command>copy "$(ProjectDir)..\..\..\..\project\vs\$(Platform)\$(Configuration)\ignite.common.dll" "$(OutDir)"
+copy "$(ProjectDir)..\..\..\..\project\vs\$(Platform)\$(Configuration)\ignite.core.dll" "$(OutDir)"</Command>
+    </PostBuildEvent>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;..\..\..\include;..\..\..\..\jni\os\win\include;..\..\..\..\jni\include;..\..\..\..\common\os\win\include;..\..\..\..\common\include;..\..\..\..\binary\include;..\..\..\..\core\os\win\include;..\..\..\..\core\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalDependencies>jvm.lib;ignite.jni.lib;ignite.binary.lib;ignite.core.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>..\..\..\..\project\vs\$(Platform)\$(Configuration)\;$(JAVA_HOME)\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+    </Link>
+    <PostBuildEvent>
+      <Command>copy "$(ProjectDir)..\..\..\..\project\vs\$(Platform)\$(Configuration)\ignite.common.dll" "$(OutDir)"
+copy "$(ProjectDir)..\..\..\..\project\vs\$(Platform)\$(Configuration)\ignite.core.dll" "$(OutDir)"</Command>
+    </PostBuildEvent>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\src\compute_example.cpp" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="..\..\config\compute-example.xml" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj.filters
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj.filters b/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj.filters
new file mode 100644
index 0000000..d43cece
--- /dev/null
+++ b/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj.filters
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Source Files">
+      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+      <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    </Filter>
+    <Filter Include="Header Files">
+      <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+      <Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
+    </Filter>
+    <Filter Include="Config">
+      <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+      <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\src\compute_example.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="..\..\config\compute-example.xml">
+      <Filter>Config</Filter>
+    </None>
+  </ItemGroup>
+</Project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/compute-example/src/compute_example.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/compute-example/src/compute_example.cpp b/modules/platforms/cpp/examples/compute-example/src/compute_example.cpp
new file mode 100644
index 0000000..3426475
--- /dev/null
+++ b/modules/platforms/cpp/examples/compute-example/src/compute_example.cpp
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdint.h>
+#include <iostream>
+#include <sstream>
+
+#include <ignite/ignition.h>
+#include <ignite/compute/compute.h>
+
+using namespace ignite;
+
+/*
+ * Function class.
+ */
+class CountWords : public compute::ComputeFunc<int32_t>
+{
+    friend struct ignite::binary::BinaryType<CountWords>;
+public:
+    /*
+     * Default constructor.
+     */
+    CountWords() :
+        text()
+    {
+        // No-op.
+    }
+
+    /*
+     * Constructor.
+     *
+     * @param text Text.
+     */
+    CountWords(const std::string& text) :
+        text(text)
+    {
+        // No-op.
+    }
+
+    /**
+     * Callback.
+     * Counts number of words in provided text.
+     *
+     * @return Number of words in provided text.
+     */
+    virtual int32_t Call()
+    {
+        std::stringstream buf(text);
+        std::string word;
+
+        int32_t wordsCount = 0;
+        while (buf >> word)
+            ++wordsCount;
+
+        return wordsCount;
+    }
+
+private:
+    /** Text in which to count words. */
+    std::string text;
+};
+
+namespace ignite
+{
+    namespace binary
+    {
+        template<>
+        struct BinaryType<CountWords>
+        {
+            static int32_t GetTypeId()
+            {
+                return GetBinaryStringHashCode("CountWords");
+            }
+
+            static void GetTypeName(std::string& dst)
+            {
+                dst = "CountWords";
+            }
+
+            static int32_t GetFieldId(const char* name)
+            {
+                return GetBinaryStringHashCode(name);
+            }
+
+            static int32_t GetHashCode(const CountWords& obj)
+            {
+                return 0;
+            }
+
+            static bool IsNull(const CountWords& obj)
+            {
+                return false;
+            }
+
+            static void GetNull(CountWords& dst)
+            {
+                dst = CountWords("");
+            }
+
+            static void Write(BinaryWriter& writer, const CountWords& obj)
+            {
+                writer.RawWriter().WriteString(obj.text);
+            }
+
+            static void Read(BinaryReader& reader, CountWords& dst)
+            {
+                dst.text = reader.RawReader().ReadString();
+            }
+        };
+    }
+}
+
+int main()
+{
+    IgniteConfiguration cfg;
+
+    cfg.springCfgPath = "platforms/cpp/examples/compute-example/config/compute-example.xml";
+
+    try
+    {
+        // Start a node.
+        Ignite ignite = Ignition::Start(cfg);
+
+        std::cout << std::endl;
+        std::cout << ">>> Compute example started." << std::endl;
+        std::cout << std::endl;
+
+        // Get binding instance.
+        IgniteBinding binding = ignite.GetBinding();
+
+        // Registering our class as a compute function.
+        binding.RegisterComputeFunc<CountWords>();
+
+        // Get compute instance.
+        compute::Compute compute = ignite.GetCompute();
+
+        std::string testText = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
+
+        // Declaring counter.
+        CountWords counter(testText);
+
+        // Making call.
+        int32_t wordsCount = compute.Call<int32_t>(counter);
+
+        // Printing result.
+        std::cout << ">>> Text contains " << wordsCount << " words" << std::endl;
+
+        // Stop node.
+        Ignition::StopAll(false);
+    }
+    catch (IgniteError& err)
+    {
+        std::cout << "An error occurred: " << err.GetText() << std::endl;
+    }
+
+    std::cout << std::endl;
+    std::cout << ">>> Example finished, press 'Enter' to exit ..." << std::endl;
+    std::cout << std::endl;
+
+    std::cin.get();
+
+    return 0;
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/configure.ac
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/configure.ac b/modules/platforms/cpp/examples/configure.ac
index 7aef863..a72471d 100644
--- a/modules/platforms/cpp/examples/configure.ac
+++ b/modules/platforms/cpp/examples/configure.ac
@@ -57,6 +57,7 @@ AC_CONFIG_FILES([ \
     odbc-example/Makefile \
     query-example/Makefile \
     continuous-query-example/Makefile \
+    compute-example/Makefile \
 ])
 
 AC_OUTPUT

http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/continuous-query-example/project/vs/continuous-query-example.vcxproj
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/continuous-query-example/project/vs/continuous-query-example.vcxproj b/modules/platforms/cpp/examples/continuous-query-example/project/vs/continuous-query-example.vcxproj
index dfc5f9d..585796b 100644
--- a/modules/platforms/cpp/examples/continuous-query-example/project/vs/continuous-query-example.vcxproj
+++ b/modules/platforms/cpp/examples/continuous-query-example/project/vs/continuous-query-example.vcxproj
@@ -48,8 +48,6 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
-      <PrecompiledHeader>
-      </PrecompiledHeader>
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
@@ -72,8 +70,6 @@ copy "$(ProjectDir)..\..\..\..\project\vs\$(Platform)\$(Configuration)\ignite.co
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
-      <PrecompiledHeader>
-      </PrecompiledHeader>
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>

http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp b/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
index 2ca3163..3b17b53 100644
--- a/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
+++ b/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
@@ -18,10 +18,9 @@
 #include <stdint.h>
 #include <iostream>
 
-#include "ignite/ignition.h"
-#include "ignite/cache/query/continuous/continuous_query.h"
+#include <ignite/ignition.h>
+#include <ignite/cache/query/continuous/continuous_query.h>
 
-#include "ignite/examples/organization.h"
 #include "ignite/examples/person.h"
 
 using namespace ignite;

http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/include/ignite/examples/address.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/include/ignite/examples/address.h b/modules/platforms/cpp/examples/include/ignite/examples/address.h
index 6a06f60..8f2689c 100644
--- a/modules/platforms/cpp/examples/include/ignite/examples/address.h
+++ b/modules/platforms/cpp/examples/include/ignite/examples/address.h
@@ -74,7 +74,7 @@ namespace ignite
                 return GetBinaryStringHashCode(name);
             }
 
-            static int32_t GetHashCode(ignite::examples::Address obj)
+            static int32_t GetHashCode(ignite::examples::Address& obj)
             {
                 return 0;
             }

http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/include/ignite/examples/organization.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/include/ignite/examples/organization.h b/modules/platforms/cpp/examples/include/ignite/examples/organization.h
index 942897d..2701933 100644
--- a/modules/platforms/cpp/examples/include/ignite/examples/organization.h
+++ b/modules/platforms/cpp/examples/include/ignite/examples/organization.h
@@ -83,12 +83,12 @@ namespace ignite
                 return GetBinaryStringHashCode(name);
             }
 
-            static int32_t GetHashCode(ignite::examples::Organization obj)
+            static int32_t GetHashCode(const ignite::examples::Organization& obj)
             {
                 return 0;
             }
 
-            static bool IsNull(ignite::examples::Organization obj)
+            static bool IsNull(const ignite::examples::Organization& obj)
             {
                 return false;
             }

http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/project/vs/ignite-examples.sln
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/project/vs/ignite-examples.sln b/modules/platforms/cpp/examples/project/vs/ignite-examples.sln
index a1865e3..bf83743 100644
--- a/modules/platforms/cpp/examples/project/vs/ignite-examples.sln
+++ b/modules/platforms/cpp/examples/project/vs/ignite-examples.sln
@@ -9,6 +9,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "continuous-query-example",
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "put-get-example", "..\..\put-get-example\project\vs\put-get-example.vcxproj", "{34935DEC-80FC-4168-AA52-3DBFF4F79B6B}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "compute-example", "..\..\compute-example\project\vs\compute-example.vcxproj", "{18BB0A18-8213-472A-81A0-9D9753697135}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Release|x64 = Release|x64
@@ -31,6 +33,10 @@ Global
 		{34935DEC-80FC-4168-AA52-3DBFF4F79B6B}.Release|x64.Build.0 = Release|x64
 		{34935DEC-80FC-4168-AA52-3DBFF4F79B6B}.Release|x86.ActiveCfg = Release|Win32
 		{34935DEC-80FC-4168-AA52-3DBFF4F79B6B}.Release|x86.Build.0 = Release|Win32
+		{18BB0A18-8213-472A-81A0-9D9753697135}.Release|x64.ActiveCfg = Release|x64
+		{18BB0A18-8213-472A-81A0-9D9753697135}.Release|x64.Build.0 = Release|x64
+		{18BB0A18-8213-472A-81A0-9D9753697135}.Release|x86.ActiveCfg = Release|Win32
+		{18BB0A18-8213-472A-81A0-9D9753697135}.Release|x86.Build.0 = Release|Win32
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE


[06/12] ignite git commit: IGNITE-5113: Implemented basic distributed/local kmeans clusterization algorithm.

Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java
new file mode 100644
index 0000000..7fb3534
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.math.*;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.Functions;
+import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.ignite.ml.clustering.KMeansUtil.checkIsInEpsilonNeighbourhood;
+
+/** */
+public class KMeansDistributedClustererTest extends GridCommonAbstractTest {
+    /** Number of nodes in grid. We should use 1 in this test because otherwise algorithm will be unstable
+     * (We cannot guarantee the order in which results are returned from each node). */
+    private static final int NODE_COUNT = 1;
+
+    /** Grid instance. */
+    private Ignite ignite;
+
+    /**
+     * Default constructor.
+     */
+    public KMeansDistributedClustererTest() {
+        super(false);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override protected void beforeTest() throws Exception {
+        ignite = grid(NODE_COUNT);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        for (int i = 1; i <= NODE_COUNT; i++)
+            startGrid(i);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** */
+    @Test
+    public void testPerformClusterAnalysisDegenerate() {
+        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+        KMeansDistributedClusterer clusterer = new KMeansDistributedClusterer(new EuclideanDistance(), 1, 1, 1L);
+
+        double[] v1 = new double[] {1959, 325100};
+        double[] v2 = new double[] {1960, 373200};
+
+        SparseDistributedMatrix points = new SparseDistributedMatrix(2, 2, StorageConstants.ROW_STORAGE_MODE,
+                StorageConstants.RANDOM_ACCESS_MODE);
+
+        points.setRow(0, v1);
+        points.setRow(1, v2);
+
+        KMeansModel mdl = clusterer.cluster(points, 1);
+
+        Assert.assertEquals(1, mdl.centers().length);
+        Assert.assertEquals(2, mdl.centers()[0].size());
+    }
+
+    /** */
+    @Test
+    public void testClusterizationOnDatasetWithObviousStructure() throws IOException {
+        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+        int ptsCnt = 10000;
+        int squareSideLen = 10000;
+
+        Random rnd = new Random(123456L);
+
+        // Let centers be in the vertices of square.
+        Map<Integer, Vector> centers = new HashMap<>();
+        centers.put(100, new DenseLocalOnHeapVector(new double[] {0.0, 0.0}));
+        centers.put(900, new DenseLocalOnHeapVector(new double[] {squareSideLen, 0.0}));
+        centers.put(3000, new DenseLocalOnHeapVector(new double[] {0.0, squareSideLen}));
+        centers.put(6000, new DenseLocalOnHeapVector(new double[] {squareSideLen, squareSideLen}));
+
+        int centersCnt = centers.size();
+
+        SparseDistributedMatrix points = new SparseDistributedMatrix(ptsCnt, 2, StorageConstants.ROW_STORAGE_MODE,
+                StorageConstants.RANDOM_ACCESS_MODE);
+
+        List<Integer> permutation = IntStream.range(0, ptsCnt).boxed().collect(Collectors.toList());
+        Collections.shuffle(permutation, rnd);
+
+        Vector[] mc = new Vector[centersCnt];
+        Arrays.fill(mc, VectorUtils.zeroes(2));
+
+        int centIndex = 0;
+        int totalCount = 0;
+
+        List<Vector> massCenters = new ArrayList<>();
+
+        for (Integer count : centers.keySet()) {
+            for (int i = 0; i < count; i++) {
+                DenseLocalOnHeapVector pnt = (DenseLocalOnHeapVector)new DenseLocalOnHeapVector(2).assign(centers.get(count));
+                // pertrubate point on random value.
+                pnt.map(val -> val + rnd.nextDouble() * squareSideLen / 100);
+                mc[centIndex] = mc[centIndex].plus(pnt);
+                points.assignRow(permutation.get(totalCount), pnt);
+                totalCount++;
+            }
+            massCenters.add(mc[centIndex].times(1 / (double)count));
+            centIndex++;
+        }
+
+        EuclideanDistance dist = new EuclideanDistance();
+        OrderedNodesComparator comp = new OrderedNodesComparator(centers.values().toArray(new Vector[] {}), dist);
+
+        massCenters.sort(comp);
+        KMeansDistributedClusterer clusterer = new KMeansDistributedClusterer(dist, 3, 100, 1L);
+
+        KMeansModel mdl = clusterer.cluster(points, 4);
+        Vector[] resCenters = mdl.centers();
+        Arrays.sort(resCenters, comp);
+
+        checkIsInEpsilonNeighbourhood(resCenters, massCenters.toArray(new Vector[]{}), 30.0);
+    }
+
+    /** */
+    private static class OrderedNodesComparator implements Comparator<Vector> {
+        /** */
+        private final DistanceMeasure measure;
+
+        /** */
+        List<Vector> orderedNodes;
+
+        public OrderedNodesComparator(Vector[] orderedNodes, DistanceMeasure measure) {
+            this.orderedNodes = Arrays.asList(orderedNodes);
+            this.measure = measure;
+        }
+
+        /** */
+        private int findClosestNodeIndex(Vector v) {
+            return Functions.argmin(orderedNodes, v1 -> measure.compute(v1, v)).get1();
+        }
+
+        /** */
+        @Override public int compare(Vector v1, Vector v2) {
+            int ind1 = findClosestNodeIndex(v1);
+            int ind2 = findClosestNodeIndex(v2);
+
+            int signum = (int)Math.signum(ind1 - ind2);
+
+            if (signum != 0)
+                return signum;
+
+            return (int)Math.signum(orderedNodes.get(ind1).minus(v1).kNorm(2) -
+                    orderedNodes.get(ind2).minus(v2).kNorm(2));
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansLocalClustererTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansLocalClustererTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansLocalClustererTest.java
new file mode 100644
index 0000000..b396f5b
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansLocalClustererTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import org.apache.ignite.ml.math.EuclideanDistance;
+import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
+import org.junit.Assert;
+import org.junit.Test;
+
+/** */
+public class KMeansLocalClustererTest {
+    /**
+     * Two points, one cluster, one iteration
+     */
+    @Test
+    public void testPerformClusterAnalysisDegenerate() {
+        KMeansLocalClusterer clusterer = new KMeansLocalClusterer(new EuclideanDistance(), 1, 1L);
+
+        double[] v1 = new double[] {1959, 325100};
+        double[] v2 = new double[] {1960, 373200};
+
+        DenseLocalOnHeapMatrix points = new DenseLocalOnHeapMatrix(new double[][] {
+            v1,
+            v2});
+
+        KMeansModel mdl = clusterer.cluster(points, 1);
+
+        Assert.assertEquals(1, mdl.centers().length);
+        Assert.assertEquals(2, mdl.centers()[0].size());
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansUtil.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansUtil.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansUtil.java
new file mode 100644
index 0000000..0a39748
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import org.apache.ignite.ml.math.Vector;
+
+import static org.junit.Assert.assertTrue;
+
+/** Base test for k-means algorithms. */
+public class KMeansUtil {
+    /** */
+    public static void checkIsInEpsilonNeighbourhood(Vector[] v1s, Vector[] v2s, double epsilon) {
+        for (int i = 0; i < v1s.length; i++) {
+            assertTrue("Not in epsilon neighbourhood (index " + i + ") ",
+                v1s[i].minus(v2s[i]).kNorm(2) < epsilon);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java
index 216fd7b..d164be9 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java
@@ -59,6 +59,7 @@ import org.apache.ignite.ml.math.impls.vector.VectorIterableTest;
 import org.apache.ignite.ml.math.impls.vector.VectorNormTest;
 import org.apache.ignite.ml.math.impls.vector.VectorToMatrixTest;
 import org.apache.ignite.ml.math.impls.vector.VectorViewTest;
+import org.apache.ignite.ml.regressions.OLSMultipleLinearRegressionTest;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
 


[10/12] ignite git commit: ignite-5075

Posted by sb...@apache.org.
ignite-5075


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/056b28fd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/056b28fd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/056b28fd

Branch: refs/heads/ignite-5075
Commit: 056b28fd3530691155156ac3c496223f5342629d
Parents: 70fc1a0
Author: sboikov <sb...@gridgain.com>
Authored: Thu Jun 1 11:41:31 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Jun 1 11:41:31 2017 +0300

----------------------------------------------------------------------
 .../continuous/CacheContinuousQueryManager.java |  13 +--
 ...nuousQueryConcurrentPartitionUpdateTest.java | 106 +++++++++++--------
 2 files changed, 69 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/056b28fd/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index d472054..184e872 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -829,14 +829,15 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
         }
         else {
             synchronized (this) {
-                added = lsnrs.putIfAbsent(lsnrId, lsnr) == null;
-
-                if (added) {
-                    int cnt = lsnrCnt.incrementAndGet();
-
-                    if (cctx.group().sharedGroup() && cnt == 1 && !cctx.isLocal())
+                if (lsnrCnt.get() == 0) {
+                    if (cctx.group().sharedGroup() && !cctx.isLocal())
                         cctx.group().addCacheWithContinuousQuery(cctx);
                 }
+
+                added = lsnrs.putIfAbsent(lsnrId, lsnr) == null;
+
+                if (added)
+                    lsnrCnt.incrementAndGet();
             }
 
             if (added)

http://git-wip-us.apache.org/repos/asf/ignite/blob/056b28fd/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
index 32320f6..34cb777 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
@@ -35,6 +35,7 @@ import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -129,7 +130,7 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
 
                 caches.add(cache.getName());
 
-                cntrs.add(startListener(cache));
+                cntrs.add(startListener(cache).get1());
             }
         }
         else {
@@ -142,7 +143,7 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
 
             caches.add(cache.getName());
 
-            cntrs.add(startListener(cache));
+            cntrs.add(startListener(cache).get1());
         }
 
         Affinity<Integer> aff = srv.affinity(caches.get(0));
@@ -206,7 +207,7 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
      * @param cache Cache.
      * @return Event counter.
      */
-    private AtomicInteger startListener(IgniteCache<Object, Object> cache) {
+    private T2<AtomicInteger, QueryCursor> startListener(IgniteCache<Object, Object> cache) {
         final AtomicInteger evtCnt = new AtomicInteger();
 
         ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
@@ -217,14 +218,15 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
                     assertNotNull(evt.getKey());
                     assertNotNull(evt.getValue());
 
-                    evtCnt.incrementAndGet();
+                    if ((Integer)evt.getValue() >= 0)
+                        evtCnt.incrementAndGet();
                 }
             }
         });
 
-        cache.query(qry);
+        QueryCursor cur = cache.query(qry);
 
-        return evtCnt;
+        return new T2<>(evtCnt, cur);
     }
 
     /**
@@ -267,14 +269,33 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
 
         Ignite client = startGrid(1);
 
-        CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+        List<String> caches = new ArrayList<>();
+
+        if (cacheGrp) {
+            for (int i = 0; i < 3; i++) {
+                CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME + i);
 
-        ccfg.setWriteSynchronizationMode(FULL_SYNC);
-        ccfg.setAtomicityMode(atomicityMode);
+                ccfg.setGroupName("testGroup");
+                ccfg.setWriteSynchronizationMode(FULL_SYNC);
+                ccfg.setAtomicityMode(atomicityMode);
 
-        IgniteCache clientCache = client.createCache(ccfg);
+                IgniteCache cache = client.createCache(ccfg);
 
-        Affinity<Integer> aff = srv.affinity(DEFAULT_CACHE_NAME);
+                caches.add(cache.getName());
+            }
+        }
+        else {
+            CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+            ccfg.setWriteSynchronizationMode(FULL_SYNC);
+            ccfg.setAtomicityMode(atomicityMode);
+
+            IgniteCache cache = client.createCache(ccfg);
+
+            caches.add(cache.getName());
+        }
+
+        Affinity<Integer> aff = srv.affinity(caches.get(0));
 
         final List<Integer> keys = new ArrayList<>();
 
@@ -294,38 +315,27 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
         final int THREADS = 10;
         final int UPDATES = 1000;
 
-        for (int i = 0; i < 5; i++) {
-            log.info("Iteration: " + i);
-
-            ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
-
-            final AtomicInteger evtCnt = new AtomicInteger();
-
-            qry.setLocalListener(new CacheEntryUpdatedListener<Object, Object>() {
-                @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts) {
-                    for (CacheEntryEvent evt : evts) {
-                        assertNotNull(evt.getKey());
-                        assertNotNull(evt.getValue());
-
-                        if ((Integer)evt.getValue() >= 0)
-                            evtCnt.incrementAndGet();
-                    }
-                }
-            });
+        final List<IgniteCache<Object, Object>> srvCaches = new ArrayList<>();
 
-            QueryCursor cur;
+        for (String cacheName : caches)
+            srvCaches.add(srv.cache(cacheName));
 
-            final IgniteCache<Object, Object> srvCache = srv.cache(DEFAULT_CACHE_NAME);
+        for (int i = 0; i < 5; i++) {
+            log.info("Iteration: " + i);
 
             final AtomicBoolean stop = new AtomicBoolean();
 
+            List<T2<AtomicInteger, QueryCursor> > qrys = new ArrayList<>();
+
             try {
                 IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
                     @Override public Void call() throws Exception {
                         ThreadLocalRandom rnd = ThreadLocalRandom.current();
 
-                        while (!stop.get())
-                            srvCache.put(keys.get(rnd.nextInt(KEYS)), rnd.nextInt(100) - 200);
+                        while (!stop.get()) {
+                            for (IgniteCache<Object, Object> srvCache : srvCaches)
+                                srvCache.put(keys.get(rnd.nextInt(KEYS)), rnd.nextInt(100) - 200);
+                        }
 
                         return null;
                     }
@@ -333,7 +343,8 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
 
                 U.sleep(1000);
 
-                cur = clientCache.query(qry);
+                for (String cache : caches)
+                    qrys.add(startListener(client.cache(cache)));
 
                 U.sleep(1000);
 
@@ -349,25 +360,32 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
                 @Override public Void call() throws Exception {
                     ThreadLocalRandom rnd = ThreadLocalRandom.current();
 
-                    for (int i = 0; i < UPDATES; i++)
-                        srvCache.put(keys.get(rnd.nextInt(KEYS)), i);
+                    for (int i = 0; i < UPDATES; i++) {
+                        for (IgniteCache<Object, Object> srvCache : srvCaches)
+                            srvCache.put(keys.get(rnd.nextInt(KEYS)), i);
+                    }
 
                     return null;
                 }
             }, THREADS, "update");
 
+            for (T2<AtomicInteger, QueryCursor>  qry : qrys) {
+                final AtomicInteger evtCnt = qry.get1();
 
-            GridTestUtils.waitForCondition(new GridAbsPredicate() {
-                @Override public boolean apply() {
-                    log.info("Events: " + evtCnt.get());
+                GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                    @Override public boolean apply() {
+                        log.info("Events: " + evtCnt.get());
 
-                    return evtCnt.get() >= THREADS * UPDATES;
-                }
-            }, 5000);
+                        return evtCnt.get() >= THREADS * UPDATES;
+                    }
+                }, 5000);
+            }
 
-            assertEquals(THREADS * UPDATES, evtCnt.get());
+            for (T2<AtomicInteger, QueryCursor>  qry : qrys) {
+                assertEquals(THREADS * UPDATES, qry.get1().get());
 
-            cur.close();
+                qry.get2().close();
+            }
         }
     }
 }


[02/12] ignite git commit: IGNITE-5342 - Skip permission check for TASK_EXECUTE for service jobs

Posted by sb...@apache.org.
IGNITE-5342 - Skip permission check for TASK_EXECUTE for service jobs

(cherry picked from commit d77a134)


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/858d11b3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/858d11b3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/858d11b3

Branch: refs/heads/ignite-5075
Commit: 858d11b33e14c799d90134cf936384532c2d0f7a
Parents: a7e5660
Author: dkarachentsev <dk...@gridgain.com>
Authored: Tue May 30 19:00:47 2017 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Wed May 31 16:10:45 2017 +0300

----------------------------------------------------------------------
 .../processors/affinity/GridAffinityProcessor.java       |  2 +-
 .../internal/processors/cache/GridCacheAdapter.java      |  3 ++-
 .../cache/datastructures/CacheDataStructuresManager.java |  4 ++--
 .../processors/closure/GridClosureProcessor.java         | 11 ++++++++---
 .../processors/service/GridServiceProcessor.java         |  4 ++--
 .../internal/processors/service/GridServiceProxy.java    |  4 ++--
 .../internal/processors/task/GridTaskProcessor.java      |  7 ++++---
 .../processors/task/GridTaskThreadContextKey.java        |  5 ++++-
 .../internal/IgniteComputeTopologyExceptionTest.java     |  2 +-
 9 files changed, 26 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
index e57cf54..87c424a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
@@ -499,7 +499,7 @@ public class GridAffinityProcessor extends GridProcessorAdapter {
     private AffinityInfo affinityInfoFromNode(String cacheName, AffinityTopologyVersion topVer, ClusterNode n)
         throws IgniteCheckedException {
         GridTuple3<GridAffinityMessage, GridAffinityMessage, GridAffinityAssignment> t = ctx.closure()
-            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0).get();
+            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0, false).get();
 
         AffinityFunction f = (AffinityFunction)unmarshall(ctx, n.id(), t.get1());
         AffinityKeyMapper m = (AffinityKeyMapper)unmarshall(ctx, n.id(), t.get2());

http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 307c5cf..ff25dd9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -3541,7 +3541,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
                 new LoadKeysCallable<>(ctx.name(), keys, update, plc, keepBinary),
                 nodes,
                 true,
-                0);
+                0,
+                false);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index 45f0cee..0b81e7a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -526,7 +526,7 @@ public class CacheDataStructuresManager extends GridCacheManagerAdapter {
                         new BlockSetCallable(cctx.name(), id),
                         nodes,
                         true,
-                        0).get();
+                        0, false).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
@@ -550,7 +550,7 @@ public class CacheDataStructuresManager extends GridCacheManagerAdapter {
                         new RemoveSetDataCallable(cctx.name(), id, topVer),
                         nodes,
                         true,
-                        0).get();
+                        0, false).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index 1051807..40e21a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -75,6 +75,7 @@ import org.jetbrains.annotations.Nullable;
 import static org.apache.ignite.compute.ComputeJobResultPolicy.FAILOVER;
 import static org.apache.ignite.compute.ComputeJobResultPolicy.REDUCE;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TIMEOUT;
 
@@ -558,12 +559,13 @@ public class GridClosureProcessor extends GridProcessorAdapter {
     }
 
     /**
+     * @param <R> Type.
      * @param mode Distribution mode.
      * @param job Closure to execute.
      * @param nodes Grid nodes.
      * @param sys If {@code true}, then system pool will be used.
      * @param timeout Timeout.
-     * @param <R> Type.
+     * @param skipAuth Skip authorization check.
      * @return Grid future for collection of closure results.
      */
     public <R> IgniteInternalFuture<R> callAsyncNoFailover(
@@ -571,8 +573,8 @@ public class GridClosureProcessor extends GridProcessorAdapter {
         @Nullable Callable<R> job,
         @Nullable Collection<ClusterNode> nodes,
         boolean sys,
-        long timeout
-    ) {
+        long timeout,
+        boolean skipAuth) {
         assert mode != null;
         assert timeout >= 0 : timeout;
 
@@ -588,6 +590,9 @@ public class GridClosureProcessor extends GridProcessorAdapter {
             ctx.task().setThreadContext(TC_NO_FAILOVER, true);
             ctx.task().setThreadContext(TC_SUBGRID, nodes);
 
+            if (skipAuth)
+                ctx.task().setThreadContext(TC_SKIP_AUTH, true);
+
             if (timeout > 0)
                 ctx.task().setThreadContext(TC_TIMEOUT, timeout);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index dda4328..c2fee02 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -708,8 +708,8 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
             call,
             Collections.singletonList(node),
             false,
-            timeout
-        ).get();
+            timeout,
+            true).get();
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index d2e96ba..d16a4c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -192,8 +192,8 @@ public class GridServiceProxy<T> implements Serializable {
                             new ServiceProxyCallable(mtd.getName(), name, mtd.getParameterTypes(), args),
                             Collections.singleton(node),
                             false,
-                            waitTimeout
-                        ).get();
+                            waitTimeout,
+                            true).get();
                     }
                 }
                 catch (GridServiceNotFoundException | ClusterTopologyCheckedException e) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 22d5716..e73d292 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -59,7 +59,6 @@ import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.managers.deployment.GridDeployment;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
-import org.apache.ignite.internal.processors.cache.CachePeekModes;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.GridConcurrentFactory;
 import org.apache.ignite.internal.util.GridSpinReadWriteLock;
@@ -82,6 +81,7 @@ import static org.apache.ignite.internal.GridTopic.TOPIC_JOB_SIBLINGS;
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK;
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK_CANCEL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBJ_ID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TASK_NAME;
@@ -533,8 +533,6 @@ public class GridTaskProcessor extends GridProcessorAdapter {
         else
             taskClsName = taskCls != null ? taskCls.getName() : taskName;
 
-        ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
-
         // Get values from thread-local context.
         Map<GridTaskThreadContextKey, Object> map = thCtx.get();
 
@@ -544,6 +542,9 @@ public class GridTaskProcessor extends GridProcessorAdapter {
             // Reset thread-local context.
             thCtx.set(null);
 
+        if (map.get(TC_SKIP_AUTH) == null)
+            ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
+
         Long timeout = (Long)map.get(TC_TIMEOUT);
 
         long timeout0 = timeout == null || timeout == 0 ? Long.MAX_VALUE : timeout;

http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
index 2ec63df..f0e56c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
@@ -37,5 +37,8 @@ public enum GridTaskThreadContextKey {
     TC_SUBJ_ID,
 
     /** IO manager policy. */
-    TC_IO_POLICY
+    TC_IO_POLICY,
+
+    /** Skip authorization for the task. */
+    TC_SKIP_AUTH
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
index 3ed91e8..a82373b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
@@ -89,7 +89,7 @@ public class IgniteComputeTopologyExceptionTest extends GridCommonAbstractTest {
             },
             nodes,
             false,
-            0);
+            0, false);
 
         try {
             fut.get();


[07/12] ignite git commit: IGNITE-5113: Implemented basic distributed/local kmeans clusterization algorithm.

Posted by sb...@apache.org.
IGNITE-5113: Implemented basic distributed/local kmeans clusterization algorithm.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b04b5800
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b04b5800
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b04b5800

Branch: refs/heads/ignite-5075
Commit: b04b58005d32debc3ae54dd3b03289fad1a590bd
Parents: d0bddfa
Author: Artem Malykh <am...@gridgain.com>
Authored: Fri May 19 19:40:12 2017 +0300
Committer: agura <ag...@apache.org>
Committed: Wed May 31 19:54:48 2017 +0300

----------------------------------------------------------------------
 modules/ml/pom.xml                              |  13 +
 .../main/java/org/apache/ignite/ml/Model.java   |  39 +++
 .../ml/clustering/BaseKMeansClusterer.java      |  98 ++++++
 .../apache/ignite/ml/clustering/Clusterer.java  |  32 ++
 .../ml/clustering/ClusterizationModel.java      |  29 ++
 .../clustering/KMeansDistributedClusterer.java  | 298 +++++++++++++++++++
 .../ml/clustering/KMeansLocalClusterer.java     | 174 +++++++++++
 .../ignite/ml/clustering/KMeansModel.java       |  79 +++++
 .../ignite/ml/clustering/WeightedClusterer.java |  38 +++
 .../ignite/ml/clustering/package-info.java      |  22 ++
 .../apache/ignite/ml/math/DistanceMeasure.java  |  39 +++
 .../ignite/ml/math/EuclideanDistance.java       |  48 +++
 .../org/apache/ignite/ml/math/MathUtils.java    |  31 ++
 .../java/org/apache/ignite/ml/math/Matrix.java  |   2 +
 .../apache/ignite/ml/math/StorageConstants.java |   3 +
 .../org/apache/ignite/ml/math/VectorUtils.java  |  41 +++
 .../math/exceptions/ConvergenceException.java   |  48 +++
 .../exceptions/MathArithmeticException.java     |   6 +-
 .../exceptions/MathIllegalNumberException.java  |  51 ++++
 .../exceptions/MathIllegalStateException.java   |  49 +++
 .../exceptions/NumberIsTooSmallException.java   |  79 +++++
 .../ignite/ml/math/functions/Functions.java     |  39 +++
 .../apache/ignite/ml/math/impls/CacheUtils.java |  47 +++
 .../ml/math/impls/matrix/AbstractMatrix.java    |   2 +
 .../impls/matrix/SparseDistributedMatrix.java   |   2 +-
 .../impls/storage/matrix/MapWrapperStorage.java |  93 ++++++
 .../vector/SparseLocalOnHeapVectorStorage.java  |  32 ++
 .../ml/math/impls/vector/DelegatingVector.java  |   5 +
 .../ml/math/impls/vector/MapWrapperVector.java  |  32 ++
 .../ml/math/impls/vector/SparseLocalVector.java |   9 +
 .../ignite/ml/math/impls/vector/VectorView.java |   1 +
 .../ignite/ml/math/statistics/Variance.java     |  53 ++++
 .../org/apache/ignite/ml/math/util/MapUtil.java |  38 +++
 .../apache/ignite/ml/math/util/MatrixUtil.java  |  38 ++-
 .../org/apache/ignite/ml/IgniteMLTestSuite.java |   4 +-
 .../ml/clustering/ClusteringTesetSuite.java     |  15 +
 .../KMeansDistributedClustererTest.java         | 184 ++++++++++++
 .../ml/clustering/KMeansLocalClustererTest.java |  46 +++
 .../apache/ignite/ml/clustering/KMeansUtil.java |  33 ++
 .../ignite/ml/math/MathImplLocalTestSuite.java  |   1 +
 40 files changed, 1885 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/pom.xml
----------------------------------------------------------------------
diff --git a/modules/ml/pom.xml b/modules/ml/pom.xml
index 1df5bb6..75ebf1b 100644
--- a/modules/ml/pom.xml
+++ b/modules/ml/pom.xml
@@ -85,6 +85,19 @@
             <version>${spring.version}</version>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-rng-core</artifactId>
+            <version>1.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-rng-simple</artifactId>
+            <version>1.0</version>
+        </dependency>
+
     </dependencies>
 
     <build>

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/Model.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/Model.java b/modules/ml/src/main/java/org/apache/ignite/ml/Model.java
new file mode 100644
index 0000000..e4d2117
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/Model.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml;
+
+import java.util.function.BiFunction;
+
+
+/** Basic interface for all models. */
+@FunctionalInterface
+public interface Model<T, V> {
+    /** Predict a result for value. */
+    public V predict(T val);
+
+    /**
+     * Combines this model with other model via specified combiner
+     *
+     * @param other Other model.
+     * @param combiner Combiner.
+     * @return Combination of models.
+     */
+    public default <X, W> Model<T, X> combine(Model<T, W> other, BiFunction<V, W, X> combiner) {
+        return v -> combiner.apply(predict(v), other.predict(v));
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java
new file mode 100644
index 0000000..a6acb8e
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.util.List;
+
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.ml.math.DistanceMeasure;
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.exceptions.ConvergenceException;
+import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
+import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
+
+/**
+ * This class is partly based on the corresponding class from Apache Common Math lib.
+ */
+public abstract class BaseKMeansClusterer<T extends Matrix> implements Clusterer<T, KMeansModel> {
+    /** The distance measure to use. */
+    private DistanceMeasure measure;
+
+    /**
+     * Build a new clusterer with the given {@link DistanceMeasure}.
+     *
+     * @param measure the distance measure to use
+     */
+    protected BaseKMeansClusterer(final DistanceMeasure measure) {
+        this.measure = measure;
+    }
+
+    /**
+     * Perform a cluster analysis on the given set of points.
+     *
+     * @param points the set of points
+     * @return a {@link List} of clusters
+     * @throws MathIllegalArgumentException if points are null or the number of data points is not compatible with this
+     * clusterer
+     * @throws ConvergenceException if the algorithm has not yet converged after the maximum number of iterations has
+     * been exceeded
+     */
+    public abstract KMeansModel cluster(T points, int k)
+        throws MathIllegalArgumentException, ConvergenceException;
+
+    /**
+     * Returns the {@link DistanceMeasure} instance used by this clusterer.
+     *
+     * @return the distance measure
+     */
+    public DistanceMeasure getDistanceMeasure() {
+        return measure;
+    }
+
+    /**
+     * Calculates the distance between two vectors.
+     * with the configured {@link DistanceMeasure}.
+     *
+     * @return the distance between the two clusterables
+     */
+    protected double distance(final Vector v1, final Vector v2) {
+        return measure.compute(v1, v2);
+    }
+
+    /**
+     * Find the closest cluster center index and distance to it from a given point.
+     *
+     * @param centers Centers to look in.
+     * @param pnt Point.
+     */
+    protected IgniteBiTuple<Integer, Double> findClosest(Vector[] centers, Vector pnt) {
+        double bestDistance = Double.POSITIVE_INFINITY;
+        int bestInd = 0;
+
+        for (int i = 0; i < centers.length; i++) {
+            double dist = distance(centers[i], pnt);
+            if (dist < bestDistance) {
+                bestDistance = dist;
+                bestInd = i;
+            }
+        }
+
+        return new IgniteBiTuple<>(bestInd, bestDistance);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java
new file mode 100644
index 0000000..f03dc95
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import org.apache.ignite.ml.Model;
+
+/**
+ * Base interface for clusterers.
+ */
+public interface Clusterer<P, M extends Model> {
+    /** Cluster given points set into k clusters.
+     *
+     * @param points Points set.
+     * @param k Clusters count.
+     */
+    public M cluster(P points, int k);
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/ClusterizationModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/ClusterizationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/ClusterizationModel.java
new file mode 100644
index 0000000..99afec5
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/ClusterizationModel.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import org.apache.ignite.ml.Model;
+
+/** Base interface for all clusterization models. */
+public interface ClusterizationModel<P, V> extends Model<P, V> {
+    /** Gets the clusters count. */
+    public int clustersCount();
+
+    /** Get cluster centers. */
+    public P[] centers();
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
new file mode 100644
index 0000000..09317d6
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.ml.math.*;
+import org.apache.ignite.ml.math.exceptions.ConvergenceException;
+import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
+import org.apache.ignite.ml.math.functions.Functions;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.impls.CacheUtils;
+import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
+import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
+import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
+import org.apache.ignite.ml.math.util.MapUtil;
+import org.apache.ignite.ml.math.util.MatrixUtil;
+
+import javax.cache.Cache;
+
+import static org.apache.ignite.ml.math.impls.CacheUtils.*;
+import static org.apache.ignite.ml.math.util.MatrixUtil.localCopyOf;
+
+/**
+ * Clustering algorithm based on Bahmani et al. paper and Apache Spark class with corresponding functionality.
+ *
+ * @see <a href="http://theory.stanford.edu/~sergei/papers/vldb12-kmpar.pdf">Scalable K-Means++(wikipedia)</a>
+ */
+public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistributedMatrix> {
+    /** */
+    private final int maxIterations;
+
+    /** */
+    private Random rnd;
+
+    /** */
+    private int initSteps;
+
+    /** */
+    private long seed;
+
+    /** */
+    private double epsilon = 1e-4;
+
+    /** */
+    public KMeansDistributedClusterer(DistanceMeasure measure, int initSteps, int maxIterations, Long seed) {
+        super(measure);
+        this.initSteps = initSteps;
+
+        this.seed = seed != null ? seed : new Random().nextLong();
+
+        this.maxIterations = maxIterations;
+        rnd = new Random(this.seed);
+    }
+
+    /** */
+    @Override public KMeansModel cluster(SparseDistributedMatrix points, int k) throws
+            MathIllegalArgumentException, ConvergenceException {
+        SparseDistributedMatrix pointsCp = (SparseDistributedMatrix) points.like(points.rowSize(), points.columnSize());
+
+        // TODO: this copy is very ineffective, just for POC. Immutability of data should be guaranteed by other methods
+        // such as logical locks for example.
+        pointsCp.assign(points);
+
+        Vector[] centers = initClusterCenters(pointsCp, k);
+
+        boolean converged = false;
+        int iteration = 0;
+        int dim = pointsCp.viewRow(0).size();
+        IgniteUuid uid = pointsCp.getUUID();
+
+        // Execute iterations of Lloyd's algorithm until converged
+        while (iteration < maxIterations && !converged) {
+            SumsAndCounts stats = getSumsAndCounts(centers, dim, uid);
+
+            converged = true;
+
+            for (Integer ind : stats.sums.keySet()) {
+                Vector massCenter = stats.sums.get(ind).times(1.0 / stats.counts.get(ind));
+
+                if (converged && distance(massCenter, centers[ind]) > epsilon * epsilon)
+                    converged = false;
+
+                centers[ind] = massCenter;
+            }
+
+            iteration++;
+        }
+
+        pointsCp.destroy();
+
+        return new KMeansModel(centers, getDistanceMeasure());
+    }
+
+    /** Initialize cluster centers. */
+    private Vector[] initClusterCenters(SparseDistributedMatrix points, int k) {
+        // Initialize empty centers and point costs.
+        int ptsCount = points.rowSize();
+
+        // Initialize the first center to a random point.
+        Vector sample = localCopyOf(points.viewRow(rnd.nextInt(ptsCount)));
+
+        List<Vector> centers = new ArrayList<>();
+        List<Vector> newCenters = new ArrayList<>();
+        newCenters.add(sample);
+        centers.add(sample);
+
+        final ConcurrentHashMap<Integer, Double> costs = new ConcurrentHashMap<>();
+
+        // On each step, sample 2 * k points on average with probability proportional
+        // to their squared distance from the centers. Note that only distances between points
+        // and new centers are computed in each iteration.
+        int step = 0;
+        IgniteUuid uid = points.getUUID();
+
+        while (step < initSteps) {
+            // We assume here that costs can fit into memory of one node.
+            ConcurrentHashMap<Integer, Double> newCosts = getNewCosts(points, newCenters);
+
+            // Merge costs with new costs.
+            for (Integer ind : newCosts.keySet())
+                costs.merge(ind, newCosts.get(ind), Math::min);
+
+            double sumCosts = costs.values().stream().mapToDouble(Double::valueOf).sum();
+
+            newCenters = getNewCenters(k, costs, uid, sumCosts);
+            centers.addAll(newCenters);
+
+            step++;
+        }
+
+        List<Vector> distinctCenters = centers.stream().distinct().collect(Collectors.toList());
+
+        if (distinctCenters.size() <= k)
+            return distinctCenters.toArray(new Vector[] {});
+        else {
+            // Finally, we might have a set of more than k distinct candidate centers; weight each
+            // candidate by the number of points in the dataset mapping to it and run a local k-means++
+            // on the weighted centers to pick k of them
+            ConcurrentHashMap<Integer, Integer> centerInd2Weight = weightCenters(uid, distinctCenters);
+
+            List<Double> weights = new ArrayList<>(centerInd2Weight.size());
+
+            for (int i = 0; i < distinctCenters.size(); i++)
+                weights.add(i, Double.valueOf(centerInd2Weight.getOrDefault(i, 0)));
+
+            DenseLocalOnHeapMatrix dCenters = MatrixUtil.fromList(distinctCenters, true);
+
+            return new KMeansLocalClusterer(getDistanceMeasure(), 30, seed).cluster(dCenters, k, weights).centers();
+        }
+    }
+
+    /** */
+    private List<Vector> getNewCenters(int k, ConcurrentHashMap<Integer, Double> costs, IgniteUuid uid, double sumCosts) {
+        return distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME,
+            (IgniteBiFunction<Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>>,
+                List<Vector>,
+                List<Vector>>)(vectorWithIndex, list) -> {
+                Integer ind = vectorWithIndex.getKey().get1();
+
+                double prob = costs.get(ind) * 2.0 * k / sumCosts;
+
+                if (new Random(seed ^ ind).nextDouble() < prob)
+                    list.add(VectorUtils.fromMap(vectorWithIndex.getValue(), false));
+
+                return list;
+            },
+            key -> key.get2().equals(uid),
+            (list1, list2) -> {
+                list1.addAll(list2);
+                return list1;
+            },
+            new ArrayList<>()
+        );
+    }
+
+    /** */
+    private ConcurrentHashMap<Integer, Double> getNewCosts(SparseDistributedMatrix points, List<Vector> newCenters) {
+        return distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME,
+            (IgniteBiFunction<Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, ConcurrentHashMap<Integer, Double>>,
+                ConcurrentHashMap<Integer, Double>,
+                ConcurrentHashMap<Integer, Double>>)(vectorWithIndex, map) -> {
+                for (Vector center : newCenters)
+                    map.merge(vectorWithIndex.getKey().get1(), distance(vectorWithIndex.getValue(), center), Functions.MIN);
+
+                return map;
+            },
+            key -> key.get2().equals(points.getUUID()),
+            (map1, map2) -> {
+                map1.putAll(map2);
+                return map1;
+            }, new ConcurrentHashMap<>());
+    }
+
+    /** */
+    private ConcurrentHashMap<Integer, Integer> weightCenters(IgniteUuid uid, List<Vector> distinctCenters) {
+        return distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME,
+            (IgniteBiFunction<Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>>,
+                ConcurrentHashMap<Integer, Integer>,
+                ConcurrentHashMap<Integer, Integer>>)(vectorWithIndex, countMap) -> {
+                Integer resInd = -1;
+                Double resDist = Double.POSITIVE_INFINITY;
+
+                int i = 0;
+                for (Vector cent : distinctCenters) {
+                    double curDist = distance(vectorWithIndex.getValue(), cent);
+
+                    if (resDist > curDist) {
+                        resDist = curDist;
+                        resInd = i;
+                    }
+
+                    i++;
+                }
+
+                countMap.compute(resInd, (ind, v) -> v != null ? v + 1 : 1);
+                return countMap;
+            },
+            key -> key.get2().equals(uid),
+            (map1, map2) -> MapUtil.mergeMaps(map1, map2, (integer, integer2) -> integer2 + integer,
+                ConcurrentHashMap::new),
+            new ConcurrentHashMap<>());
+    }
+
+    /** */
+    private double distance(Map<Integer, Double> vecMap, Vector vector) {
+        return distance(VectorUtils.fromMap(vecMap, false), vector);
+    }
+
+    /** */
+    private SumsAndCounts getSumsAndCounts(Vector[] centers, int dim, IgniteUuid uid) {
+        return CacheUtils.distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME,
+                (IgniteBiFunction<Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>>, SumsAndCounts, SumsAndCounts>)(entry, counts) -> {
+                    Map<Integer, Double> vec = entry.getValue();
+
+                    IgniteBiTuple<Integer, Double> closest = findClosest(centers, VectorUtils.fromMap(vec, false));
+                    int bestCenterIdx = closest.get1();
+
+                    counts.totalCost += closest.get2();
+                    counts.sums.putIfAbsent(bestCenterIdx, VectorUtils.zeroes(dim));
+
+                    counts.sums.compute(bestCenterIdx,
+                            (IgniteBiFunction<Integer, Vector, Vector>)(ind, v) -> v.plus(VectorUtils.fromMap(vec, false)));
+
+                    counts.counts.merge(bestCenterIdx, 1,
+                            (IgniteBiFunction<Integer, Integer, Integer>)(i1, i2) -> i1 + i2);
+
+                    return counts;
+                },
+                key -> key.get2().equals(uid),
+                SumsAndCounts::merge, new SumsAndCounts()
+        );
+    }
+
+    /** Service class used for statistics. */
+    private static class SumsAndCounts {
+        /** */
+        public double totalCost;
+
+        /** */
+        public ConcurrentHashMap<Integer, Vector> sums = new ConcurrentHashMap<>();
+
+        /** Count of points closest to the center with a given index. */
+        public ConcurrentHashMap<Integer, Integer> counts = new ConcurrentHashMap<>();
+
+        /** Merge current */
+        public SumsAndCounts merge(SumsAndCounts other) {
+            this.totalCost += totalCost;
+            MapUtil.mergeMaps(sums, other.sums, Vector::plus, ConcurrentHashMap::new);
+            MapUtil.mergeMaps(counts, other.counts, (i1, i2) -> i1 + i2, ConcurrentHashMap::new);
+            return this;
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java
new file mode 100644
index 0000000..c98b818
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.ignite.internal.util.GridArgumentCheck;
+import org.apache.ignite.ml.math.*;
+import org.apache.ignite.ml.math.exceptions.ConvergenceException;
+import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
+import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
+
+import static org.apache.ignite.ml.math.util.MatrixUtil.localCopyOf;
+
+/**
+ * Perform clusterization on local data.
+ * This class is based on Apache Spark class with corresponding functionality.
+ */
+public class KMeansLocalClusterer extends BaseKMeansClusterer<DenseLocalOnHeapMatrix> implements
+    WeightedClusterer<DenseLocalOnHeapMatrix, KMeansModel> {
+    /** */
+    private int maxIterations;
+
+    /** */
+    private Random rand;
+
+    /**
+     * Build a new clusterer with the given {@link DistanceMeasure}.
+     *
+     * @param measure Distance measure to use.
+     * @param maxIterations maximal number of iterations.
+     * @param seed Seed used in random parts of algorithm.
+     */
+    public KMeansLocalClusterer(DistanceMeasure measure, int maxIterations, Long seed) {
+        super(measure);
+        this.maxIterations = maxIterations;
+        rand = seed != null ? new Random(seed) : new Random();
+    }
+
+    /** {@inheritDoc} */
+    @Override public KMeansModel cluster(
+        DenseLocalOnHeapMatrix points, int k) throws MathIllegalArgumentException, ConvergenceException {
+        List<Double> ones = new ArrayList<>(Collections.nCopies(points.rowSize(), 1.0));
+        return cluster(points, k, ones);
+    }
+
+    /** {@inheritDoc} */
+    @Override public KMeansModel cluster(DenseLocalOnHeapMatrix points, int k,
+        List<Double> weights) throws MathIllegalArgumentException, ConvergenceException {
+
+        GridArgumentCheck.notNull(points, "points");
+
+        int dim = points.columnSize();
+        Vector[] centers = new Vector[k];
+
+        centers[0] = pickWeighted(points, weights);
+
+        Vector costs = points.foldRows(row -> distance(row,
+            centers[0]));
+
+        for (int i = 0; i < k; i++) {
+            double weightedSum = weightedSum(costs, weights);
+
+            double r = rand.nextDouble() * weightedSum;
+            double s = 0.0;
+            int j = 0;
+
+            while (j < points.rowSize() && s < r) {
+                s += weights.get(j) * costs.get(j);
+                j++;
+            }
+
+            if (j == 0)
+                // TODO: Process this case more carefully
+                centers[i] = localCopyOf(points.viewRow(0));
+            else
+                centers[i] = localCopyOf(points.viewRow(j - 1));
+
+            for (int p = 0; p < points.rowSize(); p++)
+                costs.setX(p, Math.min(getDistanceMeasure().compute(localCopyOf(points.viewRow(p)), centers[i]),
+                        costs.get(p)));
+        }
+
+        int[] oldClosest = new int[points.rowSize()];
+        Arrays.fill(oldClosest, -1);
+        int iter = 0;
+        boolean moved = true;
+
+        while (moved && iter < maxIterations) {
+            moved = false;
+
+            double[] counts = new double[k];
+            Arrays.fill(counts, 0.0);
+            Vector[] sums = new Vector[k];
+
+            Arrays.fill(sums, VectorUtils.zeroes(dim));
+
+            int i = 0;
+
+            while (i < points.rowSize()) {
+                Vector p = localCopyOf(points.viewRow(i));
+
+                int ind = findClosest(centers, p).get1();
+                sums[ind] = sums[ind].plus(p.times(weights.get(i)));
+
+                counts[ind] += weights.get(i);
+                if (ind != oldClosest[i]) {
+                    moved = true;
+                    oldClosest[i] = ind;
+                }
+                i++;
+            }
+            // Update centers
+            int j = 0;
+            while (j < k) {
+                if (counts[j] == 0.0) {
+                    // Assign center to a random point
+                    centers[j] = points.viewRow(rand.nextInt(points.rowSize()));
+                } else {
+                    sums[j] = sums[j].times(1.0 / counts[j]);
+                    centers[j] = sums[j];
+                }
+                j++;
+            }
+            iter++;
+        }
+
+        return new KMeansModel(centers, getDistanceMeasure());
+    }
+
+    /** Pick a random vector with a probability proportional to the corresponding weight. */
+    private Vector pickWeighted(Matrix points, List<Double> weights) {
+        double r = rand.nextDouble() * weights.stream().mapToDouble(Double::valueOf).sum();
+
+        int i = 0;
+        double curWeight = 0.0;
+
+        while (i < points.rowSize() && curWeight < r) {
+            curWeight += weights.get(i);
+            i += 1;
+        }
+
+        return localCopyOf(points.viewRow(i - 1));
+    }
+
+    /** Get a weighted sum of a vector v. */
+    private double weightedSum(Vector v, List<Double> weights) {
+        double res = 0.0;
+
+        for (int i = 0; i < v.size(); i++)
+            res += v.getX(i) * weights.get(i);
+
+        return res;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java
new file mode 100644
index 0000000..6584273
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.util.Arrays;
+import org.apache.ignite.ml.math.DistanceMeasure;
+import org.apache.ignite.ml.math.Vector;
+
+/**
+ * This class encapsulates result of clusterization.
+ */
+public class KMeansModel implements ClusterizationModel<Vector, Integer> {
+    /** Centers of clusters. */
+    private Vector[] centers;
+
+    /** Distance measure. */
+    private DistanceMeasure distance;
+
+    /**
+     * Construct KMeans model with given centers and distance measure.
+     *
+     * @param centers Centers.
+     * @param distance Distance measure.
+     */
+    KMeansModel(Vector[] centers, DistanceMeasure distance) {
+        this.centers = centers;
+        this.distance = distance;
+    }
+
+    /** Distance measure used while clusterization */
+    public DistanceMeasure distanceMeasure() {
+        return distance;
+    }
+
+    /** Count of centers in clusterization. */
+    @Override public int clustersCount() {
+        return centers.length;
+    }
+
+    /** Get centers of clusters. */
+    @Override public Vector[] centers() {
+        return Arrays.copyOf(centers, centers.length);
+    }
+
+    /**
+     * Predict closest center index for a given vector.
+     *
+     * @param vec Vector.
+     */
+    public Integer predict(Vector vec) {
+        int res = -1;
+        double minDist = Double.POSITIVE_INFINITY;
+
+        for (int i = 0; i < centers.length; i++) {
+            double curDist = distance.compute(centers[i], vec);
+            if (curDist < minDist) {
+                minDist = curDist;
+                res = i;
+            }
+        }
+
+        return res;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/WeightedClusterer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/WeightedClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/WeightedClusterer.java
new file mode 100644
index 0000000..55fb359
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/WeightedClusterer.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.util.List;
+import org.apache.ignite.ml.Model;
+import org.apache.ignite.ml.math.exceptions.ConvergenceException;
+import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
+
+/**
+ * Support of clusterization with given weights.
+ */
+public interface WeightedClusterer<P, M extends Model> extends Clusterer<P, M> {
+    /**
+     * Perform clusterization of given points weighted by given weights.
+     *
+     * @param points Points.
+     * @param k count of centers.
+     * @param weights Weights.
+     */
+    public KMeansModel cluster(P points, int k, List<Double> weights) throws
+        MathIllegalArgumentException, ConvergenceException;
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/package-info.java
new file mode 100644
index 0000000..e83084a
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains clustering algorithms.
+ */
+package org.apache.ignite.ml.clustering;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java
new file mode 100644
index 0000000..0fd74ac
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.math;
+
+import java.io.Externalizable;
+
+import org.apache.ignite.ml.math.exceptions.CardinalityException;
+
+/**
+ * This class is based on the corresponding class from Apache Common Math lib.
+ * Interface for distance measures of n-dimensional vectors.
+ */
+public interface DistanceMeasure extends Externalizable {
+    /**
+     * Compute the distance between two n-dimensional vectors.
+     * <p>
+     * The two vectors are required to have the same dimension.
+     *
+     * @param a the first vector
+     * @param b the second vector
+     * @return the distance between the two vectors
+     * @throws CardinalityException if the array lengths differ.
+     */
+    double compute(Vector a, Vector b) throws CardinalityException;
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java
new file mode 100644
index 0000000..b748ac5
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.math;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.ml.math.exceptions.CardinalityException;
+import org.apache.ignite.ml.math.util.MatrixUtil;
+
+/**
+ * Calculates the L<sub>2</sub> (Euclidean) distance between two points.
+ */
+public class EuclideanDistance implements DistanceMeasure {
+    /** Serializable version identifier. */
+    private static final long serialVersionUID = 1717556319784040040L;
+
+    /** {@inheritDoc} */
+    @Override
+    public double compute(Vector a, Vector b)
+    throws CardinalityException {
+        return MatrixUtil.localCopyOf(a).minus(b).kNorm(2.0);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeExternal(ObjectOutput out) throws IOException {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+        // No-op
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/MathUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/MathUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/MathUtils.java
new file mode 100644
index 0000000..c2164ab
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/MathUtils.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math;
+
+import org.apache.ignite.ml.math.exceptions.NullArgumentException;
+
+/**
+ * Miscellaneous utility functions.
+ */
+public final class MathUtils {
+    public static void checkNotNull(Object o)
+        throws NullArgumentException {
+        if (o == null)
+            throw new NullArgumentException();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java
index 2cf4e63..db822e6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.ml.math;
 
 import java.io.Externalizable;
+import java.util.function.BiFunction;
+import java.util.function.BinaryOperator;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.ml.math.exceptions.CardinalityException;
 import org.apache.ignite.ml.math.exceptions.IndexException;

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/StorageConstants.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/StorageConstants.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/StorageConstants.java
index ec2ee65..38d42ba 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/StorageConstants.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/StorageConstants.java
@@ -33,6 +33,9 @@ public interface StorageConstants {
     /** Storage mode optimized for column access. */
     public static final int COLUMN_STORAGE_MODE = 2002;
 
+    /** Storage mode is unknown. */
+    public static final int UNKNOWN_STORAGE_MODE = 3001;
+
     /**
      * @param mode Access mode to verify.
      */

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java
new file mode 100644
index 0000000..f41a5fe
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math;
+
+import java.util.Map;
+import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.math.impls.vector.MapWrapperVector;
+import org.apache.ignite.ml.math.impls.vector.SparseLocalVector;
+
+public class VectorUtils {
+    /** Create new vector like given vector initialized by zeroes. */
+    public static Vector zeroesLike(Vector v) {
+        return v.like(v.size()).assign(0.0);
+    }
+
+    /** Create new */
+    public static DenseLocalOnHeapVector zeroes(int n) {
+        return (DenseLocalOnHeapVector) new DenseLocalOnHeapVector(n).assign(0.0);
+    }
+
+    /** */
+    public static Vector fromMap(Map<Integer, Double> value, boolean copy) {
+        return new MapWrapperVector(value);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/ConvergenceException.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/ConvergenceException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/ConvergenceException.java
new file mode 100644
index 0000000..2cf0bcf
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/ConvergenceException.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.math.exceptions;
+
+/**
+ * This class is based on the corresponding class from Apache Common Math lib.
+ * Error thrown when a numerical computation can not be performed because the
+ * numerical result failed to converge to a finite value.
+ */
+public class ConvergenceException extends MathIllegalStateException {
+    /** Serializable version Id. */
+    private static final long serialVersionUID = 4330003017885151975L;
+
+    /** */
+    private static final String CONVERGENCE_FAILED = "convergence failed";
+
+    /**
+     * Construct the exception.
+     */
+    public ConvergenceException() {
+        this(CONVERGENCE_FAILED);
+    }
+
+    /**
+     * Construct the exception with a specific context and arguments.
+     *
+     * @param msg Message pattern providing the specific context of
+     * the error.
+     * @param args Arguments.
+     */
+    public ConvergenceException(String msg, Object ... args) {
+        super(msg, args);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathArithmeticException.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathArithmeticException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathArithmeticException.java
index f48f3c5..ccd019c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathArithmeticException.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathArithmeticException.java
@@ -14,14 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.ignite.ml.math.exceptions;
 
 /**
  * This class is based on the corresponding class from Apache Common Math lib.
  * Base class for arithmetic exceptions.
- * It is used for all the exceptions that have the semantics of the standard
- * {@link ArithmeticException}, but must also provide a localized
- * message.
  */
 public class MathArithmeticException extends MathRuntimeException {
     /** Serializable version Id. */
@@ -31,7 +29,7 @@ public class MathArithmeticException extends MathRuntimeException {
      * Default constructor.
      */
     public MathArithmeticException() {
-        this("arithmetic exception");
+        this("Arithmetic exception.");
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalNumberException.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalNumberException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalNumberException.java
new file mode 100644
index 0000000..2e7280b
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalNumberException.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.math.exceptions;
+
+/**
+ * This class is based on the corresponding class from Apache Common Math lib.
+ * Base class for exceptions raised by a wrong number.
+ * This class is not intended to be instantiated directly: it should serve
+ * as a base class to create all the exceptions that are raised because some
+ * precondition is violated by a number argument.
+ */
+public class MathIllegalNumberException extends MathIllegalArgumentException {
+    /** Serializable version Id. */
+    private static final long serialVersionUID = -7447085893598031110L;
+
+    /** Requested. */
+    private final Number argument;
+
+    /**
+     * Construct an exception.
+     *
+     * @param msg Localizable pattern.
+     * @param wrong Wrong number.
+     * @param arguments Arguments.
+     */
+    protected MathIllegalNumberException(String msg, Number wrong, Object... arguments) {
+        super(msg, wrong, arguments);
+        argument = wrong;
+    }
+
+    /**
+     * @return the requested value.
+     */
+    public Number getArgument() {
+        return argument;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalStateException.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalStateException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalStateException.java
new file mode 100644
index 0000000..13ef5ca
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalStateException.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.math.exceptions;
+
+/**
+ * This class is based on the corresponding class from Apache Common Math lib.
+ * Base class for all exceptions that signal that the process
+ * throwing the exception is in a state that does not comply with
+ * the set of states that it is designed to be in.
+ */
+public class MathIllegalStateException extends MathRuntimeException {
+    /** Serializable version Id. */
+    private static final long serialVersionUID = -6024911025449780478L;
+
+    /** */
+    private static final String ILLEGAL_STATE= "Illegal state.";
+
+    /**
+     * Simple constructor.
+     *
+     * @param msg Message pattern explaining the cause of the error.
+     * @param args Arguments.
+     */
+    public MathIllegalStateException(String msg, Object ... args) {
+        super(msg, args);
+    }
+
+    /**
+     * Default constructor.
+     */
+    public MathIllegalStateException() {
+        this(ILLEGAL_STATE);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/NumberIsTooSmallException.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/NumberIsTooSmallException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/NumberIsTooSmallException.java
new file mode 100644
index 0000000..7427592
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/NumberIsTooSmallException.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.math.exceptions;
+
+/**
+ * Exception to be thrown when a number is too small.
+ */
+public class NumberIsTooSmallException extends MathIllegalNumberException {
+    /** */
+    private static final String NUMBER_TOO_SMALL = "Number {1} is smaller than the minimum ({2}).";
+
+    /** */
+    private static final String NUMBER_TOO_SMALL_BOUND_EXCLUDED = "Number {1} is smaller than, or equal to, the minimum ({2}).";
+
+    /** Serializable version Id. */
+    private static final long serialVersionUID = -6100997100383932834L;
+    /**
+     * Higher bound.
+     */
+    private final Number min;
+    /**
+     * Whether the maximum is included in the allowed range.
+     */
+    private final boolean boundIsAllowed;
+
+    /**
+     * Construct the exception.
+     *
+     * @param wrong Value that is smaller than the minimum.
+     * @param min Minimum.
+     * @param boundIsAllowed Whether {@code min} is included in the allowed range.
+     */
+    public NumberIsTooSmallException(Number wrong, Number min, boolean boundIsAllowed) {
+        this(boundIsAllowed ? NUMBER_TOO_SMALL : NUMBER_TOO_SMALL_BOUND_EXCLUDED,
+            wrong, min, boundIsAllowed);
+    }
+
+    /**
+     * Construct the exception with a specific context.
+     *
+     * @param msg Specific context pattern.
+     * @param wrong Value that is smaller than the minimum.
+     * @param min Minimum.
+     * @param boundIsAllowed Whether {@code min} is included in the allowed range.
+     */
+    public NumberIsTooSmallException(String msg, Number wrong, Number min, boolean boundIsAllowed) {
+        super(msg, wrong, min);
+        this.min = min;
+        this.boundIsAllowed = boundIsAllowed;
+    }
+
+    /**
+     * @return {@code true} if the minimum is included in the allowed range.
+     */
+    public boolean getBoundIsAllowed() {
+        return boundIsAllowed;
+    }
+
+    /**
+     * @return the minimum.
+     */
+    public Number getMin() {
+        return min;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java
index e86a5eb..22a453d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java
@@ -17,6 +17,10 @@
 
 package org.apache.ignite.ml.math.functions;
 
+import org.apache.ignite.lang.IgniteBiTuple;
+
+import java.util.List;
+
 /**
  * Compatibility with Apache Mahout.
  */
@@ -63,6 +67,9 @@ public final class Functions {
     /** Function that returns {@code a - b}. */
     public static final IgniteBiFunction<Double, Double, Double> MINUS = (a, b) -> a - b;
 
+    /** Function that returns {@code min(a, b)}. */
+    public static final IgniteBiFunction<Double, Double, Double> MIN = Math::min;
+
     /** Function that returns {@code abs(a - b)}. */
     public static final IgniteBiFunction<Double, Double, Double> MINUS_ABS = (a, b) -> Math.abs(a - b);
 
@@ -81,6 +88,38 @@ public final class Functions {
     /** Function that returns {@code a &lt; b ? -1 : a &gt; b ? 1 : 0}. */
     public static final IgniteBiFunction<Double, Double, Double> COMPARE = (a, b) -> a < b ? -1.0 : a > b ? 1.0 : 0.0;
 
+    /** */
+    public  static <A, B, C> IgniteFunction<B, C> curry(IgniteBiFunction<A, B, C> f, A a) {
+        return (IgniteFunction<B, C>)b -> f.apply(a, b);
+    }
+
+    /** */
+    public static <A, B extends Comparable<B>> IgniteBiTuple<Integer, A> argmin(List<A> args, IgniteFunction<A, B> f) {
+        A res = null;
+        B fRes = null;
+
+        if (!args.isEmpty()) {
+            res = args.iterator().next();
+            fRes = f.apply(res);
+        }
+
+        int resInd = 0;
+        int i = 0;
+
+        for (A arg : args) {
+            B curRes = f.apply(arg);
+
+            if (fRes.compareTo(curRes) > 0) {
+                res = arg;
+                resInd = i;
+                fRes = curRes;
+            }
+
+            i++;
+        }
+        return new IgniteBiTuple<>(resInd, res);
+    }
+
     /**
      * Function that returns {@code a + b}. {@code a} is a variable, {@code b} is fixed.
      *

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
index ace399b..836789b 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
@@ -20,6 +20,7 @@ package org.apache.ignite.ml.math.impls;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
+import java.util.function.BinaryOperator;
 import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
@@ -28,6 +29,7 @@ import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -38,6 +40,7 @@ import org.apache.ignite.ml.math.ValueMapper;
 import org.apache.ignite.ml.math.functions.IgniteBiFunction;
 import org.apache.ignite.ml.math.functions.IgniteConsumer;
 import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
 import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
 
 /**
@@ -380,6 +383,50 @@ public class CacheUtils {
         });
     }
 
+    public static <K, V, A> A distributedFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder,
+        IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, A zeroVal) {
+        return sparseFold(cacheName, folder, keyFilter, accumulator, zeroVal, null, null, 0 ,
+            false);
+    }
+
+    private static <K, V, A> A sparseFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder,
+        IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, A zeroVal, V defVal, K defKey, long defValCnt, boolean isNilpotent) {
+
+        A defRes = zeroVal;
+
+        if (!isNilpotent)
+            for (int i = 0; i < defValCnt; i++)
+                defRes = folder.apply(new CacheEntryImpl<>(defKey, defVal), defRes);
+
+        Collection<A> totalRes = bcast(cacheName, () -> {
+            Ignite ignite = Ignition.localIgnite();
+            IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
+
+            int partsCnt = ignite.affinity(cacheName).partitions();
+
+            // Use affinity in filter for ScanQuery. Otherwise we accept consumer in each node which is wrong.
+            Affinity affinity = ignite.affinity(cacheName);
+            ClusterNode localNode = ignite.cluster().localNode();
+
+            A a = zeroVal;
+
+            // Iterate over all partitions. Some of them will be stored on that local node.
+            for (int part = 0; part < partsCnt; part++) {
+                int p = part;
+
+                // Iterate over given partition.
+                // Query returns an empty cursor if this partition is not stored on this node.
+                for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part,
+                    (k, v) -> affinity.mapPartitionToNode(p) == localNode  && (keyFilter == null || keyFilter.apply(k)))))
+                    a = folder.apply(entry, a);
+            }
+
+            return a;
+        });
+        totalRes.add(defRes);
+        return totalRes.stream().reduce(zeroVal, accumulator);
+    }
+
     /**
      * @param cacheName Cache name.
      * @param run {@link Runnable} to broadcast to cache nodes for given cache name.

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
index d1d3904..106a425 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
@@ -24,6 +24,8 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Random;
+import java.util.function.BiFunction;
+import java.util.function.BinaryOperator;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.ml.math.Matrix;
 import org.apache.ignite.ml.math.MatrixStorage;

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
index 3e508bd..cebacc5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
@@ -156,7 +156,7 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo
     }
 
     /** */
-    private IgniteUuid getUUID(){
+    public IgniteUuid getUUID(){
         return ((SparseDistributedMatrixStorage) getStorage()).getUUID();
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java
new file mode 100644
index 0000000..13b8303
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java
@@ -0,0 +1,93 @@
+package org.apache.ignite.ml.math.impls.storage.matrix;
+
+import org.apache.ignite.internal.util.GridArgumentCheck;
+import org.apache.ignite.ml.math.VectorStorage;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Storage for wrapping given map.
+ */
+public class MapWrapperStorage implements VectorStorage {
+    /** Underlying map. */
+    Map<Integer, Double> data;
+
+    /** Vector size. */
+    int size;
+
+    /**
+     * Construct a wrapper around given map.
+     *
+     * @param map Map to wrap.
+     */
+    public MapWrapperStorage(Map<Integer, Double> map) {
+        Set<Integer> keys = map.keySet();
+
+        GridArgumentCheck.notEmpty(keys, "map");
+
+        Integer min = keys.stream().mapToInt(Integer::valueOf).min().getAsInt();
+        Integer max = keys.stream().mapToInt(Integer::valueOf).max().getAsInt();
+
+        assert min >= 0;
+
+        data = map;
+        size = (max - min) + 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size() {
+        return size;
+    }
+
+    /** {@inheritDoc} */
+    @Override public double get(int i) {
+        return data.getOrDefault(i, 0.0);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void set(int i, double v) {
+        if (v != 0.0)
+            data.put(i, v);
+        else if (data.containsKey(i))
+            data.remove(i);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeExternal(ObjectOutput out) throws IOException {
+        out.writeObject(data);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+        data = (Map<Integer, Double>) in.readObject();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isSequentialAccess() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isRandomAccess() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isDense() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isArrayBased() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isDistributed() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java
index 8400758..f07a16e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java
@@ -22,6 +22,7 @@ import it.unimi.dsi.fastutil.ints.Int2DoubleRBTreeMap;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.util.HashMap;
 import java.util.Map;
 import org.apache.ignite.ml.math.StorageConstants;
 import org.apache.ignite.ml.math.VectorStorage;
@@ -46,6 +47,37 @@ public class SparseLocalOnHeapVectorStorage implements VectorStorage, StorageCon
     }
 
     /**
+     *
+     * @param map
+     */
+    public SparseLocalOnHeapVectorStorage(Map<Integer, Double> map, boolean copy) {
+        assert map.size() > 0;
+
+        this.size = map.size();
+
+        if (map instanceof Int2DoubleRBTreeMap)
+            acsMode = SEQUENTIAL_ACCESS_MODE;
+        else
+            if (map instanceof Int2DoubleOpenHashMap)
+                acsMode = RANDOM_ACCESS_MODE;
+            else
+                acsMode = UNKNOWN_STORAGE_MODE;
+
+        if (copy)
+            switch (acsMode) {
+                case SEQUENTIAL_ACCESS_MODE:
+                    sto = new Int2DoubleRBTreeMap(map);
+                case RANDOM_ACCESS_MODE:
+                    sto = new Int2DoubleOpenHashMap(map);
+                    break;
+                default:
+                    sto = new HashMap<>(map);
+            }
+        else
+            sto = map;
+    }
+
+    /**
      * @param size Vector size.
      * @param acsMode Access mode.
      */

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java
index c868160..48fbd06 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java
@@ -59,6 +59,11 @@ public class DelegatingVector implements Vector {
         this.dlg = dlg;
     }
 
+    /** Get the delegating vector */
+    public Vector getVector() {
+        return dlg;
+    }
+
     /** {@inheritDoc} */
     @Override public void writeExternal(ObjectOutput out) throws IOException {
         out.writeObject(dlg);

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java
new file mode 100644
index 0000000..729a7ea
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java
@@ -0,0 +1,32 @@
+package org.apache.ignite.ml.math.impls.vector;
+
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.impls.storage.matrix.MapWrapperStorage;
+
+import java.util.Map;
+
+/**
+ * Vector wrapping a given map.
+ */
+public class MapWrapperVector extends AbstractVector {
+
+    /**
+     * Construct a vector wrapping given map.
+     *
+     * @param map Map to wrap.
+     */
+    public MapWrapperVector(Map<Integer, Double> map) {
+        setStorage(new MapWrapperStorage(map));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Vector like(int crd) {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Matrix likeMatrix(int rows, int cols) {
+        throw new UnsupportedOperationException();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java
index e188f70..be5d0f6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.ml.math.impls.vector;
 
+import java.util.Map;
 import org.apache.ignite.ml.math.Matrix;
 import org.apache.ignite.ml.math.StorageConstants;
 import org.apache.ignite.ml.math.Vector;
@@ -35,6 +36,14 @@ public class SparseLocalVector extends AbstractVector implements StorageConstant
     }
 
     /**
+     * @param map Underlying map.
+     * @param copy Should given map be copied.
+     */
+    public SparseLocalVector(Map<Integer, Double> map, boolean copy) {
+        setStorage(new SparseLocalOnHeapVectorStorage(map, copy));
+    }
+
+    /**
      * @param size Vector size.
      * @param acsMode Vector elements access mode.
      */

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java
index f3bd4dd..0988059 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java
@@ -60,6 +60,7 @@ public class VectorView extends AbstractVector {
 
     /** {@inheritDoc} */
     @Override public Vector copy() {
+        // TODO: revise this
         DelegateVectorStorage sto = storage();
 
         return new VectorView(sto.delegate(), sto.offset(), sto.length());

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java
new file mode 100644
index 0000000..e406b5b
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.statistics;
+
+/**
+ * This class encapsulates calculating variance.
+ */
+public class Variance {
+    /** */
+    private double mean;
+
+    /** */
+    private long n;
+
+    /** */
+    private double m2;
+
+    public Variance() {
+        mean = 0;
+        n = 0;
+        m2 = 0;
+    }
+
+    /** */
+    public Variance update(Double x) {
+        n++;
+        double delta = x - mean;
+        mean += delta / n;
+        double delta2 = x - mean;
+        m2 += delta * delta2;
+        return this;
+    }
+
+    /** */
+    public double getResult() {
+        return m2;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java
new file mode 100644
index 0000000..6c25f0e
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.util;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.function.BinaryOperator;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ *
+ */
+public class MapUtil {
+    /** */
+    public static <K, V, M extends Map<K, V>> M mergeMaps(M m1, M m2, BinaryOperator<V> op, Supplier<M> mapSupplier) {
+        return Stream.of(m1, m2)
+            .map(Map::entrySet)
+            .flatMap(Collection::stream)
+            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, op, mapSupplier));
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java
index a06b773..5ef7176 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java
@@ -17,6 +17,9 @@
 
 package org.apache.ignite.ml.math.util;
 
+import java.util.List;
+
+import org.apache.ignite.internal.util.GridArgumentCheck;
 import org.apache.ignite.ml.math.Matrix;
 import org.apache.ignite.ml.math.Vector;
 import org.apache.ignite.ml.math.impls.matrix.CacheMatrix;
@@ -24,7 +27,6 @@ import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
 import org.apache.ignite.ml.math.impls.matrix.MatrixView;
 import org.apache.ignite.ml.math.impls.matrix.PivotedMatrixView;
 import org.apache.ignite.ml.math.impls.matrix.RandomMatrix;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 
 /**
@@ -117,6 +119,38 @@ public class MatrixUtil {
     /** */
     private static boolean isCopyLikeSupport(Matrix matrix) {
         return matrix instanceof RandomMatrix || matrix instanceof MatrixView || matrix instanceof CacheMatrix ||
-            matrix instanceof PivotedMatrixView || matrix instanceof SparseDistributedMatrix;
+            matrix instanceof PivotedMatrixView;
+    }
+
+    /** */
+    public static DenseLocalOnHeapMatrix fromList(List<Vector> vecs, boolean entriesAreRows) {
+        GridArgumentCheck.notEmpty(vecs, "vecs");
+
+        int dim = vecs.get(0).size();
+        int vecsSize = vecs.size();
+
+        DenseLocalOnHeapMatrix res = new DenseLocalOnHeapMatrix(entriesAreRows ? vecsSize : dim,
+            entriesAreRows ? dim : vecsSize);
+
+        for (int i = 0; i < vecsSize; i++) {
+            for (int j = 0; j < dim; j++) {
+                int r = entriesAreRows ? i : j;
+                int c = entriesAreRows ? j : i;
+
+                res.setX(r, c, vecs.get(i).get(j));
+            }
+        }
+
+        return res;
+    }
+
+    /** TODO: rewrite in a more optimal way. */
+    public static DenseLocalOnHeapVector localCopyOf(Vector vec) {
+        DenseLocalOnHeapVector res = new DenseLocalOnHeapVector(vec.size());
+
+        for (int i = 0; i < vec.size(); i++)
+            res.setX(i, vec.getX(i));
+
+        return res;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
index 92aa7db..dea3edf 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.ml;
 
+import org.apache.ignite.ml.clustering.ClusteringTesetSuite;
 import org.apache.ignite.ml.math.MathImplMainTestSuite;
 import org.apache.ignite.ml.regressions.RegressionsTestSuite;
 import org.junit.runner.RunWith;
@@ -28,7 +29,8 @@ import org.junit.runners.Suite;
 @RunWith(Suite.class)
 @Suite.SuiteClasses({
     MathImplMainTestSuite.class,
-    RegressionsTestSuite.class
+    RegressionsTestSuite.class,
+    ClusteringTesetSuite.class
 })
 public class IgniteMLTestSuite {
     // No-op.

http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTesetSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTesetSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTesetSuite.java
new file mode 100644
index 0000000..122d0c1
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTesetSuite.java
@@ -0,0 +1,15 @@
+package org.apache.ignite.ml.clustering;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/**
+ * Test suite for all tests located in org.apache.ignite.ml.clustering package.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+        KMeansDistributedClustererTest.class,
+        KMeansLocalClustererTest.class
+})
+public class ClusteringTesetSuite {
+}


[12/12] ignite git commit: Merge remote-tracking branch 'remotes/origin/master' into ignite-5075

Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-5075

# Conflicts:
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/eeba611c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/eeba611c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/eeba611c

Branch: refs/heads/ignite-5075
Commit: eeba611c80e7b63670baf7d4ba05b90d0b86169f
Parents: 5ef6900
Author: sboikov <sb...@gridgain.com>
Authored: Thu Jun 1 11:52:49 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Jun 1 11:52:49 2017 +0300

----------------------------------------------------------------------
 .../ignite/internal/processors/cache/GridCacheProcessor.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/eeba611c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index afb578b..5990c42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -2066,8 +2066,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
         Throwable err
     ) {
         ExchangeActions actions = new ExchangeActions(){
-            @Override List<DynamicCacheChangeRequest> closeRequests(UUID nodeId) {
-                return Collections.singletonList(act.request());
+            @Override List<ActionData> closeRequests(UUID nodeId) {
+                return Collections.singletonList(act);
             }
         };
 
@@ -2135,7 +2135,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
                 }
 
                 if (forceClose)
-                    completeCacheStartFuture(req, false, err);
+                    completeCacheStartFuture(req.request(), false, err);
             }
         }
     }


[08/12] ignite git commit: IGNITE-5365 Build release artifacts once and use them for all editions

Posted by sb...@apache.org.
IGNITE-5365 Build release artifacts once and use them for all editions


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/54f499ce
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/54f499ce
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/54f499ce

Branch: refs/heads/ignite-5075
Commit: 54f499cece98c9cade28f3e0a9cfe89323c64ef0
Parents: b04b580
Author: oleg-ostanin <oo...@gridgain.com>
Authored: Wed May 31 20:02:00 2017 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Wed May 31 20:02:00 2017 +0300

----------------------------------------------------------------------
 DEVNOTES.txt                                    |  74 ++++--
 assembly/dependencies-fabric-lgpl.xml           |   2 +
 assembly/dependencies-fabric.xml                |   2 +
 assembly/release-fabric-lgpl.xml                |   1 +
 assembly/release-fabric.xml                     |   1 +
 assembly/release-yardstick.xml                  |  98 +++++++
 .../Apache.Ignite.Core.nuspec                   |   5 +-
 modules/platforms/dotnet/build.ps1              |   5 +-
 modules/web-console/web-agent/pom.xml           |   8 +
 parent/pom.xml                                  |   8 +-
 pom.xml                                         | 254 ++++++-------------
 11 files changed, 246 insertions(+), 212 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/DEVNOTES.txt
----------------------------------------------------------------------
diff --git a/DEVNOTES.txt b/DEVNOTES.txt
index d3c162e..8a689b8 100644
--- a/DEVNOTES.txt
+++ b/DEVNOTES.txt
@@ -1,36 +1,63 @@
 Ignite Fabric Maven Build Instructions
 ======================================
-Without LGPL dependencies (default):
-  mvn clean package -DskipTests
+1) Optional: build Apache Ignite.NET as described at modules/platforms/dotnet/DEVNOTES.txt.
 
-Without LGPL dependencies and Scala 2.10:
-  mvn clean package -DskipTests -Dscala-2.10
+2) Compile and package:
 
-With LGPL dependencies:
-  mvn clean package -DskipTests -Prelease,lgpl -Dignite.edition=fabric-lgpl
+  mvn clean package -Pall-java,all-scala,licenses -DskipTests
 
-With LGPL dependencies and Scala 2.10:
-  mvn clean package -DskipTests -Prelease,lgpl -Dignite.edition=fabric-lgpl -Dscala-2.10
+  or if you have built Apache Ignite.NET on the first step use following command:
+  (Note that 'doxygen' should be installed before running this command.)
 
-With LGPL dependencies and Scala 2.10 and Ignite-Yardstick:
-  mvn clean package -DskipTests -Prelease,lgpl,yardstick -Dignite.edition=fabric-lgpl -Dscala-2.10
+  mvn clean package -Pall-java,all-scala,licenses -DskipTests -DclientDocs
 
-With Apache Ignite.NET:
-  Build Apache Ignite.NET as described at modules/platforms/dotnet/DEVNOTES.txt.
+3) Javadoc generation (optional):
 
-  Then build Ignite Fabric with Apache Ignite.NET:
-  mvn clean package -DskipTests -DclientDocs
+  mvn initialize -Pjavadoc
 
-  Note that 'doxygen' should be installed before running this command.
+4) Assembly Apache Ignite fabric:
+
+  mvn initialize -Prelease
 
 Look for apache-ignite-fabric-<version>-bin.zip in ./target/bin directory.
 
+Ignite Fabric with LGPL Maven Build Instructions
+======================================
+1) Optional: build Apache Ignite.NET as described at modules/platforms/dotnet/DEVNOTES.txt.
+
+2) Compile and package:
+
+  mvn clean package -Pall-java,all-scala,licenses -DskipTests
+
+  or if you have built Apache Ignite.NET on the first step use following command:
+  (Note that 'doxygen' should be installed before running this command.)
+
+  mvn clean package -Pall-java,all-scala,licenses -DskipTests -DclientDocs
+
+3) Javadoc generation with LGPL (optional):
+
+  mvn initialize -Pjavadoc,lgpl
+
+4) Assembly Apache Ignite fabric with LGPL dependencies:
+
+  mvn initialize -Prelease,lgpl -Dignite.edition=fabric-lgpl
+
+Look for apache-ignite-fabric-lgpl-<version>-bin.zip in ./target/bin directory.
+
 Ignite Hadoop Accelerator Maven Build Instructions
 ============================================
-mvn clean package -DskipTests -Dignite.edition=hadoop [-Dhadoop.version=X.X.X] [-Dspark.version=x.y.z]
+1) Compile and package:
+
+    mvn clean package -Pall-java,all-scala,licenses -DskipTests
 
 Use 'hadoop.version' parameter to build Ignite against a specific Hadoop version.
 Use 'spark.version' parameter to build ignite-spark module for a specific Spark version. Version should be >= 2.0.0.
+For example:
+
+    mvn clean package -Pall-java,all-scala,licenses -DskipTests -Dhadoop.version=2.4.2 -Dspark.version=2.1.1
+
+2) Assembly Hadoop Accelerator:
+  mvn initialize -Prelease -Dignite.edition=hadoop
 
 Look for apache-ignite-hadoop-<version>-bin.zip in ./target/bin directory. Resulting binary
 assembly will also include integration module for Apache Spark.
@@ -66,12 +93,17 @@ Maven release plugin release:prepare goal can be used to make release tag.
 
 Deploy Ignite release candidate to maven repository and dev-svn, make tag:
 
-   Following command deploys Ignite to maven repository, prepares sources and fabric edition binaries.
-      mvn deploy -P apache-release,gpg,release,scala,lgpl,deploy-ignite-site -Dignite.edition=fabric -DskipTests -B
+   1) Deploy Ignite to maven repository, prepares sources and fabric edition binaries.
+      mvn deploy -Papache-release,gpg,all-java,all-scala,licenses,deploy-ignite-site -Dignite.edition=fabric -DskipTests
+
+   2) Javadoc generation:
+      mvn initialize -Pjavadoc
+
+   3) Assembly Apache Ignite Fabric:
+      mvn initialize -Prelease
 
-   In case you want to release both fabric and hadoop editions you have to build hadoop edition first using command
-      mvn package -P apache-release,gpg,release,scala,lgpl -Dignite.edition=hadoop -DskipTests -B
-   save /target/bin/*.zip, make "mvn clean" and restore saved files before deploying fabric.
+   4) Assembly Hadoop Accelerator:
+      mvn initialize -Prelease -Dignite.edition=hadoop
 
    Binary artifact name can be changed by setting additional property -Dignite.zip.pattern. Binary artifact will be
    created inside /target/bin folder when release profile is used.

http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/assembly/dependencies-fabric-lgpl.xml
----------------------------------------------------------------------
diff --git a/assembly/dependencies-fabric-lgpl.xml b/assembly/dependencies-fabric-lgpl.xml
index 4cff146..e2fac3c 100644
--- a/assembly/dependencies-fabric-lgpl.xml
+++ b/assembly/dependencies-fabric-lgpl.xml
@@ -131,6 +131,8 @@
                 <exclude>org.apache.ignite:ignite-websphere-test</exclude>
                 <exclude>org.apache.ignite:ignite-cassandra</exclude>
                 <exclude>org.apache.ignite:ignite-yardstick</exclude>
+                <exclude>org.apache.ignite:ignite-benchmarks</exclude>
+                <exclude>org.apache.ignite:ignite-web-agent</exclude>
             </excludes>
             <sources>
                 <includeModuleDirectory>true</includeModuleDirectory>

http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/assembly/dependencies-fabric.xml
----------------------------------------------------------------------
diff --git a/assembly/dependencies-fabric.xml b/assembly/dependencies-fabric.xml
index be68d0a..6c4101e 100644
--- a/assembly/dependencies-fabric.xml
+++ b/assembly/dependencies-fabric.xml
@@ -136,6 +136,8 @@
                 <exclude>org.apache.ignite:ignite-websphere-test</exclude>
                 <exclude>org.apache.ignite:ignite-cassandra</exclude>
                 <exclude>org.apache.ignite:ignite-yardstick</exclude>
+                <exclude>org.apache.ignite:ignite-benchmarks</exclude>
+                <exclude>org.apache.ignite:ignite-web-agent</exclude>
             </excludes>
             <sources>
                 <includeModuleDirectory>true</includeModuleDirectory>

http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/assembly/release-fabric-lgpl.xml
----------------------------------------------------------------------
diff --git a/assembly/release-fabric-lgpl.xml b/assembly/release-fabric-lgpl.xml
index 1766558..ff4d8c4 100644
--- a/assembly/release-fabric-lgpl.xml
+++ b/assembly/release-fabric-lgpl.xml
@@ -32,6 +32,7 @@
     <componentDescriptors>
         <componentDescriptor>release-base.xml</componentDescriptor>
         <componentDescriptor>release-fabric-base.xml</componentDescriptor>
+        <componentDescriptor>release-yardstick.xml</componentDescriptor>
     </componentDescriptors>
 
     <files>

http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/assembly/release-fabric.xml
----------------------------------------------------------------------
diff --git a/assembly/release-fabric.xml b/assembly/release-fabric.xml
index ffde8ec..7536d4e 100644
--- a/assembly/release-fabric.xml
+++ b/assembly/release-fabric.xml
@@ -32,6 +32,7 @@
     <componentDescriptors>
         <componentDescriptor>release-base.xml</componentDescriptor>
         <componentDescriptor>release-fabric-base.xml</componentDescriptor>
+        <componentDescriptor>release-yardstick.xml</componentDescriptor>
     </componentDescriptors>
 
     <files>

http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/assembly/release-yardstick.xml
----------------------------------------------------------------------
diff --git a/assembly/release-yardstick.xml b/assembly/release-yardstick.xml
new file mode 100644
index 0000000..0bd6f41
--- /dev/null
+++ b/assembly/release-yardstick.xml
@@ -0,0 +1,98 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~  Licensed to the Apache Software Foundation (ASF) under one or more
+  ~  contributor license agreements.  See the NOTICE file distributed with
+  ~  this work for additional information regarding copyright ownership.
+  ~  The ASF licenses this file to You under the Apache License, Version 2.0
+  ~  (the "License"); you may not use this file except in compliance with
+  ~  the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~  Unless required by applicable law or agreed to in writing, software
+  ~  distributed under the License is distributed on an "AS IS" BASIS,
+  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~  See the License for the specific language governing permissions and
+  ~  limitations under the License.
+  -->
+
+<component xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/component/1.1.2"
+           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+           xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/component/1.1.2
+           http://maven.apache.org/xsd/component-1.1.2.xsd">
+    <files>
+        <file>
+            <source>modules/yardstick/pom-standalone.xml</source>
+            <outputDirectory>benchmarks/sources</outputDirectory>
+            <destName>pom.xml</destName>
+        </file>
+
+        <file>
+            <source>modules/yardstick/DEVNOTES-standalone.txt</source>
+            <outputDirectory>benchmarks/sources</outputDirectory>
+            <destName>DEVNOTES.txt</destName>
+        </file>
+
+        <file>
+            <source>modules/yardstick/README.txt</source>
+            <outputDirectory>benchmarks</outputDirectory>
+            <destName>README.txt</destName>
+        </file>
+    </files>
+
+    <fileSets>
+        <fileSet>
+            <directory>modules/yardstick/target/assembly/bin</directory>
+            <outputDirectory>benchmarks/bin</outputDirectory>
+        </fileSet>
+
+        <fileSet>
+            <directory>modules/yardstick/target/assembly/config</directory>
+            <outputDirectory>benchmarks/config</outputDirectory>
+            <includes>
+                <include>benchmark.properties</include>
+                <include>benchmark-remote.properties</include>
+                <include>benchmark-sample.properties</include>
+                <include>benchmark-remote-sample.properties</include>
+                <include>benchmark-multicast.properties</include>
+                <include>ignite-base-config.xml</include>
+                <include>ignite-localhost-config.xml</include>
+                <include>ignite-remote-config.xml</include>
+                <include>ignite-multicast-config.xml</include>
+            </includes>
+        </fileSet>
+
+        <fileSet>
+            <directory>modules/yardstick/target/assembly/libs</directory>
+            <outputDirectory>benchmarks/libs</outputDirectory>
+            <excludes>
+                <exclude>junit-*.jar</exclude>
+                <exclude>ignite-apache-license-gen-*.jar</exclude>
+                <exclude>hamcrest-core-*.jar</exclude>
+                <exclude>tools-*.jar</exclude>
+            </excludes>
+        </fileSet>
+
+        <fileSet>
+            <directory>modules/yardstick/src</directory>
+            <outputDirectory>benchmarks/sources/src</outputDirectory>
+        </fileSet>
+
+        <fileSet>
+            <directory>modules/yardstick/target/assembly/config</directory>
+            <outputDirectory>benchmarks/sources/config</outputDirectory>
+            <includes>
+                <include>benchmark.properties</include>
+                <include>benchmark-remote.properties</include>
+                <include>benchmark-sample.properties</include>
+                <include>benchmark-remote-sample.properties</include>
+                <include>benchmark-multicast.properties</include>
+                <include>ignite-base-config.xml</include>
+                <include>ignite-localhost-config.xml</include>
+                <include>ignite-remote-config.xml</include>
+                <include>ignite-multicast-config.xml</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+</component>

http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
index 7dc9f9a..9049526 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
@@ -62,10 +62,7 @@ More info: https://apacheignite-net.readme.io/
             Library files (jars) should not be included in project, so that NuGet package restore works properly.
             We keep jars in NuGet dir and copy them over in PostBuild event.
         -->
-        <file src="..\..\..\..\target\release-package\libs\*.jar" target="Libs" />
-        <file src="..\..\..\..\target\release-package\libs\ignite-spring\*.jar" target="Libs" />
-        <file src="..\..\..\..\target\release-package\libs\ignite-indexing\*.jar" target="Libs" />
-        <file src="..\..\..\..\target\release-package\libs\licenses\*.*" target="Licenses" />
+        <file src="..\bin\Libs\*.jar" target="Libs" />
     
         <!-- LINQPad samples -->
         <file src="NuGet\LINQPad\*.*" target="linqpad-samples" />

http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/modules/platforms/dotnet/build.ps1
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/build.ps1 b/modules/platforms/dotnet/build.ps1
index b8e6a37..222076a 100644
--- a/modules/platforms/dotnet/build.ps1
+++ b/modules/platforms/dotnet/build.ps1
@@ -112,10 +112,7 @@ if (!$skipJava) {
     $libsDir = "$PSScriptRoot\bin\Libs"
     mkdir -Force $libsDir; del -Force $libsDir\*.*
     
-    copy -Force target\release-package\libs\*.jar $libsDir
-    copy -Force target\release-package\libs\ignite-spring\*.jar $libsDir
-    copy -Force target\release-package\libs\ignite-indexing\*.jar $libsDir
-    copy -Force target\release-package\libs\licenses\*.jar $libsDir
+    ls modules\indexing\target,modules\core\target,modules\spring\target*.jar -recurse -include "ignite-core*","ignite-indexing*","ignite-shmem*","ignite-spring*","lucene*","h2*","cache-api*","commons-*","spring*" -exclude "*-sources*","*-javadoc*","*-tests*" | % { copy -Force $_ $libsDir }
 
     # Restore directory
     cd $PSScriptRoot

http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/modules/web-console/web-agent/pom.xml
----------------------------------------------------------------------
diff --git a/modules/web-console/web-agent/pom.xml b/modules/web-console/web-agent/pom.xml
index 2f3969e..e715fbf 100644
--- a/modules/web-console/web-agent/pom.xml
+++ b/modules/web-console/web-agent/pom.xml
@@ -183,6 +183,14 @@
 
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-deploy-plugin</artifactId>
                 <configuration>
                     <skip>true</skip>

http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index 88d4da8..270ece4 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -998,12 +998,8 @@
         </profile>
 
         <profile>
-            <id>release</id>
-            <activation>
-                <property>
-                    <name>!skipDefault</name><!--activeByDefault-->
-                </property>
-            </activation>
+            <id>licenses</id>
+
             <dependencies>
                 <dependency>
                     <groupId>org.apache.ignite</groupId>

http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7f24613..7fa3eef 100644
--- a/pom.xml
+++ b/pom.xml
@@ -164,29 +164,64 @@
         </profile>
 
         <profile>
-            <id>dev-libs</id>
-            <activation>
-                <activeByDefault>true</activeByDefault>
-            </activation>
+            <id>javadoc</id>
             <build>
                 <plugins>
                     <plugin>
                         <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-javadoc-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>core-javadoc</id>
+                                <goals>
+                                    <goal>aggregate</goal>
+                                </goals>
+                                <phase>validate</phase>
+                                <configuration>
+                                    <reportOutputDirectory>${basedir}/target/javadoc</reportOutputDirectory>
+                                    <destDir>core</destDir>
+                                    <stylesheetfile>${basedir}/assembly/docfiles/javadoc.css</stylesheetfile>
+                                    <subpackages>org.apache.ignite -exclude org.apache.ignite.client:org.apache.ignite.codegen:org.apache.ignite.examples:org.apache.ignite.internal:org.apache.ignite.schema:org.apache.ignite.tests:org.apache.ignite.tools:org.apache.ignite.util:org.apache.ignite.spi.discovery.tcp.messages:org.apache.ignite.spi.discovery.tcp.internal:org.apache.ignite.spi.deployment.uri.scanners:org.apache.ignite.spi.deployment.uri.tasks:org.apache.ignite.yardstick:org.apache.ignite.webtest</subpackages>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-antrun-plugin</artifactId>
                         <version>1.7</version>
                         <inherited>false</inherited>
+                        <dependencies>
+                            <dependency>
+                                <groupId>org.apache.ignite</groupId>
+                                <artifactId>ignite-tools</artifactId>
+                                <version>${project.version}</version>
+                            </dependency>
+                        </dependencies>
                         <executions>
                             <execution>
-                                <id>dev-libs</id>
+                                <id>javadoc-postprocessing-new</id>
                                 <goals>
                                     <goal>run</goal>
                                 </goals>
-                                <phase>package</phase>
+                                <phase>initialize</phase>
                                 <configuration>
                                     <target>
-                                        <copy todir="${basedir}/libs">
-                                            <fileset dir="${basedir}/target/release-package/libs" />
+                                        <copy todir="${basedir}/target/javadoc/core">
+                                            <fileset dir="assembly/docfiles">
+                                                <include name="img/**" />
+                                                <include name="*.js" />
+                                            </fileset>
                                         </copy>
+
+                                            <taskdef name="doctask" classname="org.apache.ignite.tools.ant.beautifier.GridJavadocAntTask" />
+
+                                            <doctask css="dotted" dir="target/javadoc/core">
+                                                <include name="**/*.html" />
+                                                <exclude name="overview-frame.html" />
+                                                <exclude name="allclasses-frame.html" />
+                                                <exclude name="**/class-use/*" />
+                                            </doctask>
                                     </target>
                                 </configuration>
                             </execution>
@@ -208,11 +243,6 @@
 
         <profile>
             <id>release</id>
-            <activation>
-                <property>
-                    <name>!skipDefault</name><!--activeByDefault-->
-                </property>
-            </activation>
             <build>
                 <plugins>
                     <plugin><!-- skipping generation of dependencies licenses
@@ -235,26 +265,6 @@
 
                     <plugin>
                         <groupId>org.apache.maven.plugins</groupId>
-                        <artifactId>maven-javadoc-plugin</artifactId>
-                        <executions>
-                            <execution>
-                                <id>core-javadoc</id>
-                                <goals>
-                                    <goal>aggregate</goal>
-                                </goals>
-                                <phase>process-test-classes</phase>
-                                <configuration>
-                                    <reportOutputDirectory>${basedir}/target/javadoc</reportOutputDirectory>
-                                    <destDir>core</destDir>
-                                    <stylesheetfile>${basedir}/assembly/docfiles/javadoc.css</stylesheetfile>
-                                    <subpackages>org.apache.ignite -exclude org.apache.ignite.client:org.apache.ignite.codegen:org.apache.ignite.examples:org.apache.ignite.internal:org.apache.ignite.schema:org.apache.ignite.tests:org.apache.ignite.tools:org.apache.ignite.util:org.apache.ignite.spi.discovery.tcp.messages:org.apache.ignite.spi.discovery.tcp.internal:org.apache.ignite.spi.deployment.uri.scanners:org.apache.ignite.spi.deployment.uri.tasks:org.apache.ignite.yardstick:org.apache.ignite.webtest</subpackages>
-                                </configuration>
-                            </execution>
-                        </executions>
-                    </plugin>
-
-                    <plugin>
-                        <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-antrun-plugin</artifactId>
                         <version>1.7</version>
                         <inherited>false</inherited>
@@ -267,44 +277,17 @@
                         </dependencies>
                         <executions>
                             <execution>
-                                <id>javadoc-postprocessing</id>
-                                <goals>
-                                    <goal>run</goal>
-                                </goals>
-                                <phase>prepare-package</phase>
-                                <configuration>
-                                    <target>
-                                        <copy todir="${basedir}/target/javadoc/core">
-                                            <fileset dir="assembly/docfiles">
-                                                <include name="img/**" />
-                                                <include name="*.js" />
-                                            </fileset>
-                                        </copy>
-
-                                        <taskdef name="doctask" classname="org.apache.ignite.tools.ant.beautifier.GridJavadocAntTask" />
-
-                                        <doctask css="dotted" dir="target/javadoc/core">
-                                            <include name="**/*.html" />
-                                            <exclude name="overview-frame.html" />
-                                            <exclude name="allclasses-frame.html" />
-                                            <exclude name="**/class-use/*" />
-                                        </doctask>
-                                    </target>
-                                </configuration>
-                            </execution>
-
-                            <execution>
                                 <id>release-postprocessing</id>
                                 <goals>
                                     <goal>run</goal>
                                 </goals>
-                                <phase>package</phase>
+                                <phase>initialize</phase>
                                 <configuration>
                                     <target>
                                         <replaceregexp byline="true">
                                             <regexp pattern="pushd &quot;%~dp0&quot;/\.\./\.\.(\s*&amp;::.+)?" />
                                             <substitution expression="pushd &quot;%~dp0&quot;/.." />
-                                            <fileset dir="${basedir}/target/release-package/bin">
+                                            <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
                                                 <include name="**/*.bat" />
                                             </fileset>
                                         </replaceregexp>
@@ -312,7 +295,7 @@
                                         <replaceregexp byline="true">
                                             <regexp pattern="IGNITE_HOME_TMP=&quot;\$\(dirname &quot;\$\{IGNITE_HOME_TMP\}&quot;\)&quot;(\s*#.*)?" />
                                             <substitution expression="" />
-                                            <fileset dir="${basedir}/target/release-package/bin">
+                                            <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
                                                 <include name="**/*.sh" />
                                             </fileset>
                                         </replaceregexp>
@@ -320,7 +303,7 @@
                                         <replaceregexp byline="true">
                                             <regexp pattern="set SCRIPTS_HOME=%IGNITE_HOME%\\bin(\s*&amp;::.*)?" />
                                             <substitution expression="set SCRIPTS_HOME=%IGNITE_HOME%\\\\bin" />
-                                            <fileset dir="${basedir}/target/release-package/bin">
+                                            <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
                                                 <include name="**/*.bat" />
                                             </fileset>
                                         </replaceregexp>
@@ -328,7 +311,7 @@
                                         <replaceregexp byline="true">
                                             <regexp pattern="SCRIPTS_HOME=&quot;\$\{IGNITE_HOME_TMP\}/bin&quot;(\s*#.*)?" />
                                             <substitution expression="SCRIPTS_HOME=&quot;$${IGNITE_HOME_TMP}/bin&quot;" />
-                                            <fileset dir="${basedir}/target/release-package/bin">
+                                            <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
                                                 <include name="**/*.sh" />
                                             </fileset>
                                         </replaceregexp>
@@ -336,7 +319,7 @@
                                         <replaceregexp byline="true">
                                             <regexp pattern="\. &quot;\$\{SCRIPTS_HOME\}&quot;/include/build-classpath.sh(\s*#.*)?" />
                                             <substitution expression="" />
-                                            <fileset dir="${basedir}/target/release-package/bin">
+                                            <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
                                                 <include name="**/*.sh" />
                                             </fileset>
                                         </replaceregexp>
@@ -344,7 +327,7 @@
                                         <replaceregexp byline="true">
                                             <regexp pattern="call &quot;%SCRIPTS_HOME%\\include\\build-classpath.bat&quot;(\s*&amp;::.*)?" />
                                             <substitution expression="" />
-                                            <fileset dir="${basedir}/target/release-package/bin">
+                                            <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
                                                 <include name="**/*.bat" />
                                             </fileset>
                                         </replaceregexp>
@@ -352,7 +335,7 @@
                                         <replaceregexp byline="true">
                                             <regexp pattern="ENABLE_ASSERTIONS=.*" />
                                             <substitution expression="ENABLE_ASSERTIONS=&quot;0&quot;" />
-                                            <fileset dir="${basedir}/target/release-package/bin">
+                                            <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
                                                 <include name="**/*.sh" />
                                             </fileset>
                                         </replaceregexp>
@@ -360,7 +343,7 @@
                                         <replaceregexp byline="true">
                                             <regexp pattern="ENABLE_ASSERTIONS=.*" />
                                             <substitution expression="ENABLE_ASSERTIONS=0" />
-                                            <fileset dir="${basedir}/target/release-package/bin">
+                                            <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
                                                 <include name="**/*.bat" />
                                             </fileset>
                                         </replaceregexp>
@@ -368,36 +351,41 @@
                                         <replaceregexp byline="true">
                                             <regexp pattern="-DIGNITE_UPDATE_NOTIFIER=false" />
                                             <substitution expression="" />
-                                            <fileset dir="${basedir}/target/release-package/bin">
+                                            <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
                                                 <include name="**/*.sh" />
                                                 <include name="**/*.bat" />
                                             </fileset>
                                         </replaceregexp>
 
-                                        <replaceregexp file="${basedir}/target/release-package/examples/pom.xml" byline="true">
+                                        <replaceregexp file="${basedir}/target/release-package-${ignite.edition}/examples/pom.xml" byline="true">
+                                            <regexp pattern="to_be_replaced_by_ignite_version" />
+                                            <substitution expression="${project.version}" />
+                                        </replaceregexp>
+
+                                        <replaceregexp file="${basedir}/target/release-package-${ignite.edition}/benchmarks/sources/pom.xml" byline="true">
                                             <regexp pattern="to_be_replaced_by_ignite_version" />
                                             <substitution expression="${project.version}" />
                                         </replaceregexp>
 
-                                        <chmod dir="${basedir}/target/release-package" perm="755" includes="**/*.sh" />
+                                        <chmod dir="${basedir}/target/release-package-${ignite.edition}" perm="755" includes="**/*.sh" />
 
                                         <!--
                                         Line ending bugfix for builds packaged in MS Windows machine to adjust Linux based
                                         end-of-line characters for Linux shell scripts
                                         -->
-                                        <fixcrlf srcdir="${basedir}/target/release-package" eol="lf" eof="remove">
+                                        <fixcrlf srcdir="${basedir}/target/release-package-${ignite.edition}" eol="lf" eof="remove">
                                             <include name="**/*.sh" />
                                         </fixcrlf>
 
                                         <zip destfile="${basedir}/target/bin/${ignite.zip.pattern}.zip" encoding="UTF-8">
-                                            <zipfileset dir="${basedir}/target/release-package" prefix="${ignite.zip.pattern}" filemode="755">
+                                            <zipfileset dir="${basedir}/target/release-package-${ignite.edition}" prefix="${ignite.zip.pattern}" filemode="755">
                                                 <include name="**/*.sh" />
                                                 <include name="**/configure" />
                                                 <include name="**/install-sh" />
                                                 <include name="**/compile" />
                                                 <include name="**/missing" />
                                             </zipfileset>
-                                            <zipfileset dir="${basedir}/target/release-package" prefix="${ignite.zip.pattern}">
+                                            <zipfileset dir="${basedir}/target/release-package-${ignite.edition}" prefix="${ignite.zip.pattern}">
                                                 <exclude name="**/*.sh" />
                                                 <exclude name="**/configure" />
                                                 <exclude name="**/install-sh" />
@@ -418,8 +406,8 @@
                         <inherited>false</inherited>
                         <executions>
                             <execution>
-                                <id>dependencies</id>
-                                <phase>prepare-package</phase>
+                                <id>ignite-dependencies</id>
+                                <phase>validate</phase>
                                 <goals>
                                     <goal>single</goal>
                                 </goals>
@@ -427,15 +415,15 @@
                                     <descriptors>
                                         <descriptor>${basedir}/assembly/dependencies-${ignite.edition}.xml</descriptor>
                                     </descriptors>
-                                    <outputDirectory>${basedir}/target/release-package</outputDirectory>
+                                    <outputDirectory>${basedir}/target/release-package-${ignite.edition}</outputDirectory>
                                     <finalName>libs</finalName>
                                     <appendAssemblyId>false</appendAssemblyId>
                                 </configuration>
                             </execution>
 
                             <execution>
-                                <id>release</id>
-                                <phase>prepare-package</phase>
+                                <id>ignite-release</id>
+                                <phase>validate</phase>
                                 <goals>
                                     <goal>single</goal>
                                 </goals>
@@ -443,14 +431,14 @@
                                     <descriptors>
                                         <descriptor>assembly/release-${ignite.edition}.xml</descriptor>
                                     </descriptors>
-                                    <finalName>release-package</finalName>
+                                    <finalName>release-package-${ignite.edition}</finalName>
                                     <appendAssemblyId>false</appendAssemblyId>
                                 </configuration>
                             </execution>
 
                             <execution>
                                 <id>dependencies-visor-console</id>
-                                <phase>prepare-package</phase>
+                                <phase>validate</phase>
                                 <goals>
                                     <goal>single</goal>
                                 </goals>
@@ -458,7 +446,7 @@
                                     <descriptors>
                                         <descriptor>assembly/dependencies-visor-console.xml</descriptor>
                                     </descriptors>
-                                    <outputDirectory>target/release-package/bin</outputDirectory>
+                                    <outputDirectory>target/release-package-${ignite.edition}/bin</outputDirectory>
                                     <finalName>include</finalName>
                                     <appendAssemblyId>false</appendAssemblyId>
                                 </configuration>
@@ -466,7 +454,7 @@
 
                             <execution>
                                 <id>scala-scripts</id>
-                                <phase>prepare-package</phase>
+                                <phase>validate</phase>
                                 <goals>
                                     <goal>single</goal>
                                 </goals>
@@ -474,7 +462,7 @@
                                     <descriptors>
                                         <descriptor>assembly/release-scala.xml</descriptor>
                                     </descriptors>
-                                    <outputDirectory>target/release-package</outputDirectory>
+                                    <outputDirectory>target/release-package-${ignite.edition}</outputDirectory>
                                     <finalName>bin</finalName>
                                     <appendAssemblyId>false</appendAssemblyId>
                                 </configuration>
@@ -498,94 +486,6 @@
             <modules>
                 <module>modules/yardstick</module>
             </modules>
-            <build>
-                <plugins>
-                    <plugin>
-                        <groupId>org.apache.maven.plugins</groupId>
-                        <artifactId>maven-antrun-plugin</artifactId>
-                        <version>1.7</version>
-                        <inherited>false</inherited>
-                        <dependencies>
-                            <dependency>
-                                <groupId>org.apache.ignite</groupId>
-                                <artifactId>ignite-tools</artifactId>
-                                <version>${project.version}</version>
-                            </dependency>
-                        </dependencies>
-                        <executions>
-                            <execution>
-                                <id>release-yardstick</id>
-                                <goals>
-                                    <goal>run</goal>
-                                </goals>
-                                <phase>prepare-package</phase>
-                                <configuration>
-                                    <target>
-                                        <mkdir dir="${basedir}/target/release-package/benchmarks" />
-
-                                        <copy todir="${basedir}/target/release-package/benchmarks/">
-                                            <fileset dir="${basedir}/modules/yardstick/target/assembly/" />
-                                        </copy>
-
-                                        <!--todo: only required jars should be exported to /benchmarks/libs during compilation-->
-                                        <delete>
-                                            <fileset dir="${basedir}/target/release-package/benchmarks/libs/">
-                                                <include name="junit-*.jar" />
-                                                <include name="ignite-apache-license-gen-*.jar" />
-                                                <include name="hamcrest-core-*.jar" />
-                                                <include name="tools-*.jar" />
-                                            </fileset>
-                                        </delete>
-
-                                        <!--todo: config structure should be refactored to be the same at "sources" and "released sources"-->
-                                        <delete>
-                                            <fileset dir="${basedir}/target/release-package/benchmarks/config/">
-                                                <include name="*.*" />
-                                                <exclude name="benchmark.properties" />
-                                                <exclude name="benchmark-remote.properties" />
-                                                <exclude name="benchmark-sample.properties" />
-                                                <exclude name="benchmark-remote-sample.properties" />
-                                                <exclude name="benchmark-multicast.properties" />
-                                                <exclude name="ignite-base-config.xml" />
-                                                <exclude name="ignite-localhost-config.xml" />
-                                                <exclude name="ignite-remote-config.xml" />
-                                                <exclude name="ignite-multicast-config.xml" />
-                                            </fileset>
-                                        </delete>
-
-                                        <mkdir dir="${basedir}/target/release-package/benchmarks/sources/src" />
-
-                                        <copy todir="${basedir}/target/release-package/benchmarks/sources/src/">
-                                            <fileset dir="${basedir}/modules/yardstick/src" />
-                                        </copy>
-
-                                        <mkdir dir="${basedir}/target/release-package/benchmarks/sources/config" />
-
-                                        <copy todir="${basedir}/target/release-package/benchmarks/sources/config/">
-                                            <fileset dir="${basedir}/target/release-package/benchmarks/config" />
-                                        </copy>
-
-                                        <copy file="${basedir}/modules/yardstick/pom-standalone.xml" tofile="${basedir}/target/release-package/benchmarks/sources/pom.xml" />
-
-                                        <replaceregexp byline="true">
-                                            <regexp pattern="to_be_replaced_by_ignite_version" />
-                                            <substitution expression="${project.version}" />
-                                            <fileset dir="${basedir}/target/release-package/benchmarks/sources/">
-                                                <include name="pom.xml" />
-                                            </fileset>
-                                        </replaceregexp>
-
-                                        <copy file="${basedir}/modules/yardstick/README.txt" tofile="${basedir}/target/release-package/benchmarks/README.txt" overwrite="true">
-                                        </copy>
-
-                                        <copy file="${basedir}/modules/yardstick/DEVNOTES-standalone.txt" tofile="${basedir}/target/release-package/benchmarks/sources/DEVNOTES.txt" />
-                                    </target>
-                                </configuration>
-                            </execution>
-                        </executions>
-                    </plugin>
-                </plugins>
-            </build>
         </profile>
 
         <profile>
@@ -658,7 +558,7 @@
                             </execution>
                             <execution>
                                 <id>source-release-assembly-ignite</id>
-                                <phase>package</phase>
+                                <phase>initialize</phase>
                                 <goals>
                                     <goal>single</goal>
                                 </goals>
@@ -681,7 +581,7 @@
                         <executions>
                             <execution>
                                 <id>attach-artifact</id><!-- allows to sign *bin.zip files -->
-                                <phase>package</phase>
+                                <phase>initialize</phase>
                                 <goals>
                                     <goal>run</goal>
                                 </goals>
@@ -744,7 +644,7 @@
                                 <goals>
                                     <goal>files</goal>
                                 </goals>
-                                <phase>install</phase>
+                                <phase>initialize</phase>
                             </execution>
                         </executions>
                         <configuration>
@@ -776,7 +676,7 @@
                                 <goals>
                                     <goal>run</goal>
                                 </goals>
-                                <phase>install</phase>
+                                <phase>initialize</phase>
                                 <configuration>
                                     <failOnError>false</failOnError>
                                     <target>


[04/12] ignite git commit: IGNITE-5154: Added RemoteFilter to C++ ContinuosQuery example.

Posted by sb...@apache.org.
IGNITE-5154: Added RemoteFilter to C++ ContinuosQuery example.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/41dcce7b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/41dcce7b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/41dcce7b

Branch: refs/heads/ignite-5075
Commit: 41dcce7b8adb86e5a053f62c658e36a9c67c35f5
Parents: abb9d94
Author: Igor Sapego <is...@gridgain.com>
Authored: Wed May 31 18:34:21 2017 +0300
Committer: Igor Sapego <is...@gridgain.com>
Committed: Wed May 31 18:34:21 2017 +0300

----------------------------------------------------------------------
 .../src/continuous_query_example.cpp            | 125 ++++++++++++++++++-
 1 file changed, 119 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/41dcce7b/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp b/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
index b08d4b8..2ca3163 100644
--- a/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
+++ b/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
@@ -33,14 +33,14 @@ using namespace examples;
 /** Cache name. */
 const char* CACHE_NAME = "cpp_cache_continuous_query";
 
-/*
+/**
  * Listener class.
  */
 template<typename K, typename V>
 class Listener : public event::CacheEntryEventListener<K, V>
 {
 public:
-    /*
+    /**
      * Default constructor.
      */
     Listener()
@@ -65,6 +65,110 @@ public:
     }
 };
 
+/**
+ * Range Filter. Only lets through keys from the specified range.
+ */
+template<typename K, typename V>
+struct RangeFilter : event::CacheEntryEventFilter<int32_t, std::string>
+{
+    /**
+     * Default constructor.
+     */
+    RangeFilter() :
+        rangeBegin(0),
+        rangeEnd(0)
+    {
+        // No-op.
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param from Range beginning. Inclusive.
+     * @param to Range end. Not inclusive.
+     */
+    RangeFilter(const K& from, const K& to) :
+        rangeBegin(from),
+        rangeEnd(to)
+    {
+        // No-op.
+    }
+
+    /**
+     * Destructor.
+     */
+    virtual ~RangeFilter()
+    {
+        // No-op.
+    }
+
+    /**
+     * Event callback.
+     *
+     * @param event Event.
+     * @return True if the event passes filter.
+     */
+    virtual bool Process(const CacheEntryEvent<K, V>& event)
+    {
+        return event.GetKey() >= rangeBegin && event.GetKey() < rangeEnd;
+    }
+
+    /** Beginning of the range. */
+    K rangeBegin;
+
+    /** End of the range. */
+    K rangeEnd;
+};
+
+namespace ignite
+{
+    namespace binary
+    {
+        template<>
+        struct BinaryType< RangeFilter<int32_t, std::string> >
+        {
+            static int32_t GetTypeId()
+            {
+                return GetBinaryStringHashCode("RangeFilter<int32_t,std::string>");
+            }
+
+            static void GetTypeName(std::string& dst)
+            {
+                dst = "RangeFilter<int32_t,std::string>";
+
+            }
+
+            static int32_t GetFieldId(const char* name)
+            {
+                return GetBinaryStringHashCode(name);
+            }
+
+            static bool IsNull(const RangeFilter<int32_t, std::string>&)
+            {
+                return false;
+            }
+
+            static void GetNull(RangeFilter<int32_t, std::string>& dst)
+            {
+                dst = RangeFilter<int32_t, std::string>();
+            }
+
+            static void Write(BinaryWriter& writer, const RangeFilter<int32_t, std::string>& obj)
+            {
+                writer.WriteInt32("rangeBegin", obj.rangeBegin);
+                writer.WriteInt32("rangeEnd", obj.rangeEnd);
+            }
+
+            static void Read(BinaryReader& reader, RangeFilter<int32_t, std::string>& dst)
+            {
+                dst.rangeBegin = reader.ReadInt32("rangeBegin");
+                dst.rangeEnd = reader.ReadInt32("rangeEnd");
+            }
+        };
+    }
+}
+
+
 int main()
 {
     IgniteConfiguration cfg;
@@ -80,6 +184,12 @@ int main()
         std::cout << ">>> Cache continuous query example started." << std::endl;
         std::cout << std::endl;
 
+        // Get binding.
+        IgniteBinding binding = ignite.GetBinding();
+
+        // Registering remote filter.
+        binding.RegisterCacheEntryEventFilter< RangeFilter<int32_t, std::string> >();
+
         // Get cache instance.
         Cache<int32_t, std::string> cache = ignite.GetOrCreateCache<int32_t, std::string>(CACHE_NAME);
 
@@ -97,17 +207,20 @@ int main()
         }
 
         // Declaring listener.
-        Listener<int, std::string> listener;
+        Listener<int32_t, std::string> listener;
+
+        // Declaring filter.
+        RangeFilter<int32_t, std::string> filter(keyCnt, keyCnt + 5);
 
         // Declaring continuous query.
-        continuous::ContinuousQuery<int, std::string> qry(MakeReference(listener));
+        continuous::ContinuousQuery<int32_t, std::string> qry(MakeReference(listener), MakeReference(filter));
 
         {
             // Continous query scope. Query is closed when scope is left.
-            continuous::ContinuousQueryHandle<int, std::string> handle = cache.QueryContinuous(qry);
+            continuous::ContinuousQueryHandle<int32_t, std::string> handle = cache.QueryContinuous(qry);
 
             // Add a few more keys and watch more query notifications.
-            for (int32_t i = keyCnt; i < keyCnt + 5; ++i)
+            for (int32_t i = keyCnt; i < keyCnt + 10; ++i)
             {
                 std::stringstream builder;
 


[09/12] ignite git commit: IGNITE-4724: SQL: fixed return type of AVG operation. This closes #1989.

Posted by sb...@apache.org.
IGNITE-4724: SQL: fixed return type of AVG operation. This closes #1989.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c4883113
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c4883113
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c4883113

Branch: refs/heads/ignite-5075
Commit: c4883113dd54a13ea68ed07548f838fde513f9f1
Parents: 54f499c
Author: Sergey Kalashnikov <sk...@gridgain.com>
Authored: Wed May 31 21:39:21 2017 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Wed May 31 21:39:21 2017 +0300

----------------------------------------------------------------------
 .../query/h2/sql/GridSqlQuerySplitter.java      |  22 +-
 ...ClientReconnectCacheQueriesFailoverTest.java |   2 +-
 .../query/IgniteSqlSplitterSelfTest.java        | 256 +++++++++++++++----
 3 files changed, 223 insertions(+), 57 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/c4883113/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 8fcd7dc..e2b82ab 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -46,6 +46,7 @@ import org.h2.command.Prepared;
 import org.h2.command.dml.Query;
 import org.h2.command.dml.SelectUnion;
 import org.h2.jdbc.JdbcPreparedStatement;
+import org.h2.value.Value;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.internal.processors.query.h2.opt.GridH2CollocationModel.isCollocated;
@@ -1836,12 +1837,12 @@ public class GridSqlQuerySplitter {
         /* Note Distinct aggregate can be performed only on reduce phase, so
            if query contains distinct aggregate then other aggregates must be processed the same way. */
         switch (agg.type()) {
-            case AVG: // SUM( AVG(CAST(x AS DOUBLE))*COUNT(x) )/SUM( COUNT(x) )  or  AVG(CAST( x AS DOUBLE))
+            case AVG: // SUM( AVG(CAST(x AS DOUBLE))*COUNT(x) )/SUM( COUNT(x) )  or  AVG(x)
                 if (hasDistinctAggregate) /* and has no collocated group by */ {
                     mapAgg = agg.child();
 
-                    rdcAgg = aggregate(agg.distinct(), agg.type()).resultType(GridSqlType.DOUBLE)
-                        .addChild(function(CAST).resultType(GridSqlType.DOUBLE).addChild(column(mapAggAlias.alias())));
+                    rdcAgg = aggregate(agg.distinct(), agg.type()).resultType(agg.resultType())
+                        .addChild(column(mapAggAlias.alias()));
                 }
                 else {
                     //-- COUNT(x) map
@@ -1868,7 +1869,13 @@ public class GridSqlQuerySplitter {
 
                     GridSqlElement sumDownRdc = aggregate(false, SUM).addChild(column(cntMapAggAlias));
 
-                    rdcAgg = op(GridSqlOperationType.DIVIDE, sumUpRdc, sumDownRdc);
+                    if (!isFractionalType(agg.resultType().type())) {
+                        sumUpRdc =  function(CAST).resultType(GridSqlType.BIGINT).addChild(sumUpRdc);
+                        sumDownRdc = function(CAST).resultType(GridSqlType.BIGINT).addChild(sumDownRdc);
+                    }
+
+                    rdcAgg = function(CAST).resultType(agg.resultType())
+                        .addChild(op(GridSqlOperationType.DIVIDE, sumUpRdc, sumDownRdc));
                 }
 
                 break;
@@ -1974,6 +1981,13 @@ public class GridSqlQuerySplitter {
     }
 
     /**
+     * @param type data type id
+     * @return true if given type is fractional
+     */
+    private static boolean isFractionalType(int type) {
+       return type == Value.DECIMAL || type == Value.FLOAT || type == Value.DOUBLE;
+    }
+    /**
      * Simplified tree-like model for a query.
      * - SELECT : All the children are list of joined query models in the FROM clause.
      * - UNION  : All the children are united left and right query models.

http://git-wip-us.apache.org/repos/asf/ignite/blob/c4883113/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientReconnectCacheQueriesFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientReconnectCacheQueriesFailoverTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientReconnectCacheQueriesFailoverTest.java
index 51620b1..39634cb 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientReconnectCacheQueriesFailoverTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientReconnectCacheQueriesFailoverTest.java
@@ -101,7 +101,7 @@ public class IgniteClientReconnectCacheQueriesFailoverTest extends IgniteClientR
 
                     assertEquals(1, res.size());
 
-                    Double avg = (Double)res.get(0).get(0);
+                    Integer avg = (Integer)res.get(0).get(0);
 
                     assertEquals(5_000, avg.intValue());
                 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/c4883113/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index 9b5c17a..d8f71a3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.query;
 
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -1778,6 +1779,98 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
         }
     }
 
+    /**
+     * Check results of aggregate functions if no rows are selected.
+     *
+     * @throws Exception If failed,
+     */
+    public void testEmptyCacheAggregates() throws Exception {
+        final String cacheName = "ints";
+
+        IgniteCache<Integer, Value> cache = ignite(0).getOrCreateCache(cacheConfig(cacheName, true,
+            Integer.class, Value.class));
+
+        try (QueryCursor<List<?>> qry = cache.query(new SqlFieldsQuery(
+            "SELECT count(fst), sum(snd), avg(snd), min(snd), max(snd) FROM Value"))) {
+            List<List<?>> result = qry.getAll();
+
+            assertEquals(1, result.size());
+
+            List<?> row = result.get(0);
+
+            assertEquals("count", 0L, ((Number)row.get(0)).longValue());
+            assertEquals("sum", null, row.get(1));
+            assertEquals("avg", null, row.get(2));
+            assertEquals("min", null, row.get(3));
+            assertEquals("max", null, row.get(4));
+        }
+        finally {
+            cache.destroy();
+        }
+    }
+
+    /**
+     * Check avg() with various data types.
+     *
+     * @throws Exception If failed.
+     */
+    public void testAvgVariousDataTypes() throws Exception {
+        final String cacheName = "avgtypes";
+
+        IgniteCache<Integer, AvgDataTypes> cache =
+            ignite(0).getOrCreateCache(cacheConfig(cacheName, true, Integer.class, AvgDataTypes.class));
+
+        // avg 13.125; int avg 13
+        double value[] = new double[] {1, 5, 7, 8, 10.5, 13.5, 20, 40};
+
+        for (int i = 0; i < value.length; i++) {
+            Number v = value[i];
+
+            cache.put(i, new AvgDataTypes(
+                v.byteValue(),
+                v.shortValue(),
+                v.intValue(),
+                v.longValue(),
+                new BigDecimal(v.toString()),
+                v.floatValue(),
+                v.doubleValue()));
+        }
+
+        try {
+            checkAvgWithVariousTypes(cache, false);
+            checkAvgWithVariousTypes(cache, true);
+        }
+        finally {
+            cache.destroy();
+        }
+    }
+
+    /**
+     * Check avg() with various data types.
+     *
+     * @param cache Cache.
+     * @param distinct Distinct flag.
+     */
+    private void checkAvgWithVariousTypes(IgniteCache<Integer, AvgDataTypes> cache, boolean distinct) {
+        String qryText = String.format("select avg(%1$s byteField), avg(%1$s shortField), " +
+                "avg(%1$s intField), avg(%1$s longField), avg(%1$s decimalField), " +
+                "avg(%1$s floatField), avg(%1$s doubleField) from AvgDataTypes", distinct ? "distinct" : "");
+
+        SqlFieldsQuery qry = new SqlFieldsQuery(qryText);
+
+        List<List<?>> result = cache.query(qry).getAll();
+
+        List<?> row = result.get(0);
+
+        assertEquals((byte)13, row.get(0));
+        assertEquals((short)13, row.get(1));
+        assertEquals(13, row.get(2));
+        assertEquals(13L, row.get(3));
+        assertEquals(new BigDecimal("13.125"), row.get(4));
+        assertEquals(13.125f, row.get(5));
+        assertEquals(13.125d, row.get(6));
+    }
+
     /** Simple query with aggregates */
     private void checkSimpleQueryWithAggr(IgniteCache<Integer, Value> cache) {
         try (QueryCursor<List<?>> qry = cache.query(new SqlFieldsQuery(
@@ -1790,7 +1883,7 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
 
             assertEquals("count", 15L, ((Number)row.get(0)).longValue());
             assertEquals("sum", 30L, ((Number)row.get(1)).longValue());
-            assertEquals("avg", 2.0d, ((Number)row.get(2)).doubleValue(), 0.001);
+            assertEquals("avg", 2, ((Integer)row.get(2)).intValue());
             assertEquals("min", 1, ((Integer)row.get(3)).intValue());
             assertEquals("max", 3, ((Integer)row.get(4)).intValue());
         }
@@ -1809,7 +1902,7 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
 
             assertEquals("count distinct", 6L, ((Number)row.get(0)).longValue());
             assertEquals("sum distinct", 6L, ((Number)row.get(1)).longValue());
-            assertEquals("avg distinct", 2.0d, ((Number)row.get(2)).doubleValue(), 0.001);
+            assertEquals("avg distinct", 2, ((Integer)row.get(2)).intValue());
             assertEquals("min distinct", 1, ((Integer)row.get(3)).intValue());
             assertEquals("max distinct", 3, ((Integer)row.get(4)).intValue());
         }
@@ -1829,12 +1922,12 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
 
             assertEquals("count", 15L, ((Number)row.get(0)).longValue());
             assertEquals("sum", 30L, ((Number)row.get(1)).longValue());
-            assertEquals("avg", 2.0d, ((Number)row.get(2)).doubleValue(), 0.001);
+            assertEquals("avg", 2, ((Integer)row.get(2)).intValue());
             assertEquals("min", 1, ((Integer)row.get(3)).intValue());
             assertEquals("max", 3, ((Integer)row.get(4)).intValue());
             assertEquals("count distinct", 6L, ((Number)row.get(5)).longValue());
             assertEquals("sum distinct", 6L, ((Number)row.get(6)).longValue());
-            assertEquals("avg distinct", 2.0d, ((Number)row.get(7)).doubleValue(), 0.001);
+            assertEquals("avg distinct", 2, ((Integer)row.get(7)).intValue());
             assertEquals("min distinct", 1, ((Integer)row.get(8)).intValue());
             assertEquals("max distinct", 3, ((Integer)row.get(9)).intValue());
         }
@@ -1843,7 +1936,8 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
     /** Query with aggregates and groups */
     private void checkQueryWithGroupsAndAggrs(IgniteCache<Integer, Value> cache) {
         try (QueryCursor<List<?>> qry = cache.query(new SqlFieldsQuery(
-            "SELECT fst, count(snd), sum(snd), avg(snd), min(snd), max(snd) FROM Value GROUP BY fst ORDER BY fst"))) {
+            "SELECT fst, count(snd), sum(snd), avg(snd), avg(CAST(snd AS DOUBLE)), " +
+            "min(snd), max(snd) FROM Value GROUP BY fst ORDER BY fst"))) {
             List<List<?>> result = qry.getAll();
 
             assertEquals(6, result.size());
@@ -1852,33 +1946,39 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
             assertEquals("fst", 1, ((Number)row.get(0)).intValue());
             assertEquals("count", 3L, ((Number)row.get(1)).longValue());
             assertEquals("sum", 9L, ((Number)row.get(2)).longValue());
-            assertEquals("avg", 3.0d, ((Number)row.get(3)).doubleValue(), 0.001);
-            assertEquals("min", 3, ((Integer)row.get(4)).intValue());
-            assertEquals("max", 3, ((Integer)row.get(5)).intValue());
+            assertEquals("avg", 3, ((Number)row.get(3)).doubleValue(), 0.001);
+            assertEquals("avg dbl", 3d, ((Number)row.get(4)).doubleValue(), 0.001);
+            assertEquals("min", 3, ((Integer)row.get(5)).intValue());
+            assertEquals("max", 3, ((Integer)row.get(6)).intValue());
+
 
             row = result.get(1);
             assertEquals("fst", 2, ((Number)row.get(0)).intValue());
             assertEquals("count", 3L, ((Number)row.get(1)).longValue());
             assertEquals("sum", 6L, ((Number)row.get(2)).longValue());
-            assertEquals("avg", 2.0d, ((Number)row.get(3)).doubleValue(), 0.001);
-            assertEquals("min", 1, ((Integer)row.get(4)).intValue());
-            assertEquals("max", 3, ((Integer)row.get(5)).intValue());
+            assertEquals("avg", 2, ((Number)row.get(3)).doubleValue(), 0.001);
+            assertEquals("avg dbl", 2d, ((Number)row.get(4)).doubleValue(), 0.001);
+            assertEquals("min", 1, ((Integer)row.get(5)).intValue());
+            assertEquals("max", 3, ((Integer)row.get(6)).intValue());
+
 
             row = result.get(2);
             assertEquals("fst", 3, ((Number)row.get(0)).intValue());
             assertEquals("count", 6L, ((Number)row.get(1)).longValue());
             assertEquals("sum", 9L, ((Number)row.get(2)).longValue());
-            assertEquals("avg", 1.5d, ((Number)row.get(3)).doubleValue(), 0.001);
-            assertEquals("min", 1, ((Integer)row.get(4)).intValue());
-            assertEquals("max", 2, ((Integer)row.get(5)).intValue());
+            assertEquals("avg", 1, ((Integer)row.get(3)).intValue());
+            assertEquals("avg dbl", 1.5d, ((Number)row.get(4)).doubleValue(), 0.001);
+            assertEquals("min", 1, ((Integer)row.get(5)).intValue());
+            assertEquals("max", 2, ((Integer)row.get(6)).intValue());
         }
     }
 
     /** Query with distinct aggregates and groups */
     private void checkQueryWithGroupsAndDistinctAggr(IgniteCache<Integer, Value> cache) {
         try (QueryCursor<List<?>> qry = cache.query(new SqlFieldsQuery(
-            "SELECT count(distinct snd), sum(distinct snd), avg(distinct snd), min(distinct snd), max(distinct snd) " +
-                "FROM Value GROUP BY fst"))) {
+            "SELECT count(distinct snd), sum(distinct snd), avg(distinct snd), " +
+            "avg(distinct cast(snd as double)), min(distinct snd), max(distinct snd) " +
+            "FROM Value GROUP BY fst"))) {
             List<List<?>> result = qry.getAll();
 
             assertEquals(6, result.size());
@@ -1886,32 +1986,35 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
             List<?> row = result.get(0);
             assertEquals("count distinct", 1L, ((Number)row.get(0)).longValue());
             assertEquals("sum distinct", 3L, ((Number)row.get(1)).longValue());
-            assertEquals("avg distinct", 3.0d, ((Number)row.get(2)).doubleValue(), 0.001);
-            assertEquals("min distinct", 3, ((Integer)row.get(3)).intValue());
-            assertEquals("max distinct", 3, ((Integer)row.get(4)).intValue());
+            assertEquals("avg distinct", 3, ((Integer)row.get(2)).intValue());
+            assertEquals("avg distinct dbl", 3.0d, ((Number)row.get(3)).doubleValue(), 0.001);
+            assertEquals("min distinct", 3, ((Integer)row.get(4)).intValue());
+            assertEquals("max distinct", 3, ((Integer)row.get(5)).intValue());
 
             row = result.get(1);
             assertEquals("count distinct", 3L, ((Number)row.get(0)).longValue());
             assertEquals("sum distinct", 6L, ((Number)row.get(1)).longValue());
-            assertEquals("avg distinct", 2.0d, ((Number)row.get(2)).doubleValue(), 0.001);
-            assertEquals("min distinct", 1, ((Integer)row.get(3)).intValue());
-            assertEquals("max distinct", 3, ((Integer)row.get(4)).intValue());
+            assertEquals("avg distinct", 2, ((Integer)row.get(2)).intValue());
+            assertEquals("avg distinct dbl", 2.0d, ((Number)row.get(3)).doubleValue(), 0.001);
+            assertEquals("min distinct", 1, ((Integer)row.get(4)).intValue());
+            assertEquals("max distinct", 3, ((Integer)row.get(5)).intValue());
 
             row = result.get(2);
             assertEquals("count distinct", 2L, ((Number)row.get(0)).longValue());
             assertEquals("sum distinct", 3L, ((Number)row.get(1)).longValue());
-            assertEquals("avg distinct", 1.5d, ((Number)row.get(2)).doubleValue(), 0.001);
-            assertEquals("min distinct", 1, ((Integer)row.get(3)).intValue());
-            assertEquals("max distinct", 2, ((Integer)row.get(4)).intValue());
+            assertEquals("avg distinct", 1, ((Integer)row.get(2)).intValue());
+            assertEquals("avg distinct dbl", 1.5d, ((Number)row.get(3)).doubleValue(), 0.001);
+            assertEquals("min distinct", 1, ((Integer)row.get(4)).intValue());
+            assertEquals("max distinct", 2, ((Integer)row.get(5)).intValue());
         }
     }
 
     /** Query with distinct aggregates and groups */
     private void checkQueryWithGroupsAndAggrMixed(IgniteCache<Integer, Value> cache) {
         try (QueryCursor<List<?>> qry = cache.query(new SqlFieldsQuery(
-            "SELECT fst, count(snd), sum(snd), avg(snd), min(snd), max(snd)," +
-                "count(distinct snd), sum(distinct snd), avg(distinct snd), min(distinct snd), max(distinct snd) " +
-                "FROM Value GROUP BY fst"))) {
+            "SELECT fst, count(snd), sum(snd), avg(snd), avg(cast(snd as double)), min(snd), max(snd)," +
+            "count(distinct snd), sum(distinct snd), avg(distinct snd), avg(distinct cast(snd as double)), " +
+            "min(distinct snd), max(distinct snd) FROM Value GROUP BY fst"))) {
             List<List<?>> result = qry.getAll();
 
             assertEquals(6, result.size());
@@ -1920,40 +2023,46 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
             assertEquals("fst", 1, ((Number)row.get(0)).intValue());
             assertEquals("count", 3L, ((Number)row.get(1)).longValue());
             assertEquals("sum", 9L, ((Number)row.get(2)).longValue());
-            assertEquals("avg", 3.0d, ((Number)row.get(3)).doubleValue(), 0.001);
-            assertEquals("min", 3, ((Integer)row.get(4)).intValue());
-            assertEquals("max", 3, ((Integer)row.get(5)).intValue());
-            assertEquals("count distinct", 1L, ((Number)row.get(6)).longValue());
-            assertEquals("sum distinct", 3L, ((Number)row.get(7)).longValue());
-            assertEquals("avg distinct", 3.0d, ((Number)row.get(8)).doubleValue(), 0.001);
-            assertEquals("min distinct", 3, ((Integer)row.get(9)).intValue());
-            assertEquals("max distinct", 3, ((Integer)row.get(10)).intValue());
+            assertEquals("avg", 3, ((Integer)row.get(3)).intValue());
+            assertEquals("avg dbl", 3.0d, ((Number)row.get(4)).doubleValue(), 0.001);
+            assertEquals("min", 3, ((Integer)row.get(5)).intValue());
+            assertEquals("max", 3, ((Integer)row.get(6)).intValue());
+            assertEquals("count distinct", 1L, ((Number)row.get(7)).longValue());
+            assertEquals("sum distinct", 3L, ((Number)row.get(8)).longValue());
+            assertEquals("avg distinct", 3, ((Integer)row.get(9)).intValue());
+            assertEquals("avg distinct dbl", 3.0d, ((Number)row.get(10)).doubleValue(), 0.001);
+            assertEquals("min distinct", 3, ((Integer)row.get(11)).intValue());
+            assertEquals("max distinct", 3, ((Integer)row.get(12)).intValue());
 
             row = result.get(1);
             assertEquals("fst", 2, ((Number)row.get(0)).intValue());
             assertEquals("count", 3L, ((Number)row.get(1)).longValue());
             assertEquals("sum", 6L, ((Number)row.get(2)).longValue());
-            assertEquals("avg", 2.0d, ((Number)row.get(3)).doubleValue(), 0.001);
-            assertEquals("min", 1, ((Integer)row.get(4)).intValue());
-            assertEquals("max", 3, ((Integer)row.get(5)).intValue());
-            assertEquals("count distinct", 3L, ((Number)row.get(6)).longValue());
-            assertEquals("sum distinct", 6L, ((Number)row.get(7)).longValue());
-            assertEquals("avg distinct", 2.0d, ((Number)row.get(8)).doubleValue(), 0.001);
-            assertEquals("min distinct", 1, ((Integer)row.get(9)).intValue());
-            assertEquals("max distinct", 3, ((Integer)row.get(10)).intValue());
+            assertEquals("avg", 2, ((Integer)row.get(3)).intValue());
+            assertEquals("avg dbl", 2.0d, ((Number)row.get(4)).doubleValue(), 0.001);
+            assertEquals("min", 1, ((Integer)row.get(5)).intValue());
+            assertEquals("max", 3, ((Integer)row.get(6)).intValue());
+            assertEquals("count distinct", 3L, ((Number)row.get(7)).longValue());
+            assertEquals("sum distinct", 6L, ((Number)row.get(8)).longValue());
+            assertEquals("avg distinct", 2, ((Integer)row.get(9)).intValue());
+            assertEquals("avg distinct dbl", 2.0d, ((Number)row.get(10)).doubleValue(), 0.001);
+            assertEquals("min distinct", 1, ((Integer)row.get(11)).intValue());
+            assertEquals("max distinct", 3, ((Integer)row.get(12)).intValue());
 
             row = result.get(2);
             assertEquals("fst", 3, ((Number)row.get(0)).intValue());
             assertEquals("count", 6L, ((Number)row.get(1)).longValue());
             assertEquals("sum", 9L, ((Number)row.get(2)).longValue());
-            assertEquals("avg", 1.5d, ((Number)row.get(3)).doubleValue(), 0.001);
-            assertEquals("min", 1, ((Integer)row.get(4)).intValue());
-            assertEquals("max", 2, ((Integer)row.get(5)).intValue());
-            assertEquals("count distinct", 2L, ((Number)row.get(6)).longValue());
-            assertEquals("sum distinct", 3L, ((Number)row.get(7)).longValue());
-            assertEquals("avg distinct", 1.5d, ((Number)row.get(8)).doubleValue(), 0.001);
-            assertEquals("min distinct", 1, ((Integer)row.get(9)).intValue());
-            assertEquals("max distinct", 2, ((Integer)row.get(10)).intValue());
+            assertEquals("avg", 1, ((Integer)row.get(3)).intValue());
+            assertEquals("avg dbl", 1.5d, ((Number)row.get(4)).doubleValue(), 0.001);
+            assertEquals("min", 1, ((Integer)row.get(5)).intValue());
+            assertEquals("max", 2, ((Integer)row.get(6)).intValue());
+            assertEquals("count distinct", 2L, ((Number)row.get(7)).longValue());
+            assertEquals("sum distinct", 3L, ((Number)row.get(8)).longValue());
+            assertEquals("avg distinct", 1, ((Integer)row.get(9)).intValue());
+            assertEquals("avg distinct dbl", 1.5d, ((Number)row.get(10)).doubleValue(), 0.001);
+            assertEquals("min distinct", 1, ((Integer)row.get(11)).intValue());
+            assertEquals("max distinct", 2, ((Integer)row.get(12)).intValue());
         }
     }
 
@@ -2245,4 +2354,47 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
             this.OFFER_ID = offer_Id;
         }
     }
+
+    /** */
+    public class AvgDataTypes {
+        /** */
+        @QuerySqlField
+        private Byte byteField;
+
+        /** */
+        @QuerySqlField
+        private Short shortField;
+
+        /** */
+        @QuerySqlField
+        private Integer intField;
+
+        /** */
+        @QuerySqlField
+        private Long longField;
+
+        /** */
+        @QuerySqlField
+        private BigDecimal decimalField;
+
+        /** */
+        @QuerySqlField
+        private Float floatField;
+
+        /** */
+        @QuerySqlField
+        private Double doubleField;
+
+        /** */
+        public AvgDataTypes(Byte byteField, Short shortField, Integer intField, Long longField,
+            BigDecimal decimalField, Float floatField, Double doubleField) {
+            this.byteField = byteField;
+            this.shortField = shortField;
+            this.intField = intField;
+            this.longField = longField;
+            this.decimalField = decimalField;
+            this.floatField = floatField;
+            this.doubleField = doubleField;
+        }
+    }
 }


[03/12] ignite git commit: IGNITE-5342 - Skip permission check for TASK_EXECUTE for service jobs

Posted by sb...@apache.org.
IGNITE-5342 - Skip permission check for TASK_EXECUTE for service jobs

(cherry picked from commit d77a134)


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/abb9d942
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/abb9d942
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/abb9d942

Branch: refs/heads/ignite-5075
Commit: abb9d9428696ddb3a250e9a491d4abea437d2a38
Parents: 858d11b
Author: dkarachentsev <dk...@gridgain.com>
Authored: Wed May 31 16:57:29 2017 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Wed May 31 16:57:29 2017 +0300

----------------------------------------------------------------------
 .../cache/CacheAffinitySharedManager.java       |  52 +--
 .../processors/cache/GridCacheProcessor.java    |  39 +-
 .../GridDhtPartitionsExchangeFuture.java        |   2 +-
 ...IgniteClientCacheInitializationFailTest.java | 367 +++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java       |   2 +
 5 files changed, 432 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/abb9d942/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 889eeda..9063a12 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -396,40 +396,48 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
                     CU.affinityNode(cctx.localNode(), req.startCacheConfiguration().getNodeFilter());
             }
 
-            if (startCache) {
-                cctx.cache().prepareCacheStart(cacheDesc, nearCfg, fut.topologyVersion());
+            try {
+                if (startCache) {
+                    cctx.cache().prepareCacheStart(cacheDesc, nearCfg, fut.topologyVersion());
 
-                if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) {
-                    if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
-                        U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+                    if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) {
+                        if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
+                            U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+                    }
                 }
-            }
 
-            if (!crd || !lateAffAssign) {
-                GridCacheContext cacheCtx = cctx.cacheContext(cacheDesc.cacheId());
+                if (!crd || !lateAffAssign) {
+                    GridCacheContext cacheCtx = cctx.cacheContext(cacheDesc.cacheId());
 
-                if (cacheCtx != null && !cacheCtx.isLocal()) {
-                    boolean clientCacheStarted =
-                        req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
+                    if (cacheCtx != null && !cacheCtx.isLocal()) {
+                        boolean clientCacheStarted =
+                            req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
 
-                    if (clientCacheStarted)
-                        initAffinity(cacheDesc, cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
-                    else if (!req.clientStartOnly()) {
-                        assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
+                        if (clientCacheStarted)
+                            initAffinity(cacheDesc, cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
+                        else if (!req.clientStartOnly()) {
+                            assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
 
-                        GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
+                            GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
 
-                        assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
+                            assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
 
-                        List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
-                            fut.discoveryEvent(), fut.discoCache());
+                            List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
+                                fut.discoveryEvent(), fut.discoCache());
 
-                        aff.initialize(fut.topologyVersion(), assignment);
+                            aff.initialize(fut.topologyVersion(), assignment);
+                        }
                     }
                 }
+                else
+                    initStartedCacheOnCoordinator(fut, cacheDesc.cacheId());
+            }
+            catch (IgniteCheckedException e) {
+                U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
+                    "[cacheName=" + req.cacheName() + ']', e);
+
+                cctx.cache().forceCloseCache(fut.topologyVersion(), action, e);
             }
-            else
-                initStartedCacheOnCoordinator(fut, cacheDesc.cacheId());
         }
 
         for (DynamicCacheChangeRequest req : exchActions.closeRequests(cctx.localNodeId())) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/abb9d942/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index dbcddf4..c7e8670 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1890,8 +1890,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     /**
      * @param req Stop request.
      */
-    private void prepareCacheStop(DynamicCacheChangeRequest req) {
-        assert req.stop() || req.close() : req;
+    private void prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose) {
+        assert req.stop() || req.close() || forceClose : req;
 
         GridCacheAdapter<?, ?> cache = caches.remove(req.cacheName());
 
@@ -1907,6 +1907,27 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     }
 
     /**
+     * Closes cache even if it's not fully initialized (e.g. fail on cache init stage).
+     *
+     * @param topVer Completed topology version.
+     * @param act Exchange action.
+     * @param err Error.
+     */
+    void forceCloseCache(
+        AffinityTopologyVersion topVer,
+        final ExchangeActions.ActionData act,
+        Throwable err
+    ) {
+        ExchangeActions actions = new ExchangeActions(){
+            @Override List<DynamicCacheChangeRequest> closeRequests(UUID nodeId) {
+                return Collections.singletonList(act.request());
+            }
+        };
+
+        onExchangeDone(topVer, actions, err, true);
+    }
+
+    /**
      * Callback invoked when first exchange future for dynamic cache is completed.
      *
      * @param topVer Completed topology version.
@@ -1917,7 +1938,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     public void onExchangeDone(
         AffinityTopologyVersion topVer,
         @Nullable ExchangeActions exchActions,
-        Throwable err
+        Throwable err,
+        boolean forceClose
     ) {
         for (GridCacheAdapter<?, ?> cache : caches.values()) {
             GridCacheContext<?, ?> cacheCtx = cache.context();
@@ -1930,11 +1952,11 @@ public class GridCacheProcessor extends GridProcessorAdapter {
             }
         }
 
-        if (exchActions != null && err == null) {
+        if (exchActions != null && (err == null || forceClose)) {
             for (ExchangeActions.ActionData action : exchActions.cacheStopRequests()) {
                 stopGateway(action.request());
 
-                prepareCacheStop(action.request());
+                prepareCacheStop(action.request(), forceClose);
             }
 
             for (DynamicCacheChangeRequest req : exchActions.closeRequests(ctx.localNodeId())) {
@@ -1955,9 +1977,12 @@ public class GridCacheProcessor extends GridProcessorAdapter {
 
                         proxy.context().gate().onStopped();
 
-                        prepareCacheStop(req);
+                        prepareCacheStop(req, forceClose);
                     }
                 }
+
+                if (forceClose)
+                    completeCacheStartFuture(req, false, err);
             }
         }
     }
@@ -1978,7 +2003,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
      * @param req Request to complete future for.
      * @param err Error if any.
      */
-    void completeCacheStartFuture(DynamicCacheChangeRequest req, boolean success, @Nullable Exception err) {
+    void completeCacheStartFuture(DynamicCacheChangeRequest req, boolean success, @Nullable Throwable err) {
         if (req.initiatingNodeId().equals(ctx.localNodeId())) {
             DynamicCacheStartFuture fut = (DynamicCacheStartFuture)pendingFuts.get(req.requestId());
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/abb9d942/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 72c5bbc..2cd5629 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1170,7 +1170,7 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
             cacheValidRes = m;
         }
 
-        cctx.cache().onExchangeDone(exchId.topologyVersion(), exchActions, err);
+        cctx.cache().onExchangeDone(exchId.topologyVersion(), exchActions, err, false);
 
         cctx.exchange().onExchangeDone(this, err);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/abb9d942/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
new file mode 100644
index 0000000..1d616f8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import javax.cache.Cache;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.query.GridQueryCancel;
+import org.apache.ignite.internal.processors.query.GridQueryIndexing;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
+import org.apache.ignite.internal.processors.query.QueryIndexDescriptorImpl;
+import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitor;
+import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.internal.util.lang.GridCloseableIterator;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.spi.indexing.IndexingQueryFilter;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Test checks whether cache initialization error on client side
+ * doesn't causes hangs and doesn't impact other caches.
+ */
+public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractTest {
+    /** Failed cache name. */
+    private static final String CACHE_NAME = "cache";
+
+    /** Atomic cache name. */
+    private static final String ATOMIC_CACHE_NAME = "atomic-cache";
+
+    /** Tx cache name. */
+    private static final String TX_CACHE_NAME = "tx-cache";
+
+    /** Near atomic cache name. */
+    private static final String NEAR_ATOMIC_CACHE_NAME = "near-atomic-cache";
+
+    /** Near tx cache name. */
+    private static final String NEAR_TX_CACHE_NAME = "near-tx-cache";
+
+    /** Failed caches. */
+    private static final Set<String> FAILED_CACHES;
+
+    static {
+        Set<String> set = new HashSet<>();
+
+        set.add(ATOMIC_CACHE_NAME);
+        set.add(TX_CACHE_NAME);
+        set.add(NEAR_ATOMIC_CACHE_NAME);
+        set.add(NEAR_TX_CACHE_NAME);
+
+        FAILED_CACHES = Collections.unmodifiableSet(set);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrid("server");
+        startGrid("client");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.contains("server")) {
+            CacheConfiguration<Integer, String> ccfg1 = new CacheConfiguration<>();
+
+            ccfg1.setIndexedTypes(Integer.class, String.class);
+            ccfg1.setName(ATOMIC_CACHE_NAME);
+            ccfg1.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+            CacheConfiguration<Integer, String> ccfg2 = new CacheConfiguration<>();
+
+            ccfg2.setIndexedTypes(Integer.class, String.class);
+            ccfg2.setName(TX_CACHE_NAME);
+            ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+            cfg.setCacheConfiguration(ccfg1, ccfg2);
+        }
+        else {
+            GridQueryProcessor.idxCls = FailedIndexing.class;
+
+            cfg.setClientMode(true);
+        }
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicCacheInitialization() throws Exception {
+        checkCacheInitialization(ATOMIC_CACHE_NAME);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTransactionalCacheInitialization() throws Exception {
+        checkCacheInitialization(TX_CACHE_NAME);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicNearCacheInitialization() throws Exception {
+        checkCacheInitialization(NEAR_ATOMIC_CACHE_NAME);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTransactionalNearCacheInitialization() throws Exception {
+        checkCacheInitialization(NEAR_TX_CACHE_NAME);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkCacheInitialization(final String cacheName) throws Exception {
+        Ignite client = grid("client");
+
+        checkFailedCache(client, cacheName);
+
+        checkFineCache(client, CACHE_NAME + 1);
+
+        assertNull(client.cache(cacheName));
+        assertNull(client.getOrCreateCache(cacheName));
+
+        checkFineCache(client, CACHE_NAME + 2);
+    }
+
+    /**
+     * @param client Client.
+     * @param cacheName Cache name.
+     */
+    private void checkFineCache(Ignite client, String cacheName) {
+        IgniteCache<Integer, String> cache = client.getOrCreateCache(cacheName);
+
+        cache.put(1, "1");
+
+        assertEquals("1", cache.get(1));
+    }
+
+    /**
+     * @param client Client.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    private void checkFailedCache(final Ignite client, final String cacheName) {
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                IgniteCache<Integer, String> cache;
+
+                // Start cache with near enabled.
+                if (NEAR_ATOMIC_CACHE_NAME.equals(cacheName) || NEAR_TX_CACHE_NAME.equals(cacheName)) {
+                    CacheConfiguration<Integer, String> ccfg = new CacheConfiguration<Integer, String>(cacheName)
+                        .setNearConfiguration(new NearCacheConfiguration<Integer, String>());
+
+                    if (NEAR_TX_CACHE_NAME.equals(cacheName))
+                        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+                    cache = client.getOrCreateCache(ccfg);
+                }
+                else
+                    cache = client.cache(cacheName);
+
+
+                cache.put(1, "1");
+
+                assertEquals("1", cache.get(1));
+
+                return null;
+            }
+        }, CacheException.class, null);
+    }
+
+    /**
+     * To fail on cache start.
+     */
+    private static class FailedIndexing implements GridQueryIndexing {
+        /** {@inheritDoc} */
+        @Override public void start(GridKernalContext ctx, GridSpinBusyLock busyLock) throws IgniteCheckedException {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void stop() throws IgniteCheckedException {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public <K, V> QueryCursor<Cache.Entry<K, V>> queryDistributedSql(String schemaName, SqlQuery qry,
+            boolean keepBinary, int mainCacheId) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public FieldsQueryCursor<List<?>> queryDistributedSqlFields(String schemaName, SqlFieldsQuery qry,
+            boolean keepBinary, GridQueryCancel cancel, @Nullable Integer mainCacheId) throws IgniteCheckedException {
+            return null;
+        }
+
+
+        /** {@inheritDoc} */
+        @Override public long streamUpdateQuery(String spaceName, String qry, @Nullable Object[] params,
+            IgniteDataStreamer<?, ?> streamer) throws IgniteCheckedException {
+            return 0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public <K, V> QueryCursor<Cache.Entry<K, V>> queryLocalSql(String schemaName, SqlQuery qry,
+            IndexingQueryFilter filter, boolean keepBinary) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public FieldsQueryCursor<List<?>> queryLocalSqlFields(String schemaName, SqlFieldsQuery qry,
+            boolean keepBinary, IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalText(String spaceName, String qry,
+            String typeName, IndexingQueryFilter filter) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void dynamicIndexCreate(String spaceName, String tblName, QueryIndexDescriptorImpl idxDesc,
+            boolean ifNotExists, SchemaIndexCacheVisitor cacheVisitor) throws IgniteCheckedException {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void dynamicIndexDrop(String spaceName, String idxName,
+            boolean ifExists) throws IgniteCheckedException {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void registerCache(String cacheName, String schemaName,
+            GridCacheContext<?, ?> cctx) throws IgniteCheckedException {
+            if (FAILED_CACHES.contains(cctx.name()) && cctx.kernalContext().clientNode())
+                throw new IgniteCheckedException("Test query exception " + cctx.name() + " " + new Random().nextInt());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void unregisterCache(String spaceName) throws IgniteCheckedException {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean registerType(GridCacheContext cctx,
+            GridQueryTypeDescriptor desc) throws IgniteCheckedException {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void store(String cacheName, GridQueryTypeDescriptor type, KeyCacheObject key, int partId,
+            CacheObject val, GridCacheVersion ver, long expirationTime, long link) throws IgniteCheckedException {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void remove(String spaceName, GridQueryTypeDescriptor type, KeyCacheObject key, int partId,
+            CacheObject val, GridCacheVersion ver) throws IgniteCheckedException {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void rebuildIndexesFromHash(GridCacheContext cctx, String schemaName,
+            String typeName) throws IgniteCheckedException {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void markForRebuildFromHash(String cacheName, String typeName) {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public IndexingQueryFilter backupFilter(AffinityTopologyVersion topVer, int[] parts) {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onDisconnected(IgniteFuture<?> reconnectFut) {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public PreparedStatement prepareNativeStatement(String space, String sql) throws SQLException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Collection<GridRunningQueryInfo> runningQueries(long duration) {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancelQueries(Collection<Long> queries) {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancelAllQueries() {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public String schema(String cacheName) {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isInsertStatement(PreparedStatement nativeStmt) {
+            return false;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/abb9d942/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 6370a10..23f8445 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -71,6 +71,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheTxNearPeekModesTes
 import org.apache.ignite.internal.processors.cache.IgniteCacheTxPeekModesTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheTxPreloadNoWriteTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheTxReplicatedPeekModesTest;
+import org.apache.ignite.internal.processors.cache.IgniteClientCacheInitializationFailTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheFilterTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartNoExchangeTimeoutTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartSelfTest;
@@ -227,6 +228,7 @@ public class IgniteCacheTestSuite4 extends TestSuite {
         suite.addTestSuite(CacheStartOnJoinTest.class);
         suite.addTestSuite(IgniteCacheStartTest.class);
         suite.addTestSuite(CacheDiscoveryDataConcurrentJoinTest.class);
+        suite.addTestSuite(IgniteClientCacheInitializationFailTest.class);
 
         suite.addTestSuite(GridCacheTxLoadFromStoreOnLockSelfTest.class);