You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/06/01 08:52:58 UTC
[01/12] ignite git commit: IGNITE-5210 - If enabled security
authentication, server is unable to restart if client tries to reconnect
Repository: ignite
Updated Branches:
refs/heads/ignite-5075 70fc1a019 -> eeba611c8
IGNITE-5210 - If enabled security authentication, server is unable to restart if client tries to reconnect
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a7e56606
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a7e56606
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a7e56606
Branch: refs/heads/ignite-5075
Commit: a7e566068c143a5c33ea0f583157bd197068b321
Parents: 738451c
Author: dkarachentsev <dk...@gridgain.com>
Authored: Wed May 31 15:35:34 2017 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Wed May 31 15:35:34 2017 +0300
----------------------------------------------------------------------
.../ignite/spi/discovery/tcp/ClientImpl.java | 14 +++-
.../discovery/AuthenticationRestartTest.java | 88 ++++++++++++++++++++
.../IgniteSpiDiscoverySelfTestSuite.java | 2 +
3 files changed, 100 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a7e56606/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 619b4cc..d5bacdb 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -89,8 +89,8 @@ import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryCustomEventMessa
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryDuplicateIdMessage;
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryHandshakeRequest;
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryHandshakeResponse;
-import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryMetricsUpdateMessage;
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryJoinRequestMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryMetricsUpdateMessage;
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddedMessage;
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeFailedMessage;
@@ -639,9 +639,12 @@ class ClientImpl extends TcpDiscoveryImpl {
if (!recon) {
TcpDiscoveryNode node = locNode;
- if (locNode.order() > 0)
+ if (locNode.order() > 0) {
node = locNode.clientReconnectNode(spi.spiCtx.nodeAttributes());
+ marshalCredentials(node);
+ }
+
msg = new TcpDiscoveryJoinRequestMessage(
node,
spi.collectExchangeData(new DiscoveryDataPacket(getLocalNodeId())));
@@ -736,8 +739,11 @@ class ClientImpl extends TcpDiscoveryImpl {
// Use security-unsafe getter.
Map<String, Object> attrs = new HashMap<>(node.getAttributes());
- attrs.put(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS,
- U.marshal(spi.marshaller(), attrs.get(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS)));
+ Object creds = attrs.get(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS);
+
+ assert !(creds instanceof byte[]);
+
+ attrs.put(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS, U.marshal(spi.marshaller(), creds));
node.setAttributes(attrs);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a7e56606/modules/core/src/test/java/org/apache/ignite/spi/discovery/AuthenticationRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/AuthenticationRestartTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/AuthenticationRestartTest.java
new file mode 100644
index 0000000..5841094
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/AuthenticationRestartTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TestReconnectPluginProvider;
+import org.apache.ignite.spi.discovery.tcp.TestReconnectProcessor;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
+/**
+ * Checks whether client is able to reconnect to restarted cluster with
+ * enabled security.
+ */
+public class AuthenticationRestartTest extends GridCommonAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setClientMode(igniteInstanceName.contains("client"));
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setJoinTimeout(1120_000);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ TestReconnectPluginProvider.enabled = true;
+ TestReconnectProcessor.enabled = true;
+
+ startGrid("server");
+ startGrid("client");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ TestReconnectPluginProvider.enabled = false;
+ TestReconnectProcessor.enabled = false;
+
+ stopAllGrids();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testClientReconnect() throws Exception {
+ stopGrid("server");
+
+ final IgniteEx client = grid("client");
+
+ waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return client.cluster().clientReconnectFuture() != null;
+ }
+ }, 10_000);
+
+ startGrid("server");
+
+ IgniteFuture<?> fut = client.cluster().clientReconnectFuture();
+
+ assertNotNull(fut);
+
+ fut.get();
+
+ assertEquals(2, client.cluster().nodes().size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a7e56606/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index e6b39f7..1287149 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -19,6 +19,7 @@ package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.spi.GridTcpSpiForwardingSelfTest;
+import org.apache.ignite.spi.discovery.AuthenticationRestartTest;
import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoveryMarshallerCheckSelfTest;
import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest;
import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiMulticastTest;
@@ -87,6 +88,7 @@ public class IgniteSpiDiscoverySelfTestSuite extends TestSuite {
suite.addTest(new TestSuite(TcpDiscoveryMultiThreadedTest.class));
suite.addTest(new TestSuite(TcpDiscoveryNodeAttributesUpdateOnReconnectTest.class));
+ suite.addTest(new TestSuite(AuthenticationRestartTest.class));
// SSL.
suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
[11/12] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-5075
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-5075
# Conflicts:
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5ef69009
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5ef69009
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5ef69009
Branch: refs/heads/ignite-5075
Commit: 5ef69009894cc2a6d33e31d923daed2f00bc45a1
Parents: 056b28f c488311
Author: sboikov <sb...@gridgain.com>
Authored: Thu Jun 1 11:52:35 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Jun 1 11:52:35 2017 +0300
----------------------------------------------------------------------
DEVNOTES.txt | 74 ++--
assembly/dependencies-fabric-lgpl.xml | 2 +
assembly/dependencies-fabric.xml | 2 +
assembly/release-fabric-lgpl.xml | 1 +
assembly/release-fabric.xml | 1 +
assembly/release-yardstick.xml | 98 +++++
.../affinity/GridAffinityProcessor.java | 2 +-
.../cache/CacheAffinitySharedManager.java | 35 +-
.../processors/cache/GridCacheAdapter.java | 3 +-
.../processors/cache/GridCacheProcessor.java | 39 +-
.../CacheDataStructuresManager.java | 4 +-
.../GridDhtPartitionsExchangeFuture.java | 2 +-
.../closure/GridClosureProcessor.java | 11 +-
.../service/GridServiceProcessor.java | 4 +-
.../processors/service/GridServiceProxy.java | 4 +-
.../processors/task/GridTaskProcessor.java | 7 +-
.../task/GridTaskThreadContextKey.java | 5 +-
.../ignite/spi/discovery/tcp/ClientImpl.java | 14 +-
.../IgniteComputeTopologyExceptionTest.java | 2 +-
...IgniteClientCacheInitializationFailTest.java | 367 +++++++++++++++++++
.../discovery/AuthenticationRestartTest.java | 88 +++++
.../testsuites/IgniteCacheTestSuite4.java | 2 +
.../IgniteSpiDiscoverySelfTestSuite.java | 2 +
.../query/h2/sql/GridSqlQuerySplitter.java | 22 +-
...ClientReconnectCacheQueriesFailoverTest.java | 2 +-
.../query/IgniteSqlSplitterSelfTest.java | 256 ++++++++++---
modules/ml/pom.xml | 13 +
.../main/java/org/apache/ignite/ml/Model.java | 39 ++
.../ml/clustering/BaseKMeansClusterer.java | 98 +++++
.../apache/ignite/ml/clustering/Clusterer.java | 32 ++
.../ml/clustering/ClusterizationModel.java | 29 ++
.../clustering/KMeansDistributedClusterer.java | 298 +++++++++++++++
.../ml/clustering/KMeansLocalClusterer.java | 174 +++++++++
.../ignite/ml/clustering/KMeansModel.java | 79 ++++
.../ignite/ml/clustering/WeightedClusterer.java | 38 ++
.../ignite/ml/clustering/package-info.java | 22 ++
.../apache/ignite/ml/math/DistanceMeasure.java | 39 ++
.../ignite/ml/math/EuclideanDistance.java | 48 +++
.../org/apache/ignite/ml/math/MathUtils.java | 31 ++
.../java/org/apache/ignite/ml/math/Matrix.java | 2 +
.../apache/ignite/ml/math/StorageConstants.java | 3 +
.../org/apache/ignite/ml/math/VectorUtils.java | 41 +++
.../math/exceptions/ConvergenceException.java | 48 +++
.../exceptions/MathArithmeticException.java | 6 +-
.../exceptions/MathIllegalNumberException.java | 51 +++
.../exceptions/MathIllegalStateException.java | 49 +++
.../exceptions/NumberIsTooSmallException.java | 79 ++++
.../ignite/ml/math/functions/Functions.java | 39 ++
.../apache/ignite/ml/math/impls/CacheUtils.java | 47 +++
.../ml/math/impls/matrix/AbstractMatrix.java | 2 +
.../impls/matrix/SparseDistributedMatrix.java | 2 +-
.../impls/storage/matrix/MapWrapperStorage.java | 93 +++++
.../vector/SparseLocalOnHeapVectorStorage.java | 32 ++
.../ml/math/impls/vector/DelegatingVector.java | 5 +
.../ml/math/impls/vector/MapWrapperVector.java | 32 ++
.../ml/math/impls/vector/SparseLocalVector.java | 9 +
.../ignite/ml/math/impls/vector/VectorView.java | 1 +
.../ignite/ml/math/statistics/Variance.java | 53 +++
.../org/apache/ignite/ml/math/util/MapUtil.java | 38 ++
.../apache/ignite/ml/math/util/MatrixUtil.java | 38 +-
.../org/apache/ignite/ml/IgniteMLTestSuite.java | 4 +-
.../ml/clustering/ClusteringTesetSuite.java | 15 +
.../KMeansDistributedClustererTest.java | 184 ++++++++++
.../ml/clustering/KMeansLocalClustererTest.java | 46 +++
.../apache/ignite/ml/clustering/KMeansUtil.java | 33 ++
.../ignite/ml/math/MathImplLocalTestSuite.java | 1 +
modules/platforms/cpp/examples/Makefile.am | 1 +
.../cpp/examples/compute-example/Makefile.am | 59 +++
.../compute-example/config/compute-example.xml | 52 +++
.../project/vs/compute-example.vcxproj | 102 ++++++
.../project/vs/compute-example.vcxproj.filters | 27 ++
.../compute-example/src/compute_example.cpp | 177 +++++++++
modules/platforms/cpp/examples/configure.ac | 1 +
.../project/vs/continuous-query-example.vcxproj | 4 -
.../src/continuous_query_example.cpp | 130 ++++++-
.../examples/include/ignite/examples/address.h | 2 +-
.../include/ignite/examples/organization.h | 4 +-
.../cpp/examples/project/vs/ignite-examples.sln | 6 +
.../Apache.Ignite.Core.nuspec | 5 +-
modules/platforms/dotnet/build.ps1 | 5 +-
modules/web-console/web-agent/pom.xml | 8 +
parent/pom.xml | 8 +-
pom.xml | 254 ++++---------
83 files changed, 3452 insertions(+), 335 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5ef69009/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 1c71dab,9063a12..c563b81
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@@ -378,37 -392,51 +378,44 @@@ public class CacheAffinitySharedManager
nearCfg = req.nearCacheConfiguration();
}
else {
- startCache = cctx.cacheContext(action.descriptor().cacheId()) == null &&
- CU.affinityNode(cctx.localNode(), req.startCacheConfiguration().getNodeFilter());
+ startCache = cctx.cacheContext(cacheDesc.cacheId()) == null &&
+ CU.affinityNode(cctx.localNode(), cacheDesc.groupDescriptor().config().getNodeFilter());
}
- if (startCache) {
- cctx.cache().prepareCacheStart(cacheDesc, nearCfg, fut.topologyVersion());
+ try {
+ if (startCache) {
+ cctx.cache().prepareCacheStart(cacheDesc, nearCfg, fut.topologyVersion());
- if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) {
- if (fut.discoCache().cacheGroupAffinityNodes(cacheDesc.groupId()).isEmpty())
- U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+ if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) {
- if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
++ if (fut.discoCache().cacheGroupAffinityNodes(cacheDesc.groupId()).isEmpty())
+ U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+ }
}
+ }
++ catch (IgniteCheckedException e) {
++ U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
++ "[cacheName=" + req.cacheName() + ']', e);
+
- if (!crd || !lateAffAssign) {
- GridCacheContext cacheCtx = cctx.cacheContext(cacheDesc.cacheId());
-
- if (cacheCtx != null && !cacheCtx.isLocal()) {
- boolean clientCacheStarted =
- req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
++ cctx.cache().forceCloseCache(fut.topologyVersion(), action, e);
++ }
+ }
- Set<Integer> gprs = new HashSet<>();
- if (clientCacheStarted)
- initAffinity(cacheDesc, cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
- else if (!req.clientStartOnly()) {
- assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
++ Set<Integer> gprs = new HashSet<>();
- for (ExchangeActions.ActionData action : exchActions.newAndClientCachesStartRequests()) {
- Integer grpId = action.descriptor().groupId();
- GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
++ for (ExchangeActions.ActionData action : exchActions.newAndClientCachesStartRequests()) {
++ Integer grpId = action.descriptor().groupId();
- if (gprs.add(grpId)) {
- if (crd && lateAffAssign)
- initStartedGroupOnCoordinator(fut, action.descriptor().groupDescriptor());
- else {
- CacheGroupInfrastructure grp = cctx.cache().cacheGroup(grpId);
- assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
++ if (gprs.add(grpId)) {
++ if (crd && lateAffAssign)
++ initStartedGroupOnCoordinator(fut, action.descriptor().groupDescriptor());else {
++ CacheGroupInfrastructure grp = cctx.cache().cacheGroup(grpId);
- if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.topologyVersion())) {
- List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
- fut.discoveryEvent(), fut.discoCache());
++ if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.topologyVersion())) {
+ assert grp.affinity().lastVersion().equals(AffinityTopologyVersion.NONE) : grp.affinity().lastVersion();
- aff.initialize(fut.topologyVersion(), assignment);
- }
+ initAffinity(registeredGrps.get(grp.groupId()), grp.affinity(), fut);
}
}
- else
- initStartedCacheOnCoordinator(fut, cacheDesc.cacheId());
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
- "[cacheName=" + req.cacheName() + ']', e);
-
- cctx.cache().forceCloseCache(fut.topologyVersion(), action, e);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5ef69009/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5ef69009/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index cac8420,c7e8670..afb578b
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@@ -2031,10 -1889,9 +2031,10 @@@ public class GridCacheProcessor extend
/**
* @param req Stop request.
+ * @return Cache group for stopped cache.
*/
- private CacheGroupInfrastructure prepareCacheStop(DynamicCacheChangeRequest req) {
- assert req.stop() || req.close() : req;
- private void prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose) {
++ private CacheGroupInfrastructure prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose) {
+ assert req.stop() || req.close() || forceClose : req;
GridCacheAdapter<?, ?> cache = caches.remove(req.cacheName());
@@@ -2081,14 -1956,11 +2103,14 @@@
for (ExchangeActions.ActionData action : exchActions.cacheStopRequests()) {
stopGateway(action.request());
- prepareCacheStop(action.request());
+ prepareCacheStop(action.request(), forceClose);
}
- for (DynamicCacheChangeRequest req : exchActions.closeRequests(ctx.localNodeId())) {
- String cacheName = req.cacheName();
+ for (CacheGroupDescriptor grpDesc : exchActions.cacheGroupsToStop())
+ stopCacheGroup(grpDesc.groupId());
+
+ for (ExchangeActions.ActionData req : exchActions.closeRequests(ctx.localNodeId())) {
+ String cacheName = req.request().cacheName();
IgniteCacheProxy<?, ?> proxy = jCacheProxies.get(cacheName);
@@@ -2105,12 -1977,12 +2127,15 @@@
proxy.context().gate().onStopped();
- CacheGroupInfrastructure grp = prepareCacheStop(req.request());
- prepareCacheStop(req, forceClose);
++ CacheGroupInfrastructure grp = prepareCacheStop(req.request(), forceClose);
+
+ if (grp != null && !grp.hasCaches())
+ stopCacheGroup(grp.groupId());
}
}
+
+ if (forceClose)
+ completeCacheStartFuture(req, false, err);
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5ef69009/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 65a20eb,2cd5629..d000e26
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@@ -1197,16 -1161,16 +1197,16 @@@ public class GridDhtPartitionsExchangeF
boolean valid = true;
- if (cacheCtx.config().getTopologyValidator() != null && !CU.isSystemCache(cacheCtx.name()))
- valid = cacheCtx.config().getTopologyValidator().validate(discoEvt.topologyNodes());
+ if (grp.topologyValidator() != null && !grp.systemCache())
+ valid = grp.topologyValidator().validate(discoEvt.topologyNodes());
- m.put(cacheCtx.cacheId(), new CacheValidation(valid, lostParts));
+ m.put(grp.groupId(), new CacheValidation(valid, lostParts));
}
- cacheValidRes = m;
+ grpValidRes = m;
}
- cctx.cache().onExchangeDone(exchId.topologyVersion(), exchActions, err);
+ cctx.cache().onExchangeDone(exchId.topologyVersion(), exchActions, err, false);
cctx.exchange().onExchangeDone(this, err);
[05/12] ignite git commit: IGNITE-5161: Implemented C++ compute
example.
Posted by sb...@apache.org.
IGNITE-5161: Implemented C++ compute example.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d0bddfa5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d0bddfa5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d0bddfa5
Branch: refs/heads/ignite-5075
Commit: d0bddfa527af6c5cf4554e484ce04c078bfd69e9
Parents: 41dcce7
Author: Igor Sapego <is...@gridgain.com>
Authored: Wed May 31 18:36:52 2017 +0300
Committer: Igor Sapego <is...@gridgain.com>
Committed: Wed May 31 18:36:52 2017 +0300
----------------------------------------------------------------------
modules/platforms/cpp/examples/Makefile.am | 1 +
.../cpp/examples/compute-example/Makefile.am | 59 +++++++
.../compute-example/config/compute-example.xml | 52 ++++++
.../project/vs/compute-example.vcxproj | 102 +++++++++++
.../project/vs/compute-example.vcxproj.filters | 27 +++
.../compute-example/src/compute_example.cpp | 177 +++++++++++++++++++
modules/platforms/cpp/examples/configure.ac | 1 +
.../project/vs/continuous-query-example.vcxproj | 4 -
.../src/continuous_query_example.cpp | 5 +-
.../examples/include/ignite/examples/address.h | 2 +-
.../include/ignite/examples/organization.h | 4 +-
.../cpp/examples/project/vs/ignite-examples.sln | 6 +
12 files changed, 430 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/Makefile.am
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/Makefile.am b/modules/platforms/cpp/examples/Makefile.am
index 8cfb999..29b140b 100644
--- a/modules/platforms/cpp/examples/Makefile.am
+++ b/modules/platforms/cpp/examples/Makefile.am
@@ -22,5 +22,6 @@ SUBDIRS = \
odbc-example \
query-example \
continuous-query-example \
+ compute-example \
include
http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/compute-example/Makefile.am
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/compute-example/Makefile.am b/modules/platforms/cpp/examples/compute-example/Makefile.am
new file mode 100644
index 0000000..d757bf7
--- /dev/null
+++ b/modules/platforms/cpp/examples/compute-example/Makefile.am
@@ -0,0 +1,59 @@
+##
+## Licensed to the Apache Software Foundation (ASF) under one or more
+## contributor license agreements. See the NOTICE file distributed with
+## this work for additional information regarding copyright ownership.
+## The ASF licenses this file to You under the Apache License, Version 2.0
+## (the "License"); you may not use this file except in compliance with
+## the License. You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+
+ACLOCAL_AMFLAGS =-I m4
+
+noinst_PROGRAMS = ignite-compute-example
+
+AM_CPPFLAGS = \
+ -I@top_srcdir@/include \
+ -I@top_srcdir@/../core/include \
+ -I@top_srcdir@/../core/os/linux/include \
+ -I@top_srcdir@/../common/include \
+ -I@top_srcdir@/../common/os/linux/include \
+ -I@top_srcdir@/../binary/include \
+ -I@top_srcdir@/../jni/include \
+ -I@top_srcdir@/../jni/os/linux/include \
+ -I$(JAVA_HOME)/include \
+ -I$(JAVA_HOME)/include/linux \
+ -DIGNITE_IMPL \
+ -D__STDC_LIMIT_MACROS \
+ -D__STDC_CONSTANT_MACROS
+
+AM_CXXFLAGS = \
+ -Wall \
+ -std=c++03
+
+ignite_compute_example_LDADD = \
+ @top_srcdir@/../core/libignite.la \
+ -lodbc \
+ -lpthread
+
+ignite_compute_example_LDFLAGS = \
+ -static-libtool-libs
+
+ignite_compute_example_SOURCES = \
+ src/compute_example.cpp
+
+run-check: check
+ ./ignite-compute-example -p
+
+clean-local: clean-check
+ $(RM) *.gcno *.gcda
+
+clean-check:
+ $(RM) $(ignite_compute_example_OBJECTS)
http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/compute-example/config/compute-example.xml
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/compute-example/config/compute-example.xml b/modules/platforms/cpp/examples/compute-example/config/compute-example.xml
new file mode 100644
index 0000000..1348b1c
--- /dev/null
+++ b/modules/platforms/cpp/examples/compute-example/config/compute-example.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:util="http://www.springframework.org/schema/util"
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd
+ http://www.springframework.org/schema/util
+ http://www.springframework.org/schema/util/spring-util.xsd">
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+
+ <!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. -->
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <!--
+ Ignite provides several options for automatic discovery that can be used
+ instead os static IP based discovery.
+ -->
+ <!-- Uncomment static IP finder to enable static-based discovery of initial nodes. -->
+ <!--<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">-->
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">
+ <property name="addresses">
+ <list>
+ <!-- In distributed environment, replace with actual host IP address. -->
+ <value>127.0.0.1:47550..47551</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </property>
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj b/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj
new file mode 100644
index 0000000..1b84a9c
--- /dev/null
+++ b/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj
@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <RootNamespace>igniteexamples</RootNamespace>
+ <ProjectGuid>{18BB0A18-8213-472A-81A0-9D9753697135}</ProjectGuid>
+ <ProjectName>compute-example</ProjectName>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v100</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v100</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level3</WarningLevel>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <AdditionalIncludeDirectories>$(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;..\..\..\include;..\..\..\..\jni\os\win\include;..\..\..\..\jni\include;..\..\..\..\common\os\win\include;..\..\..\..\common\include;..\..\..\..\binary\include;..\..\..\..\core\os\win\include;..\..\..\..\core\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ <AdditionalDependencies>jvm.lib;ignite.jni.lib;ignite.binary.lib;ignite.core.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalLibraryDirectories>..\..\..\..\project\vs\$(Platform)\$(Configuration)\;$(JAVA_HOME)\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+ </Link>
+ <PostBuildEvent>
+ <Command>copy "$(ProjectDir)..\..\..\..\project\vs\$(Platform)\$(Configuration)\ignite.common.dll" "$(OutDir)"
+copy "$(ProjectDir)..\..\..\..\project\vs\$(Platform)\$(Configuration)\ignite.core.dll" "$(OutDir)"</Command>
+ </PostBuildEvent>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level3</WarningLevel>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <AdditionalIncludeDirectories>$(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;..\..\..\include;..\..\..\..\jni\os\win\include;..\..\..\..\jni\include;..\..\..\..\common\os\win\include;..\..\..\..\common\include;..\..\..\..\binary\include;..\..\..\..\core\os\win\include;..\..\..\..\core\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ <AdditionalDependencies>jvm.lib;ignite.jni.lib;ignite.binary.lib;ignite.core.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalLibraryDirectories>..\..\..\..\project\vs\$(Platform)\$(Configuration)\;$(JAVA_HOME)\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+ </Link>
+ <PostBuildEvent>
+ <Command>copy "$(ProjectDir)..\..\..\..\project\vs\$(Platform)\$(Configuration)\ignite.common.dll" "$(OutDir)"
+copy "$(ProjectDir)..\..\..\..\project\vs\$(Platform)\$(Configuration)\ignite.core.dll" "$(OutDir)"</Command>
+ </PostBuildEvent>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\src\compute_example.cpp" />
+ </ItemGroup>
+ <ItemGroup>
+ <None Include="..\..\config\compute-example.xml" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj.filters
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj.filters b/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj.filters
new file mode 100644
index 0000000..d43cece
--- /dev/null
+++ b/modules/platforms/cpp/examples/compute-example/project/vs/compute-example.vcxproj.filters
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup>
+ <Filter Include="Source Files">
+ <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+ <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+ </Filter>
+ <Filter Include="Header Files">
+ <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+ <Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
+ </Filter>
+ <Filter Include="Config">
+ <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+ <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+ </Filter>
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\src\compute_example.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ </ItemGroup>
+ <ItemGroup>
+ <None Include="..\..\config\compute-example.xml">
+ <Filter>Config</Filter>
+ </None>
+ </ItemGroup>
+</Project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/compute-example/src/compute_example.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/compute-example/src/compute_example.cpp b/modules/platforms/cpp/examples/compute-example/src/compute_example.cpp
new file mode 100644
index 0000000..3426475
--- /dev/null
+++ b/modules/platforms/cpp/examples/compute-example/src/compute_example.cpp
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdint.h>
+#include <iostream>
+#include <sstream>
+
+#include <ignite/ignition.h>
+#include <ignite/compute/compute.h>
+
+using namespace ignite;
+
+/*
+ * Function class.
+ */
+class CountWords : public compute::ComputeFunc<int32_t>
+{
+ friend struct ignite::binary::BinaryType<CountWords>;
+public:
+ /*
+ * Default constructor.
+ */
+ CountWords() :
+ text()
+ {
+ // No-op.
+ }
+
+ /*
+ * Constructor.
+ *
+ * @param text Text.
+ */
+ CountWords(const std::string& text) :
+ text(text)
+ {
+ // No-op.
+ }
+
+ /**
+ * Callback.
+ * Counts number of words in provided text.
+ *
+ * @return Number of words in provided text.
+ */
+ virtual int32_t Call()
+ {
+ std::stringstream buf(text);
+ std::string word;
+
+ int32_t wordsCount = 0;
+ while (buf >> word)
+ ++wordsCount;
+
+ return wordsCount;
+ }
+
+private:
+ /** Text in which to count words. */
+ std::string text;
+};
+
+namespace ignite
+{
+ namespace binary
+ {
+ template<>
+ struct BinaryType<CountWords>
+ {
+ static int32_t GetTypeId()
+ {
+ return GetBinaryStringHashCode("CountWords");
+ }
+
+ static void GetTypeName(std::string& dst)
+ {
+ dst = "CountWords";
+ }
+
+ static int32_t GetFieldId(const char* name)
+ {
+ return GetBinaryStringHashCode(name);
+ }
+
+ static int32_t GetHashCode(const CountWords& obj)
+ {
+ return 0;
+ }
+
+ static bool IsNull(const CountWords& obj)
+ {
+ return false;
+ }
+
+ static void GetNull(CountWords& dst)
+ {
+ dst = CountWords("");
+ }
+
+ static void Write(BinaryWriter& writer, const CountWords& obj)
+ {
+ writer.RawWriter().WriteString(obj.text);
+ }
+
+ static void Read(BinaryReader& reader, CountWords& dst)
+ {
+ dst.text = reader.RawReader().ReadString();
+ }
+ };
+ }
+}
+
+int main()
+{
+ IgniteConfiguration cfg;
+
+ cfg.springCfgPath = "platforms/cpp/examples/compute-example/config/compute-example.xml";
+
+ try
+ {
+ // Start a node.
+ Ignite ignite = Ignition::Start(cfg);
+
+ std::cout << std::endl;
+ std::cout << ">>> Compute example started." << std::endl;
+ std::cout << std::endl;
+
+ // Get binding instance.
+ IgniteBinding binding = ignite.GetBinding();
+
+ // Registering our class as a compute function.
+ binding.RegisterComputeFunc<CountWords>();
+
+ // Get compute instance.
+ compute::Compute compute = ignite.GetCompute();
+
+ std::string testText = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
+
+ // Declaring counter.
+ CountWords counter(testText);
+
+ // Making call.
+ int32_t wordsCount = compute.Call<int32_t>(counter);
+
+ // Printing result.
+ std::cout << ">>> Text contains " << wordsCount << " words" << std::endl;
+
+ // Stop node.
+ Ignition::StopAll(false);
+ }
+ catch (IgniteError& err)
+ {
+ std::cout << "An error occurred: " << err.GetText() << std::endl;
+ }
+
+ std::cout << std::endl;
+ std::cout << ">>> Example finished, press 'Enter' to exit ..." << std::endl;
+ std::cout << std::endl;
+
+ std::cin.get();
+
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/configure.ac
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/configure.ac b/modules/platforms/cpp/examples/configure.ac
index 7aef863..a72471d 100644
--- a/modules/platforms/cpp/examples/configure.ac
+++ b/modules/platforms/cpp/examples/configure.ac
@@ -57,6 +57,7 @@ AC_CONFIG_FILES([ \
odbc-example/Makefile \
query-example/Makefile \
continuous-query-example/Makefile \
+ compute-example/Makefile \
])
AC_OUTPUT
http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/continuous-query-example/project/vs/continuous-query-example.vcxproj
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/continuous-query-example/project/vs/continuous-query-example.vcxproj b/modules/platforms/cpp/examples/continuous-query-example/project/vs/continuous-query-example.vcxproj
index dfc5f9d..585796b 100644
--- a/modules/platforms/cpp/examples/continuous-query-example/project/vs/continuous-query-example.vcxproj
+++ b/modules/platforms/cpp/examples/continuous-query-example/project/vs/continuous-query-example.vcxproj
@@ -48,8 +48,6 @@
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
- <PrecompiledHeader>
- </PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
@@ -72,8 +70,6 @@ copy "$(ProjectDir)..\..\..\..\project\vs\$(Platform)\$(Configuration)\ignite.co
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
- <PrecompiledHeader>
- </PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp b/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
index 2ca3163..3b17b53 100644
--- a/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
+++ b/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
@@ -18,10 +18,9 @@
#include <stdint.h>
#include <iostream>
-#include "ignite/ignition.h"
-#include "ignite/cache/query/continuous/continuous_query.h"
+#include <ignite/ignition.h>
+#include <ignite/cache/query/continuous/continuous_query.h>
-#include "ignite/examples/organization.h"
#include "ignite/examples/person.h"
using namespace ignite;
http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/include/ignite/examples/address.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/include/ignite/examples/address.h b/modules/platforms/cpp/examples/include/ignite/examples/address.h
index 6a06f60..8f2689c 100644
--- a/modules/platforms/cpp/examples/include/ignite/examples/address.h
+++ b/modules/platforms/cpp/examples/include/ignite/examples/address.h
@@ -74,7 +74,7 @@ namespace ignite
return GetBinaryStringHashCode(name);
}
- static int32_t GetHashCode(ignite::examples::Address obj)
+ static int32_t GetHashCode(ignite::examples::Address& obj)
{
return 0;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/include/ignite/examples/organization.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/include/ignite/examples/organization.h b/modules/platforms/cpp/examples/include/ignite/examples/organization.h
index 942897d..2701933 100644
--- a/modules/platforms/cpp/examples/include/ignite/examples/organization.h
+++ b/modules/platforms/cpp/examples/include/ignite/examples/organization.h
@@ -83,12 +83,12 @@ namespace ignite
return GetBinaryStringHashCode(name);
}
- static int32_t GetHashCode(ignite::examples::Organization obj)
+ static int32_t GetHashCode(const ignite::examples::Organization& obj)
{
return 0;
}
- static bool IsNull(ignite::examples::Organization obj)
+ static bool IsNull(const ignite::examples::Organization& obj)
{
return false;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d0bddfa5/modules/platforms/cpp/examples/project/vs/ignite-examples.sln
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/project/vs/ignite-examples.sln b/modules/platforms/cpp/examples/project/vs/ignite-examples.sln
index a1865e3..bf83743 100644
--- a/modules/platforms/cpp/examples/project/vs/ignite-examples.sln
+++ b/modules/platforms/cpp/examples/project/vs/ignite-examples.sln
@@ -9,6 +9,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "continuous-query-example",
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "put-get-example", "..\..\put-get-example\project\vs\put-get-example.vcxproj", "{34935DEC-80FC-4168-AA52-3DBFF4F79B6B}"
EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "compute-example", "..\..\compute-example\project\vs\compute-example.vcxproj", "{18BB0A18-8213-472A-81A0-9D9753697135}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Release|x64 = Release|x64
@@ -31,6 +33,10 @@ Global
{34935DEC-80FC-4168-AA52-3DBFF4F79B6B}.Release|x64.Build.0 = Release|x64
{34935DEC-80FC-4168-AA52-3DBFF4F79B6B}.Release|x86.ActiveCfg = Release|Win32
{34935DEC-80FC-4168-AA52-3DBFF4F79B6B}.Release|x86.Build.0 = Release|Win32
+ {18BB0A18-8213-472A-81A0-9D9753697135}.Release|x64.ActiveCfg = Release|x64
+ {18BB0A18-8213-472A-81A0-9D9753697135}.Release|x64.Build.0 = Release|x64
+ {18BB0A18-8213-472A-81A0-9D9753697135}.Release|x86.ActiveCfg = Release|Win32
+ {18BB0A18-8213-472A-81A0-9D9753697135}.Release|x86.Build.0 = Release|Win32
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
[06/12] ignite git commit: IGNITE-5113: Implemented basic
distributed/local kmeans clusterization algorithm.
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java
new file mode 100644
index 0000000..7fb3534
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.math.*;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.Functions;
+import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.ignite.ml.clustering.KMeansUtil.checkIsInEpsilonNeighbourhood;
+
+/** */
+public class KMeansDistributedClustererTest extends GridCommonAbstractTest {
+ /** Number of nodes in grid. We should use 1 in this test because otherwise algorithm will be unstable
+ * (We cannot guarantee the order in which results are returned from each node). */
+ private static final int NODE_COUNT = 1;
+
+ /** Grid instance. */
+ private Ignite ignite;
+
+ /**
+ * Default constructor.
+ */
+ public KMeansDistributedClustererTest() {
+ super(false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void beforeTest() throws Exception {
+ ignite = grid(NODE_COUNT);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ for (int i = 1; i <= NODE_COUNT; i++)
+ startGrid(i);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+ }
+
+ /** */
+ @Test
+ public void testPerformClusterAnalysisDegenerate() {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ KMeansDistributedClusterer clusterer = new KMeansDistributedClusterer(new EuclideanDistance(), 1, 1, 1L);
+
+ double[] v1 = new double[] {1959, 325100};
+ double[] v2 = new double[] {1960, 373200};
+
+ SparseDistributedMatrix points = new SparseDistributedMatrix(2, 2, StorageConstants.ROW_STORAGE_MODE,
+ StorageConstants.RANDOM_ACCESS_MODE);
+
+ points.setRow(0, v1);
+ points.setRow(1, v2);
+
+ KMeansModel mdl = clusterer.cluster(points, 1);
+
+ Assert.assertEquals(1, mdl.centers().length);
+ Assert.assertEquals(2, mdl.centers()[0].size());
+ }
+
+ /** */
+ @Test
+ public void testClusterizationOnDatasetWithObviousStructure() throws IOException {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ int ptsCnt = 10000;
+ int squareSideLen = 10000;
+
+ Random rnd = new Random(123456L);
+
+ // Let centers be in the vertices of square.
+ Map<Integer, Vector> centers = new HashMap<>();
+ centers.put(100, new DenseLocalOnHeapVector(new double[] {0.0, 0.0}));
+ centers.put(900, new DenseLocalOnHeapVector(new double[] {squareSideLen, 0.0}));
+ centers.put(3000, new DenseLocalOnHeapVector(new double[] {0.0, squareSideLen}));
+ centers.put(6000, new DenseLocalOnHeapVector(new double[] {squareSideLen, squareSideLen}));
+
+ int centersCnt = centers.size();
+
+ SparseDistributedMatrix points = new SparseDistributedMatrix(ptsCnt, 2, StorageConstants.ROW_STORAGE_MODE,
+ StorageConstants.RANDOM_ACCESS_MODE);
+
+ List<Integer> permutation = IntStream.range(0, ptsCnt).boxed().collect(Collectors.toList());
+ Collections.shuffle(permutation, rnd);
+
+ Vector[] mc = new Vector[centersCnt];
+ Arrays.fill(mc, VectorUtils.zeroes(2));
+
+ int centIndex = 0;
+ int totalCount = 0;
+
+ List<Vector> massCenters = new ArrayList<>();
+
+ for (Integer count : centers.keySet()) {
+ for (int i = 0; i < count; i++) {
+ DenseLocalOnHeapVector pnt = (DenseLocalOnHeapVector)new DenseLocalOnHeapVector(2).assign(centers.get(count));
+ // pertrubate point on random value.
+ pnt.map(val -> val + rnd.nextDouble() * squareSideLen / 100);
+ mc[centIndex] = mc[centIndex].plus(pnt);
+ points.assignRow(permutation.get(totalCount), pnt);
+ totalCount++;
+ }
+ massCenters.add(mc[centIndex].times(1 / (double)count));
+ centIndex++;
+ }
+
+ EuclideanDistance dist = new EuclideanDistance();
+ OrderedNodesComparator comp = new OrderedNodesComparator(centers.values().toArray(new Vector[] {}), dist);
+
+ massCenters.sort(comp);
+ KMeansDistributedClusterer clusterer = new KMeansDistributedClusterer(dist, 3, 100, 1L);
+
+ KMeansModel mdl = clusterer.cluster(points, 4);
+ Vector[] resCenters = mdl.centers();
+ Arrays.sort(resCenters, comp);
+
+ checkIsInEpsilonNeighbourhood(resCenters, massCenters.toArray(new Vector[]{}), 30.0);
+ }
+
+ /** */
+ private static class OrderedNodesComparator implements Comparator<Vector> {
+ /** */
+ private final DistanceMeasure measure;
+
+ /** */
+ List<Vector> orderedNodes;
+
+ public OrderedNodesComparator(Vector[] orderedNodes, DistanceMeasure measure) {
+ this.orderedNodes = Arrays.asList(orderedNodes);
+ this.measure = measure;
+ }
+
+ /** */
+ private int findClosestNodeIndex(Vector v) {
+ return Functions.argmin(orderedNodes, v1 -> measure.compute(v1, v)).get1();
+ }
+
+ /** */
+ @Override public int compare(Vector v1, Vector v2) {
+ int ind1 = findClosestNodeIndex(v1);
+ int ind2 = findClosestNodeIndex(v2);
+
+ int signum = (int)Math.signum(ind1 - ind2);
+
+ if (signum != 0)
+ return signum;
+
+ return (int)Math.signum(orderedNodes.get(ind1).minus(v1).kNorm(2) -
+ orderedNodes.get(ind2).minus(v2).kNorm(2));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansLocalClustererTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansLocalClustererTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansLocalClustererTest.java
new file mode 100644
index 0000000..b396f5b
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansLocalClustererTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import org.apache.ignite.ml.math.EuclideanDistance;
+import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
+import org.junit.Assert;
+import org.junit.Test;
+
+/** */
+public class KMeansLocalClustererTest {
+ /**
+ * Two points, one cluster, one iteration
+ */
+ @Test
+ public void testPerformClusterAnalysisDegenerate() {
+ KMeansLocalClusterer clusterer = new KMeansLocalClusterer(new EuclideanDistance(), 1, 1L);
+
+ double[] v1 = new double[] {1959, 325100};
+ double[] v2 = new double[] {1960, 373200};
+
+ DenseLocalOnHeapMatrix points = new DenseLocalOnHeapMatrix(new double[][] {
+ v1,
+ v2});
+
+ KMeansModel mdl = clusterer.cluster(points, 1);
+
+ Assert.assertEquals(1, mdl.centers().length);
+ Assert.assertEquals(2, mdl.centers()[0].size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansUtil.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansUtil.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansUtil.java
new file mode 100644
index 0000000..0a39748
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import org.apache.ignite.ml.math.Vector;
+
+import static org.junit.Assert.assertTrue;
+
+/** Base test for k-means algorithms. */
+public class KMeansUtil {
+ /** */
+ public static void checkIsInEpsilonNeighbourhood(Vector[] v1s, Vector[] v2s, double epsilon) {
+ for (int i = 0; i < v1s.length; i++) {
+ assertTrue("Not in epsilon neighbourhood (index " + i + ") ",
+ v1s[i].minus(v2s[i]).kNorm(2) < epsilon);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java
index 216fd7b..d164be9 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java
@@ -59,6 +59,7 @@ import org.apache.ignite.ml.math.impls.vector.VectorIterableTest;
import org.apache.ignite.ml.math.impls.vector.VectorNormTest;
import org.apache.ignite.ml.math.impls.vector.VectorToMatrixTest;
import org.apache.ignite.ml.math.impls.vector.VectorViewTest;
+import org.apache.ignite.ml.regressions.OLSMultipleLinearRegressionTest;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
[10/12] ignite git commit: ignite-5075
Posted by sb...@apache.org.
ignite-5075
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/056b28fd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/056b28fd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/056b28fd
Branch: refs/heads/ignite-5075
Commit: 056b28fd3530691155156ac3c496223f5342629d
Parents: 70fc1a0
Author: sboikov <sb...@gridgain.com>
Authored: Thu Jun 1 11:41:31 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Jun 1 11:41:31 2017 +0300
----------------------------------------------------------------------
.../continuous/CacheContinuousQueryManager.java | 13 +--
...nuousQueryConcurrentPartitionUpdateTest.java | 106 +++++++++++--------
2 files changed, 69 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/056b28fd/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index d472054..184e872 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -829,14 +829,15 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
}
else {
synchronized (this) {
- added = lsnrs.putIfAbsent(lsnrId, lsnr) == null;
-
- if (added) {
- int cnt = lsnrCnt.incrementAndGet();
-
- if (cctx.group().sharedGroup() && cnt == 1 && !cctx.isLocal())
+ if (lsnrCnt.get() == 0) {
+ if (cctx.group().sharedGroup() && !cctx.isLocal())
cctx.group().addCacheWithContinuousQuery(cctx);
}
+
+ added = lsnrs.putIfAbsent(lsnrId, lsnr) == null;
+
+ if (added)
+ lsnrCnt.incrementAndGet();
}
if (added)
http://git-wip-us.apache.org/repos/asf/ignite/blob/056b28fd/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
index 32320f6..34cb777 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
@@ -35,6 +35,7 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -129,7 +130,7 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
caches.add(cache.getName());
- cntrs.add(startListener(cache));
+ cntrs.add(startListener(cache).get1());
}
}
else {
@@ -142,7 +143,7 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
caches.add(cache.getName());
- cntrs.add(startListener(cache));
+ cntrs.add(startListener(cache).get1());
}
Affinity<Integer> aff = srv.affinity(caches.get(0));
@@ -206,7 +207,7 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
* @param cache Cache.
* @return Event counter.
*/
- private AtomicInteger startListener(IgniteCache<Object, Object> cache) {
+ private T2<AtomicInteger, QueryCursor> startListener(IgniteCache<Object, Object> cache) {
final AtomicInteger evtCnt = new AtomicInteger();
ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
@@ -217,14 +218,15 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
assertNotNull(evt.getKey());
assertNotNull(evt.getValue());
- evtCnt.incrementAndGet();
+ if ((Integer)evt.getValue() >= 0)
+ evtCnt.incrementAndGet();
}
}
});
- cache.query(qry);
+ QueryCursor cur = cache.query(qry);
- return evtCnt;
+ return new T2<>(evtCnt, cur);
}
/**
@@ -267,14 +269,33 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
Ignite client = startGrid(1);
- CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+ List<String> caches = new ArrayList<>();
+
+ if (cacheGrp) {
+ for (int i = 0; i < 3; i++) {
+ CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME + i);
- ccfg.setWriteSynchronizationMode(FULL_SYNC);
- ccfg.setAtomicityMode(atomicityMode);
+ ccfg.setGroupName("testGroup");
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg.setAtomicityMode(atomicityMode);
- IgniteCache clientCache = client.createCache(ccfg);
+ IgniteCache cache = client.createCache(ccfg);
- Affinity<Integer> aff = srv.affinity(DEFAULT_CACHE_NAME);
+ caches.add(cache.getName());
+ }
+ }
+ else {
+ CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg.setAtomicityMode(atomicityMode);
+
+ IgniteCache cache = client.createCache(ccfg);
+
+ caches.add(cache.getName());
+ }
+
+ Affinity<Integer> aff = srv.affinity(caches.get(0));
final List<Integer> keys = new ArrayList<>();
@@ -294,38 +315,27 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
final int THREADS = 10;
final int UPDATES = 1000;
- for (int i = 0; i < 5; i++) {
- log.info("Iteration: " + i);
-
- ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
-
- final AtomicInteger evtCnt = new AtomicInteger();
-
- qry.setLocalListener(new CacheEntryUpdatedListener<Object, Object>() {
- @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts) {
- for (CacheEntryEvent evt : evts) {
- assertNotNull(evt.getKey());
- assertNotNull(evt.getValue());
-
- if ((Integer)evt.getValue() >= 0)
- evtCnt.incrementAndGet();
- }
- }
- });
+ final List<IgniteCache<Object, Object>> srvCaches = new ArrayList<>();
- QueryCursor cur;
+ for (String cacheName : caches)
+ srvCaches.add(srv.cache(cacheName));
- final IgniteCache<Object, Object> srvCache = srv.cache(DEFAULT_CACHE_NAME);
+ for (int i = 0; i < 5; i++) {
+ log.info("Iteration: " + i);
final AtomicBoolean stop = new AtomicBoolean();
+ List<T2<AtomicInteger, QueryCursor> > qrys = new ArrayList<>();
+
try {
IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
@Override public Void call() throws Exception {
ThreadLocalRandom rnd = ThreadLocalRandom.current();
- while (!stop.get())
- srvCache.put(keys.get(rnd.nextInt(KEYS)), rnd.nextInt(100) - 200);
+ while (!stop.get()) {
+ for (IgniteCache<Object, Object> srvCache : srvCaches)
+ srvCache.put(keys.get(rnd.nextInt(KEYS)), rnd.nextInt(100) - 200);
+ }
return null;
}
@@ -333,7 +343,8 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
U.sleep(1000);
- cur = clientCache.query(qry);
+ for (String cache : caches)
+ qrys.add(startListener(client.cache(cache)));
U.sleep(1000);
@@ -349,25 +360,32 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
@Override public Void call() throws Exception {
ThreadLocalRandom rnd = ThreadLocalRandom.current();
- for (int i = 0; i < UPDATES; i++)
- srvCache.put(keys.get(rnd.nextInt(KEYS)), i);
+ for (int i = 0; i < UPDATES; i++) {
+ for (IgniteCache<Object, Object> srvCache : srvCaches)
+ srvCache.put(keys.get(rnd.nextInt(KEYS)), i);
+ }
return null;
}
}, THREADS, "update");
+ for (T2<AtomicInteger, QueryCursor> qry : qrys) {
+ final AtomicInteger evtCnt = qry.get1();
- GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- log.info("Events: " + evtCnt.get());
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ log.info("Events: " + evtCnt.get());
- return evtCnt.get() >= THREADS * UPDATES;
- }
- }, 5000);
+ return evtCnt.get() >= THREADS * UPDATES;
+ }
+ }, 5000);
+ }
- assertEquals(THREADS * UPDATES, evtCnt.get());
+ for (T2<AtomicInteger, QueryCursor> qry : qrys) {
+ assertEquals(THREADS * UPDATES, qry.get1().get());
- cur.close();
+ qry.get2().close();
+ }
}
}
}
[02/12] ignite git commit: IGNITE-5342 - Skip permission check for
TASK_EXECUTE for service jobs
Posted by sb...@apache.org.
IGNITE-5342 - Skip permission check for TASK_EXECUTE for service jobs
(cherry picked from commit d77a134)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/858d11b3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/858d11b3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/858d11b3
Branch: refs/heads/ignite-5075
Commit: 858d11b33e14c799d90134cf936384532c2d0f7a
Parents: a7e5660
Author: dkarachentsev <dk...@gridgain.com>
Authored: Tue May 30 19:00:47 2017 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Wed May 31 16:10:45 2017 +0300
----------------------------------------------------------------------
.../processors/affinity/GridAffinityProcessor.java | 2 +-
.../internal/processors/cache/GridCacheAdapter.java | 3 ++-
.../cache/datastructures/CacheDataStructuresManager.java | 4 ++--
.../processors/closure/GridClosureProcessor.java | 11 ++++++++---
.../processors/service/GridServiceProcessor.java | 4 ++--
.../internal/processors/service/GridServiceProxy.java | 4 ++--
.../internal/processors/task/GridTaskProcessor.java | 7 ++++---
.../processors/task/GridTaskThreadContextKey.java | 5 ++++-
.../internal/IgniteComputeTopologyExceptionTest.java | 2 +-
9 files changed, 26 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
index e57cf54..87c424a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
@@ -499,7 +499,7 @@ public class GridAffinityProcessor extends GridProcessorAdapter {
private AffinityInfo affinityInfoFromNode(String cacheName, AffinityTopologyVersion topVer, ClusterNode n)
throws IgniteCheckedException {
GridTuple3<GridAffinityMessage, GridAffinityMessage, GridAffinityAssignment> t = ctx.closure()
- .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0).get();
+ .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0, false).get();
AffinityFunction f = (AffinityFunction)unmarshall(ctx, n.id(), t.get1());
AffinityKeyMapper m = (AffinityKeyMapper)unmarshall(ctx, n.id(), t.get2());
http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 307c5cf..ff25dd9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -3541,7 +3541,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
new LoadKeysCallable<>(ctx.name(), keys, update, plc, keepBinary),
nodes,
true,
- 0);
+ 0,
+ false);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index 45f0cee..0b81e7a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -526,7 +526,7 @@ public class CacheDataStructuresManager extends GridCacheManagerAdapter {
new BlockSetCallable(cctx.name(), id),
nodes,
true,
- 0).get();
+ 0, false).get();
}
catch (IgniteCheckedException e) {
if (e.hasCause(ClusterTopologyCheckedException.class)) {
@@ -550,7 +550,7 @@ public class CacheDataStructuresManager extends GridCacheManagerAdapter {
new RemoveSetDataCallable(cctx.name(), id, topVer),
nodes,
true,
- 0).get();
+ 0, false).get();
}
catch (IgniteCheckedException e) {
if (e.hasCause(ClusterTopologyCheckedException.class)) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index 1051807..40e21a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -75,6 +75,7 @@ import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.compute.ComputeJobResultPolicy.FAILOVER;
import static org.apache.ignite.compute.ComputeJobResultPolicy.REDUCE;
import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TIMEOUT;
@@ -558,12 +559,13 @@ public class GridClosureProcessor extends GridProcessorAdapter {
}
/**
+ * @param <R> Type.
* @param mode Distribution mode.
* @param job Closure to execute.
* @param nodes Grid nodes.
* @param sys If {@code true}, then system pool will be used.
* @param timeout Timeout.
- * @param <R> Type.
+ * @param skipAuth Skip authorization check.
* @return Grid future for collection of closure results.
*/
public <R> IgniteInternalFuture<R> callAsyncNoFailover(
@@ -571,8 +573,8 @@ public class GridClosureProcessor extends GridProcessorAdapter {
@Nullable Callable<R> job,
@Nullable Collection<ClusterNode> nodes,
boolean sys,
- long timeout
- ) {
+ long timeout,
+ boolean skipAuth) {
assert mode != null;
assert timeout >= 0 : timeout;
@@ -588,6 +590,9 @@ public class GridClosureProcessor extends GridProcessorAdapter {
ctx.task().setThreadContext(TC_NO_FAILOVER, true);
ctx.task().setThreadContext(TC_SUBGRID, nodes);
+ if (skipAuth)
+ ctx.task().setThreadContext(TC_SKIP_AUTH, true);
+
if (timeout > 0)
ctx.task().setThreadContext(TC_TIMEOUT, timeout);
http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index dda4328..c2fee02 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -708,8 +708,8 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
call,
Collections.singletonList(node),
false,
- timeout
- ).get();
+ timeout,
+ true).get();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index d2e96ba..d16a4c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -192,8 +192,8 @@ public class GridServiceProxy<T> implements Serializable {
new ServiceProxyCallable(mtd.getName(), name, mtd.getParameterTypes(), args),
Collections.singleton(node),
false,
- waitTimeout
- ).get();
+ waitTimeout,
+ true).get();
}
}
catch (GridServiceNotFoundException | ClusterTopologyCheckedException e) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 22d5716..e73d292 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -59,7 +59,6 @@ import org.apache.ignite.internal.managers.communication.GridMessageListener;
import org.apache.ignite.internal.managers.deployment.GridDeployment;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.GridProcessorAdapter;
-import org.apache.ignite.internal.processors.cache.CachePeekModes;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.util.GridConcurrentFactory;
import org.apache.ignite.internal.util.GridSpinReadWriteLock;
@@ -82,6 +81,7 @@ import static org.apache.ignite.internal.GridTopic.TOPIC_JOB_SIBLINGS;
import static org.apache.ignite.internal.GridTopic.TOPIC_TASK;
import static org.apache.ignite.internal.GridTopic.TOPIC_TASK_CANCEL;
import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBJ_ID;
import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TASK_NAME;
@@ -533,8 +533,6 @@ public class GridTaskProcessor extends GridProcessorAdapter {
else
taskClsName = taskCls != null ? taskCls.getName() : taskName;
- ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
-
// Get values from thread-local context.
Map<GridTaskThreadContextKey, Object> map = thCtx.get();
@@ -544,6 +542,9 @@ public class GridTaskProcessor extends GridProcessorAdapter {
// Reset thread-local context.
thCtx.set(null);
+ if (map.get(TC_SKIP_AUTH) == null)
+ ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
+
Long timeout = (Long)map.get(TC_TIMEOUT);
long timeout0 = timeout == null || timeout == 0 ? Long.MAX_VALUE : timeout;
http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
index 2ec63df..f0e56c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
@@ -37,5 +37,8 @@ public enum GridTaskThreadContextKey {
TC_SUBJ_ID,
/** IO manager policy. */
- TC_IO_POLICY
+ TC_IO_POLICY,
+
+ /** Skip authorization for the task. */
+ TC_SKIP_AUTH
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/858d11b3/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
index 3ed91e8..a82373b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
@@ -89,7 +89,7 @@ public class IgniteComputeTopologyExceptionTest extends GridCommonAbstractTest {
},
nodes,
false,
- 0);
+ 0, false);
try {
fut.get();
[07/12] ignite git commit: IGNITE-5113: Implemented basic
distributed/local kmeans clusterization algorithm.
Posted by sb...@apache.org.
IGNITE-5113: Implemented basic distributed/local kmeans clusterization algorithm.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b04b5800
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b04b5800
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b04b5800
Branch: refs/heads/ignite-5075
Commit: b04b58005d32debc3ae54dd3b03289fad1a590bd
Parents: d0bddfa
Author: Artem Malykh <am...@gridgain.com>
Authored: Fri May 19 19:40:12 2017 +0300
Committer: agura <ag...@apache.org>
Committed: Wed May 31 19:54:48 2017 +0300
----------------------------------------------------------------------
modules/ml/pom.xml | 13 +
.../main/java/org/apache/ignite/ml/Model.java | 39 +++
.../ml/clustering/BaseKMeansClusterer.java | 98 ++++++
.../apache/ignite/ml/clustering/Clusterer.java | 32 ++
.../ml/clustering/ClusterizationModel.java | 29 ++
.../clustering/KMeansDistributedClusterer.java | 298 +++++++++++++++++++
.../ml/clustering/KMeansLocalClusterer.java | 174 +++++++++++
.../ignite/ml/clustering/KMeansModel.java | 79 +++++
.../ignite/ml/clustering/WeightedClusterer.java | 38 +++
.../ignite/ml/clustering/package-info.java | 22 ++
.../apache/ignite/ml/math/DistanceMeasure.java | 39 +++
.../ignite/ml/math/EuclideanDistance.java | 48 +++
.../org/apache/ignite/ml/math/MathUtils.java | 31 ++
.../java/org/apache/ignite/ml/math/Matrix.java | 2 +
.../apache/ignite/ml/math/StorageConstants.java | 3 +
.../org/apache/ignite/ml/math/VectorUtils.java | 41 +++
.../math/exceptions/ConvergenceException.java | 48 +++
.../exceptions/MathArithmeticException.java | 6 +-
.../exceptions/MathIllegalNumberException.java | 51 ++++
.../exceptions/MathIllegalStateException.java | 49 +++
.../exceptions/NumberIsTooSmallException.java | 79 +++++
.../ignite/ml/math/functions/Functions.java | 39 +++
.../apache/ignite/ml/math/impls/CacheUtils.java | 47 +++
.../ml/math/impls/matrix/AbstractMatrix.java | 2 +
.../impls/matrix/SparseDistributedMatrix.java | 2 +-
.../impls/storage/matrix/MapWrapperStorage.java | 93 ++++++
.../vector/SparseLocalOnHeapVectorStorage.java | 32 ++
.../ml/math/impls/vector/DelegatingVector.java | 5 +
.../ml/math/impls/vector/MapWrapperVector.java | 32 ++
.../ml/math/impls/vector/SparseLocalVector.java | 9 +
.../ignite/ml/math/impls/vector/VectorView.java | 1 +
.../ignite/ml/math/statistics/Variance.java | 53 ++++
.../org/apache/ignite/ml/math/util/MapUtil.java | 38 +++
.../apache/ignite/ml/math/util/MatrixUtil.java | 38 ++-
.../org/apache/ignite/ml/IgniteMLTestSuite.java | 4 +-
.../ml/clustering/ClusteringTesetSuite.java | 15 +
.../KMeansDistributedClustererTest.java | 184 ++++++++++++
.../ml/clustering/KMeansLocalClustererTest.java | 46 +++
.../apache/ignite/ml/clustering/KMeansUtil.java | 33 ++
.../ignite/ml/math/MathImplLocalTestSuite.java | 1 +
40 files changed, 1885 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/pom.xml
----------------------------------------------------------------------
diff --git a/modules/ml/pom.xml b/modules/ml/pom.xml
index 1df5bb6..75ebf1b 100644
--- a/modules/ml/pom.xml
+++ b/modules/ml/pom.xml
@@ -85,6 +85,19 @@
<version>${spring.version}</version>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-rng-core</artifactId>
+ <version>1.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-rng-simple</artifactId>
+ <version>1.0</version>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/Model.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/Model.java b/modules/ml/src/main/java/org/apache/ignite/ml/Model.java
new file mode 100644
index 0000000..e4d2117
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/Model.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml;
+
+import java.util.function.BiFunction;
+
+
+/** Basic interface for all models. */
+@FunctionalInterface
+public interface Model<T, V> {
+ /** Predict a result for value. */
+ public V predict(T val);
+
+ /**
+ * Combines this model with other model via specified combiner
+ *
+ * @param other Other model.
+ * @param combiner Combiner.
+ * @return Combination of models.
+ */
+ public default <X, W> Model<T, X> combine(Model<T, W> other, BiFunction<V, W, X> combiner) {
+ return v -> combiner.apply(predict(v), other.predict(v));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java
new file mode 100644
index 0000000..a6acb8e
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.util.List;
+
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.ml.math.DistanceMeasure;
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.exceptions.ConvergenceException;
+import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
+import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
+
+/**
+ * This class is partly based on the corresponding class from Apache Common Math lib.
+ */
+public abstract class BaseKMeansClusterer<T extends Matrix> implements Clusterer<T, KMeansModel> {
+ /** The distance measure to use. */
+ private DistanceMeasure measure;
+
+ /**
+ * Build a new clusterer with the given {@link DistanceMeasure}.
+ *
+ * @param measure the distance measure to use
+ */
+ protected BaseKMeansClusterer(final DistanceMeasure measure) {
+ this.measure = measure;
+ }
+
+ /**
+ * Perform a cluster analysis on the given set of points.
+ *
+ * @param points the set of points
+ * @return a {@link List} of clusters
+ * @throws MathIllegalArgumentException if points are null or the number of data points is not compatible with this
+ * clusterer
+ * @throws ConvergenceException if the algorithm has not yet converged after the maximum number of iterations has
+ * been exceeded
+ */
+ public abstract KMeansModel cluster(T points, int k)
+ throws MathIllegalArgumentException, ConvergenceException;
+
+ /**
+ * Returns the {@link DistanceMeasure} instance used by this clusterer.
+ *
+ * @return the distance measure
+ */
+ public DistanceMeasure getDistanceMeasure() {
+ return measure;
+ }
+
+ /**
+ * Calculates the distance between two vectors.
+ * with the configured {@link DistanceMeasure}.
+ *
+ * @return the distance between the two clusterables
+ */
+ protected double distance(final Vector v1, final Vector v2) {
+ return measure.compute(v1, v2);
+ }
+
+ /**
+ * Find the closest cluster center index and distance to it from a given point.
+ *
+ * @param centers Centers to look in.
+ * @param pnt Point.
+ */
+ protected IgniteBiTuple<Integer, Double> findClosest(Vector[] centers, Vector pnt) {
+ double bestDistance = Double.POSITIVE_INFINITY;
+ int bestInd = 0;
+
+ for (int i = 0; i < centers.length; i++) {
+ double dist = distance(centers[i], pnt);
+ if (dist < bestDistance) {
+ bestDistance = dist;
+ bestInd = i;
+ }
+ }
+
+ return new IgniteBiTuple<>(bestInd, bestDistance);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java
new file mode 100644
index 0000000..f03dc95
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import org.apache.ignite.ml.Model;
+
+/**
+ * Base interface for clusterers.
+ */
+public interface Clusterer<P, M extends Model> {
+ /** Cluster given points set into k clusters.
+ *
+ * @param points Points set.
+ * @param k Clusters count.
+ */
+ public M cluster(P points, int k);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/ClusterizationModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/ClusterizationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/ClusterizationModel.java
new file mode 100644
index 0000000..99afec5
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/ClusterizationModel.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import org.apache.ignite.ml.Model;
+
+/** Base interface for all clusterization models. */
+public interface ClusterizationModel<P, V> extends Model<P, V> {
+ /** Gets the clusters count. */
+ public int clustersCount();
+
+ /** Get cluster centers. */
+ public P[] centers();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
new file mode 100644
index 0000000..09317d6
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.ml.math.*;
+import org.apache.ignite.ml.math.exceptions.ConvergenceException;
+import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
+import org.apache.ignite.ml.math.functions.Functions;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.impls.CacheUtils;
+import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
+import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
+import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
+import org.apache.ignite.ml.math.util.MapUtil;
+import org.apache.ignite.ml.math.util.MatrixUtil;
+
+import javax.cache.Cache;
+
+import static org.apache.ignite.ml.math.impls.CacheUtils.*;
+import static org.apache.ignite.ml.math.util.MatrixUtil.localCopyOf;
+
+/**
+ * Clustering algorithm based on Bahmani et al. paper and Apache Spark class with corresponding functionality.
+ *
+ * @see <a href="http://theory.stanford.edu/~sergei/papers/vldb12-kmpar.pdf">Scalable K-Means++(wikipedia)</a>
+ */
+public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistributedMatrix> {
+ /** */
+ private final int maxIterations;
+
+ /** */
+ private Random rnd;
+
+ /** */
+ private int initSteps;
+
+ /** */
+ private long seed;
+
+ /** */
+ private double epsilon = 1e-4;
+
+ /** */
+ public KMeansDistributedClusterer(DistanceMeasure measure, int initSteps, int maxIterations, Long seed) {
+ super(measure);
+ this.initSteps = initSteps;
+
+ this.seed = seed != null ? seed : new Random().nextLong();
+
+ this.maxIterations = maxIterations;
+ rnd = new Random(this.seed);
+ }
+
+ /** */
+ @Override public KMeansModel cluster(SparseDistributedMatrix points, int k) throws
+ MathIllegalArgumentException, ConvergenceException {
+ SparseDistributedMatrix pointsCp = (SparseDistributedMatrix) points.like(points.rowSize(), points.columnSize());
+
+ // TODO: this copy is very ineffective, just for POC. Immutability of data should be guaranteed by other methods
+ // such as logical locks for example.
+ pointsCp.assign(points);
+
+ Vector[] centers = initClusterCenters(pointsCp, k);
+
+ boolean converged = false;
+ int iteration = 0;
+ int dim = pointsCp.viewRow(0).size();
+ IgniteUuid uid = pointsCp.getUUID();
+
+ // Execute iterations of Lloyd's algorithm until converged
+ while (iteration < maxIterations && !converged) {
+ SumsAndCounts stats = getSumsAndCounts(centers, dim, uid);
+
+ converged = true;
+
+ for (Integer ind : stats.sums.keySet()) {
+ Vector massCenter = stats.sums.get(ind).times(1.0 / stats.counts.get(ind));
+
+ if (converged && distance(massCenter, centers[ind]) > epsilon * epsilon)
+ converged = false;
+
+ centers[ind] = massCenter;
+ }
+
+ iteration++;
+ }
+
+ pointsCp.destroy();
+
+ return new KMeansModel(centers, getDistanceMeasure());
+ }
+
+ /** Initialize cluster centers. */
+ private Vector[] initClusterCenters(SparseDistributedMatrix points, int k) {
+ // Initialize empty centers and point costs.
+ int ptsCount = points.rowSize();
+
+ // Initialize the first center to a random point.
+ Vector sample = localCopyOf(points.viewRow(rnd.nextInt(ptsCount)));
+
+ List<Vector> centers = new ArrayList<>();
+ List<Vector> newCenters = new ArrayList<>();
+ newCenters.add(sample);
+ centers.add(sample);
+
+ final ConcurrentHashMap<Integer, Double> costs = new ConcurrentHashMap<>();
+
+ // On each step, sample 2 * k points on average with probability proportional
+ // to their squared distance from the centers. Note that only distances between points
+ // and new centers are computed in each iteration.
+ int step = 0;
+ IgniteUuid uid = points.getUUID();
+
+ while (step < initSteps) {
+ // We assume here that costs can fit into memory of one node.
+ ConcurrentHashMap<Integer, Double> newCosts = getNewCosts(points, newCenters);
+
+ // Merge costs with new costs.
+ for (Integer ind : newCosts.keySet())
+ costs.merge(ind, newCosts.get(ind), Math::min);
+
+ double sumCosts = costs.values().stream().mapToDouble(Double::valueOf).sum();
+
+ newCenters = getNewCenters(k, costs, uid, sumCosts);
+ centers.addAll(newCenters);
+
+ step++;
+ }
+
+ List<Vector> distinctCenters = centers.stream().distinct().collect(Collectors.toList());
+
+ if (distinctCenters.size() <= k)
+ return distinctCenters.toArray(new Vector[] {});
+ else {
+ // Finally, we might have a set of more than k distinct candidate centers; weight each
+ // candidate by the number of points in the dataset mapping to it and run a local k-means++
+ // on the weighted centers to pick k of them
+ ConcurrentHashMap<Integer, Integer> centerInd2Weight = weightCenters(uid, distinctCenters);
+
+ List<Double> weights = new ArrayList<>(centerInd2Weight.size());
+
+ for (int i = 0; i < distinctCenters.size(); i++)
+ weights.add(i, Double.valueOf(centerInd2Weight.getOrDefault(i, 0)));
+
+ DenseLocalOnHeapMatrix dCenters = MatrixUtil.fromList(distinctCenters, true);
+
+ return new KMeansLocalClusterer(getDistanceMeasure(), 30, seed).cluster(dCenters, k, weights).centers();
+ }
+ }
+
+ /** */
+ private List<Vector> getNewCenters(int k, ConcurrentHashMap<Integer, Double> costs, IgniteUuid uid, double sumCosts) {
+ return distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME,
+ (IgniteBiFunction<Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>>,
+ List<Vector>,
+ List<Vector>>)(vectorWithIndex, list) -> {
+ Integer ind = vectorWithIndex.getKey().get1();
+
+ double prob = costs.get(ind) * 2.0 * k / sumCosts;
+
+ if (new Random(seed ^ ind).nextDouble() < prob)
+ list.add(VectorUtils.fromMap(vectorWithIndex.getValue(), false));
+
+ return list;
+ },
+ key -> key.get2().equals(uid),
+ (list1, list2) -> {
+ list1.addAll(list2);
+ return list1;
+ },
+ new ArrayList<>()
+ );
+ }
+
+ /** */
+ private ConcurrentHashMap<Integer, Double> getNewCosts(SparseDistributedMatrix points, List<Vector> newCenters) {
+ return distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME,
+ (IgniteBiFunction<Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, ConcurrentHashMap<Integer, Double>>,
+ ConcurrentHashMap<Integer, Double>,
+ ConcurrentHashMap<Integer, Double>>)(vectorWithIndex, map) -> {
+ for (Vector center : newCenters)
+ map.merge(vectorWithIndex.getKey().get1(), distance(vectorWithIndex.getValue(), center), Functions.MIN);
+
+ return map;
+ },
+ key -> key.get2().equals(points.getUUID()),
+ (map1, map2) -> {
+ map1.putAll(map2);
+ return map1;
+ }, new ConcurrentHashMap<>());
+ }
+
+ /** */
+ private ConcurrentHashMap<Integer, Integer> weightCenters(IgniteUuid uid, List<Vector> distinctCenters) {
+ return distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME,
+ (IgniteBiFunction<Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>>,
+ ConcurrentHashMap<Integer, Integer>,
+ ConcurrentHashMap<Integer, Integer>>)(vectorWithIndex, countMap) -> {
+ Integer resInd = -1;
+ Double resDist = Double.POSITIVE_INFINITY;
+
+ int i = 0;
+ for (Vector cent : distinctCenters) {
+ double curDist = distance(vectorWithIndex.getValue(), cent);
+
+ if (resDist > curDist) {
+ resDist = curDist;
+ resInd = i;
+ }
+
+ i++;
+ }
+
+ countMap.compute(resInd, (ind, v) -> v != null ? v + 1 : 1);
+ return countMap;
+ },
+ key -> key.get2().equals(uid),
+ (map1, map2) -> MapUtil.mergeMaps(map1, map2, (integer, integer2) -> integer2 + integer,
+ ConcurrentHashMap::new),
+ new ConcurrentHashMap<>());
+ }
+
+ /** */
+ private double distance(Map<Integer, Double> vecMap, Vector vector) {
+ return distance(VectorUtils.fromMap(vecMap, false), vector);
+ }
+
+ /** */
+ private SumsAndCounts getSumsAndCounts(Vector[] centers, int dim, IgniteUuid uid) {
+ return CacheUtils.distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME,
+ (IgniteBiFunction<Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>>, SumsAndCounts, SumsAndCounts>)(entry, counts) -> {
+ Map<Integer, Double> vec = entry.getValue();
+
+ IgniteBiTuple<Integer, Double> closest = findClosest(centers, VectorUtils.fromMap(vec, false));
+ int bestCenterIdx = closest.get1();
+
+ counts.totalCost += closest.get2();
+ counts.sums.putIfAbsent(bestCenterIdx, VectorUtils.zeroes(dim));
+
+ counts.sums.compute(bestCenterIdx,
+ (IgniteBiFunction<Integer, Vector, Vector>)(ind, v) -> v.plus(VectorUtils.fromMap(vec, false)));
+
+ counts.counts.merge(bestCenterIdx, 1,
+ (IgniteBiFunction<Integer, Integer, Integer>)(i1, i2) -> i1 + i2);
+
+ return counts;
+ },
+ key -> key.get2().equals(uid),
+ SumsAndCounts::merge, new SumsAndCounts()
+ );
+ }
+
+ /** Service class used for statistics. */
+ private static class SumsAndCounts {
+ /** */
+ public double totalCost;
+
+ /** */
+ public ConcurrentHashMap<Integer, Vector> sums = new ConcurrentHashMap<>();
+
+ /** Count of points closest to the center with a given index. */
+ public ConcurrentHashMap<Integer, Integer> counts = new ConcurrentHashMap<>();
+
+ /** Merge current */
+ public SumsAndCounts merge(SumsAndCounts other) {
+ this.totalCost += totalCost;
+ MapUtil.mergeMaps(sums, other.sums, Vector::plus, ConcurrentHashMap::new);
+ MapUtil.mergeMaps(counts, other.counts, (i1, i2) -> i1 + i2, ConcurrentHashMap::new);
+ return this;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java
new file mode 100644
index 0000000..c98b818
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.ignite.internal.util.GridArgumentCheck;
+import org.apache.ignite.ml.math.*;
+import org.apache.ignite.ml.math.exceptions.ConvergenceException;
+import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
+import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
+
+import static org.apache.ignite.ml.math.util.MatrixUtil.localCopyOf;
+
+/**
+ * Perform clusterization on local data.
+ * This class is based on Apache Spark class with corresponding functionality.
+ */
+public class KMeansLocalClusterer extends BaseKMeansClusterer<DenseLocalOnHeapMatrix> implements
+ WeightedClusterer<DenseLocalOnHeapMatrix, KMeansModel> {
+ /** */
+ private int maxIterations;
+
+ /** */
+ private Random rand;
+
+ /**
+ * Build a new clusterer with the given {@link DistanceMeasure}.
+ *
+ * @param measure Distance measure to use.
+ * @param maxIterations maximal number of iterations.
+ * @param seed Seed used in random parts of algorithm.
+ */
+ public KMeansLocalClusterer(DistanceMeasure measure, int maxIterations, Long seed) {
+ super(measure);
+ this.maxIterations = maxIterations;
+ rand = seed != null ? new Random(seed) : new Random();
+ }
+
+ /** {@inheritDoc} */
+ @Override public KMeansModel cluster(
+ DenseLocalOnHeapMatrix points, int k) throws MathIllegalArgumentException, ConvergenceException {
+ List<Double> ones = new ArrayList<>(Collections.nCopies(points.rowSize(), 1.0));
+ return cluster(points, k, ones);
+ }
+
+ /** {@inheritDoc} */
+ @Override public KMeansModel cluster(DenseLocalOnHeapMatrix points, int k,
+ List<Double> weights) throws MathIllegalArgumentException, ConvergenceException {
+
+ GridArgumentCheck.notNull(points, "points");
+
+ int dim = points.columnSize();
+ Vector[] centers = new Vector[k];
+
+ centers[0] = pickWeighted(points, weights);
+
+ Vector costs = points.foldRows(row -> distance(row,
+ centers[0]));
+
+ for (int i = 0; i < k; i++) {
+ double weightedSum = weightedSum(costs, weights);
+
+ double r = rand.nextDouble() * weightedSum;
+ double s = 0.0;
+ int j = 0;
+
+ while (j < points.rowSize() && s < r) {
+ s += weights.get(j) * costs.get(j);
+ j++;
+ }
+
+ if (j == 0)
+ // TODO: Process this case more carefully
+ centers[i] = localCopyOf(points.viewRow(0));
+ else
+ centers[i] = localCopyOf(points.viewRow(j - 1));
+
+ for (int p = 0; p < points.rowSize(); p++)
+ costs.setX(p, Math.min(getDistanceMeasure().compute(localCopyOf(points.viewRow(p)), centers[i]),
+ costs.get(p)));
+ }
+
+ int[] oldClosest = new int[points.rowSize()];
+ Arrays.fill(oldClosest, -1);
+ int iter = 0;
+ boolean moved = true;
+
+ while (moved && iter < maxIterations) {
+ moved = false;
+
+ double[] counts = new double[k];
+ Arrays.fill(counts, 0.0);
+ Vector[] sums = new Vector[k];
+
+ Arrays.fill(sums, VectorUtils.zeroes(dim));
+
+ int i = 0;
+
+ while (i < points.rowSize()) {
+ Vector p = localCopyOf(points.viewRow(i));
+
+ int ind = findClosest(centers, p).get1();
+ sums[ind] = sums[ind].plus(p.times(weights.get(i)));
+
+ counts[ind] += weights.get(i);
+ if (ind != oldClosest[i]) {
+ moved = true;
+ oldClosest[i] = ind;
+ }
+ i++;
+ }
+ // Update centers
+ int j = 0;
+ while (j < k) {
+ if (counts[j] == 0.0) {
+ // Assign center to a random point
+ centers[j] = points.viewRow(rand.nextInt(points.rowSize()));
+ } else {
+ sums[j] = sums[j].times(1.0 / counts[j]);
+ centers[j] = sums[j];
+ }
+ j++;
+ }
+ iter++;
+ }
+
+ return new KMeansModel(centers, getDistanceMeasure());
+ }
+
+ /** Pick a random vector with a probability proportional to the corresponding weight. */
+ private Vector pickWeighted(Matrix points, List<Double> weights) {
+ double r = rand.nextDouble() * weights.stream().mapToDouble(Double::valueOf).sum();
+
+ int i = 0;
+ double curWeight = 0.0;
+
+ while (i < points.rowSize() && curWeight < r) {
+ curWeight += weights.get(i);
+ i += 1;
+ }
+
+ return localCopyOf(points.viewRow(i - 1));
+ }
+
+ /** Get a weighted sum of a vector v. */
+ private double weightedSum(Vector v, List<Double> weights) {
+ double res = 0.0;
+
+ for (int i = 0; i < v.size(); i++)
+ res += v.getX(i) * weights.get(i);
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java
new file mode 100644
index 0000000..6584273
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.util.Arrays;
+import org.apache.ignite.ml.math.DistanceMeasure;
+import org.apache.ignite.ml.math.Vector;
+
+/**
+ * This class encapsulates result of clusterization.
+ */
+public class KMeansModel implements ClusterizationModel<Vector, Integer> {
+ /** Centers of clusters. */
+ private Vector[] centers;
+
+ /** Distance measure. */
+ private DistanceMeasure distance;
+
+ /**
+ * Construct KMeans model with given centers and distance measure.
+ *
+ * @param centers Centers.
+ * @param distance Distance measure.
+ */
+ KMeansModel(Vector[] centers, DistanceMeasure distance) {
+ this.centers = centers;
+ this.distance = distance;
+ }
+
+ /** Distance measure used while clusterization */
+ public DistanceMeasure distanceMeasure() {
+ return distance;
+ }
+
+ /** Count of centers in clusterization. */
+ @Override public int clustersCount() {
+ return centers.length;
+ }
+
+ /** Get centers of clusters. */
+ @Override public Vector[] centers() {
+ return Arrays.copyOf(centers, centers.length);
+ }
+
+ /**
+ * Predict closest center index for a given vector.
+ *
+ * @param vec Vector.
+ */
+ public Integer predict(Vector vec) {
+ int res = -1;
+ double minDist = Double.POSITIVE_INFINITY;
+
+ for (int i = 0; i < centers.length; i++) {
+ double curDist = distance.compute(centers[i], vec);
+ if (curDist < minDist) {
+ minDist = curDist;
+ res = i;
+ }
+ }
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/WeightedClusterer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/WeightedClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/WeightedClusterer.java
new file mode 100644
index 0000000..55fb359
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/WeightedClusterer.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.util.List;
+import org.apache.ignite.ml.Model;
+import org.apache.ignite.ml.math.exceptions.ConvergenceException;
+import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
+
+/**
+ * Support of clusterization with given weights.
+ */
+public interface WeightedClusterer<P, M extends Model> extends Clusterer<P, M> {
+ /**
+ * Perform clusterization of given points weighted by given weights.
+ *
+ * @param points Points.
+ * @param k count of centers.
+ * @param weights Weights.
+ */
+ public KMeansModel cluster(P points, int k, List<Double> weights) throws
+ MathIllegalArgumentException, ConvergenceException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/clustering/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/package-info.java
new file mode 100644
index 0000000..e83084a
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains clustering algorithms.
+ */
+package org.apache.ignite.ml.clustering;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java
new file mode 100644
index 0000000..0fd74ac
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.math;
+
+import java.io.Externalizable;
+
+import org.apache.ignite.ml.math.exceptions.CardinalityException;
+
+/**
+ * This class is based on the corresponding class from Apache Common Math lib.
+ * Interface for distance measures of n-dimensional vectors.
+ */
+public interface DistanceMeasure extends Externalizable {
+ /**
+ * Compute the distance between two n-dimensional vectors.
+ * <p>
+ * The two vectors are required to have the same dimension.
+ *
+ * @param a the first vector
+ * @param b the second vector
+ * @return the distance between the two vectors
+ * @throws CardinalityException if the array lengths differ.
+ */
+ double compute(Vector a, Vector b) throws CardinalityException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java
new file mode 100644
index 0000000..b748ac5
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.math;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.ml.math.exceptions.CardinalityException;
+import org.apache.ignite.ml.math.util.MatrixUtil;
+
+/**
+ * Calculates the L<sub>2</sub> (Euclidean) distance between two points.
+ */
+public class EuclideanDistance implements DistanceMeasure {
+ /** Serializable version identifier. */
+ private static final long serialVersionUID = 1717556319784040040L;
+
+ /** {@inheritDoc} */
+ @Override
+ public double compute(Vector a, Vector b)
+ throws CardinalityException {
+ return MatrixUtil.localCopyOf(a).minus(b).kNorm(2.0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ // No-op
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/MathUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/MathUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/MathUtils.java
new file mode 100644
index 0000000..c2164ab
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/MathUtils.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math;
+
+import org.apache.ignite.ml.math.exceptions.NullArgumentException;
+
+/**
+ * Miscellaneous utility functions.
+ */
+public final class MathUtils {
+ public static void checkNotNull(Object o)
+ throws NullArgumentException {
+ if (o == null)
+ throw new NullArgumentException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java
index 2cf4e63..db822e6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java
@@ -18,6 +18,8 @@
package org.apache.ignite.ml.math;
import java.io.Externalizable;
+import java.util.function.BiFunction;
+import java.util.function.BinaryOperator;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.ml.math.exceptions.CardinalityException;
import org.apache.ignite.ml.math.exceptions.IndexException;
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/StorageConstants.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/StorageConstants.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/StorageConstants.java
index ec2ee65..38d42ba 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/StorageConstants.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/StorageConstants.java
@@ -33,6 +33,9 @@ public interface StorageConstants {
/** Storage mode optimized for column access. */
public static final int COLUMN_STORAGE_MODE = 2002;
+ /** Storage mode is unknown. */
+ public static final int UNKNOWN_STORAGE_MODE = 3001;
+
/**
* @param mode Access mode to verify.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java
new file mode 100644
index 0000000..f41a5fe
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math;
+
+import java.util.Map;
+import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.math.impls.vector.MapWrapperVector;
+import org.apache.ignite.ml.math.impls.vector.SparseLocalVector;
+
+public class VectorUtils {
+ /** Create new vector like given vector initialized by zeroes. */
+ public static Vector zeroesLike(Vector v) {
+ return v.like(v.size()).assign(0.0);
+ }
+
+ /** Create new */
+ public static DenseLocalOnHeapVector zeroes(int n) {
+ return (DenseLocalOnHeapVector) new DenseLocalOnHeapVector(n).assign(0.0);
+ }
+
+ /** */
+ public static Vector fromMap(Map<Integer, Double> value, boolean copy) {
+ return new MapWrapperVector(value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/ConvergenceException.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/ConvergenceException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/ConvergenceException.java
new file mode 100644
index 0000000..2cf0bcf
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/ConvergenceException.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.math.exceptions;
+
+/**
+ * This class is based on the corresponding class from Apache Common Math lib.
+ * Error thrown when a numerical computation can not be performed because the
+ * numerical result failed to converge to a finite value.
+ */
+public class ConvergenceException extends MathIllegalStateException {
+ /** Serializable version Id. */
+ private static final long serialVersionUID = 4330003017885151975L;
+
+ /** */
+ private static final String CONVERGENCE_FAILED = "convergence failed";
+
+ /**
+ * Construct the exception.
+ */
+ public ConvergenceException() {
+ this(CONVERGENCE_FAILED);
+ }
+
+ /**
+ * Construct the exception with a specific context and arguments.
+ *
+ * @param msg Message pattern providing the specific context of
+ * the error.
+ * @param args Arguments.
+ */
+ public ConvergenceException(String msg, Object ... args) {
+ super(msg, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathArithmeticException.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathArithmeticException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathArithmeticException.java
index f48f3c5..ccd019c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathArithmeticException.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathArithmeticException.java
@@ -14,14 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.ignite.ml.math.exceptions;
/**
* This class is based on the corresponding class from Apache Common Math lib.
* Base class for arithmetic exceptions.
- * It is used for all the exceptions that have the semantics of the standard
- * {@link ArithmeticException}, but must also provide a localized
- * message.
*/
public class MathArithmeticException extends MathRuntimeException {
/** Serializable version Id. */
@@ -31,7 +29,7 @@ public class MathArithmeticException extends MathRuntimeException {
* Default constructor.
*/
public MathArithmeticException() {
- this("arithmetic exception");
+ this("Arithmetic exception.");
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalNumberException.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalNumberException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalNumberException.java
new file mode 100644
index 0000000..2e7280b
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalNumberException.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.math.exceptions;
+
+/**
+ * This class is based on the corresponding class from Apache Common Math lib.
+ * Base class for exceptions raised by a wrong number.
+ * This class is not intended to be instantiated directly: it should serve
+ * as a base class to create all the exceptions that are raised because some
+ * precondition is violated by a number argument.
+ */
+public class MathIllegalNumberException extends MathIllegalArgumentException {
+ /** Serializable version Id. */
+ private static final long serialVersionUID = -7447085893598031110L;
+
+ /** Requested. */
+ private final Number argument;
+
+ /**
+ * Construct an exception.
+ *
+ * @param msg Localizable pattern.
+ * @param wrong Wrong number.
+ * @param arguments Arguments.
+ */
+ protected MathIllegalNumberException(String msg, Number wrong, Object... arguments) {
+ super(msg, wrong, arguments);
+ argument = wrong;
+ }
+
+ /**
+ * @return the requested value.
+ */
+ public Number getArgument() {
+ return argument;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalStateException.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalStateException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalStateException.java
new file mode 100644
index 0000000..13ef5ca
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalStateException.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.math.exceptions;
+
+/**
+ * This class is based on the corresponding class from Apache Common Math lib.
+ * Base class for all exceptions that signal that the process
+ * throwing the exception is in a state that does not comply with
+ * the set of states that it is designed to be in.
+ */
+public class MathIllegalStateException extends MathRuntimeException {
+ /** Serializable version Id. */
+ private static final long serialVersionUID = -6024911025449780478L;
+
+ /** */
+ private static final String ILLEGAL_STATE= "Illegal state.";
+
+ /**
+ * Simple constructor.
+ *
+ * @param msg Message pattern explaining the cause of the error.
+ * @param args Arguments.
+ */
+ public MathIllegalStateException(String msg, Object ... args) {
+ super(msg, args);
+ }
+
+ /**
+ * Default constructor.
+ */
+ public MathIllegalStateException() {
+ this(ILLEGAL_STATE);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/NumberIsTooSmallException.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/NumberIsTooSmallException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/NumberIsTooSmallException.java
new file mode 100644
index 0000000..7427592
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/NumberIsTooSmallException.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.math.exceptions;
+
+/**
+ * Exception to be thrown when a number is too small.
+ */
+public class NumberIsTooSmallException extends MathIllegalNumberException {
+ /** */
+ private static final String NUMBER_TOO_SMALL = "Number {1} is smaller than the minimum ({2}).";
+
+ /** */
+ private static final String NUMBER_TOO_SMALL_BOUND_EXCLUDED = "Number {1} is smaller than, or equal to, the minimum ({2}).";
+
+ /** Serializable version Id. */
+ private static final long serialVersionUID = -6100997100383932834L;
+ /**
+ * Higher bound.
+ */
+ private final Number min;
+ /**
+ * Whether the maximum is included in the allowed range.
+ */
+ private final boolean boundIsAllowed;
+
+ /**
+ * Construct the exception.
+ *
+ * @param wrong Value that is smaller than the minimum.
+ * @param min Minimum.
+ * @param boundIsAllowed Whether {@code min} is included in the allowed range.
+ */
+ public NumberIsTooSmallException(Number wrong, Number min, boolean boundIsAllowed) {
+ this(boundIsAllowed ? NUMBER_TOO_SMALL : NUMBER_TOO_SMALL_BOUND_EXCLUDED,
+ wrong, min, boundIsAllowed);
+ }
+
+ /**
+ * Construct the exception with a specific context.
+ *
+ * @param msg Specific context pattern.
+ * @param wrong Value that is smaller than the minimum.
+ * @param min Minimum.
+ * @param boundIsAllowed Whether {@code min} is included in the allowed range.
+ */
+ public NumberIsTooSmallException(String msg, Number wrong, Number min, boolean boundIsAllowed) {
+ super(msg, wrong, min);
+ this.min = min;
+ this.boundIsAllowed = boundIsAllowed;
+ }
+
+ /**
+ * @return {@code true} if the minimum is included in the allowed range.
+ */
+ public boolean getBoundIsAllowed() {
+ return boundIsAllowed;
+ }
+
+ /**
+ * @return the minimum.
+ */
+ public Number getMin() {
+ return min;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java
index e86a5eb..22a453d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java
@@ -17,6 +17,10 @@
package org.apache.ignite.ml.math.functions;
+import org.apache.ignite.lang.IgniteBiTuple;
+
+import java.util.List;
+
/**
* Compatibility with Apache Mahout.
*/
@@ -63,6 +67,9 @@ public final class Functions {
/** Function that returns {@code a - b}. */
public static final IgniteBiFunction<Double, Double, Double> MINUS = (a, b) -> a - b;
+ /** Function that returns {@code min(a, b)}. */
+ public static final IgniteBiFunction<Double, Double, Double> MIN = Math::min;
+
/** Function that returns {@code abs(a - b)}. */
public static final IgniteBiFunction<Double, Double, Double> MINUS_ABS = (a, b) -> Math.abs(a - b);
@@ -81,6 +88,38 @@ public final class Functions {
/** Function that returns {@code a < b ? -1 : a > b ? 1 : 0}. */
public static final IgniteBiFunction<Double, Double, Double> COMPARE = (a, b) -> a < b ? -1.0 : a > b ? 1.0 : 0.0;
+ /** */
+ public static <A, B, C> IgniteFunction<B, C> curry(IgniteBiFunction<A, B, C> f, A a) {
+ return (IgniteFunction<B, C>)b -> f.apply(a, b);
+ }
+
+ /** */
+ public static <A, B extends Comparable<B>> IgniteBiTuple<Integer, A> argmin(List<A> args, IgniteFunction<A, B> f) {
+ A res = null;
+ B fRes = null;
+
+ if (!args.isEmpty()) {
+ res = args.iterator().next();
+ fRes = f.apply(res);
+ }
+
+ int resInd = 0;
+ int i = 0;
+
+ for (A arg : args) {
+ B curRes = f.apply(arg);
+
+ if (fRes.compareTo(curRes) > 0) {
+ res = arg;
+ resInd = i;
+ fRes = curRes;
+ }
+
+ i++;
+ }
+ return new IgniteBiTuple<>(resInd, res);
+ }
+
/**
* Function that returns {@code a + b}. {@code a} is a variable, {@code b} is fixed.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
index ace399b..836789b 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java
@@ -20,6 +20,7 @@ package org.apache.ignite.ml.math.impls;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
+import java.util.function.BinaryOperator;
import javax.cache.Cache;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
@@ -28,6 +29,7 @@ import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.cluster.ClusterGroup;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.lang.IgnitePredicate;
@@ -38,6 +40,7 @@ import org.apache.ignite.ml.math.ValueMapper;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.functions.IgniteConsumer;
import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
/**
@@ -380,6 +383,50 @@ public class CacheUtils {
});
}
+ public static <K, V, A> A distributedFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder,
+ IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, A zeroVal) {
+ return sparseFold(cacheName, folder, keyFilter, accumulator, zeroVal, null, null, 0 ,
+ false);
+ }
+
+ private static <K, V, A> A sparseFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder,
+ IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, A zeroVal, V defVal, K defKey, long defValCnt, boolean isNilpotent) {
+
+ A defRes = zeroVal;
+
+ if (!isNilpotent)
+ for (int i = 0; i < defValCnt; i++)
+ defRes = folder.apply(new CacheEntryImpl<>(defKey, defVal), defRes);
+
+ Collection<A> totalRes = bcast(cacheName, () -> {
+ Ignite ignite = Ignition.localIgnite();
+ IgniteCache<K, V> cache = ignite.getOrCreateCache(cacheName);
+
+ int partsCnt = ignite.affinity(cacheName).partitions();
+
+ // Use affinity in filter for ScanQuery. Otherwise we accept consumer in each node which is wrong.
+ Affinity affinity = ignite.affinity(cacheName);
+ ClusterNode localNode = ignite.cluster().localNode();
+
+ A a = zeroVal;
+
+ // Iterate over all partitions. Some of them will be stored on that local node.
+ for (int part = 0; part < partsCnt; part++) {
+ int p = part;
+
+ // Iterate over given partition.
+ // Query returns an empty cursor if this partition is not stored on this node.
+ for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part,
+ (k, v) -> affinity.mapPartitionToNode(p) == localNode && (keyFilter == null || keyFilter.apply(k)))))
+ a = folder.apply(entry, a);
+ }
+
+ return a;
+ });
+ totalRes.add(defRes);
+ return totalRes.stream().reduce(zeroVal, accumulator);
+ }
+
/**
* @param cacheName Cache name.
* @param run {@link Runnable} to broadcast to cache nodes for given cache name.
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
index d1d3904..106a425 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java
@@ -24,6 +24,8 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
+import java.util.function.BiFunction;
+import java.util.function.BinaryOperator;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.MatrixStorage;
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
index 3e508bd..cebacc5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
@@ -156,7 +156,7 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo
}
/** */
- private IgniteUuid getUUID(){
+ public IgniteUuid getUUID(){
return ((SparseDistributedMatrixStorage) getStorage()).getUUID();
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java
new file mode 100644
index 0000000..13b8303
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java
@@ -0,0 +1,93 @@
+package org.apache.ignite.ml.math.impls.storage.matrix;
+
+import org.apache.ignite.internal.util.GridArgumentCheck;
+import org.apache.ignite.ml.math.VectorStorage;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Storage for wrapping given map.
+ */
+public class MapWrapperStorage implements VectorStorage {
+ /** Underlying map. */
+ Map<Integer, Double> data;
+
+ /** Vector size. */
+ int size;
+
+ /**
+ * Construct a wrapper around given map.
+ *
+ * @param map Map to wrap.
+ */
+ public MapWrapperStorage(Map<Integer, Double> map) {
+ Set<Integer> keys = map.keySet();
+
+ GridArgumentCheck.notEmpty(keys, "map");
+
+ Integer min = keys.stream().mapToInt(Integer::valueOf).min().getAsInt();
+ Integer max = keys.stream().mapToInt(Integer::valueOf).max().getAsInt();
+
+ assert min >= 0;
+
+ data = map;
+ size = (max - min) + 1;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int size() {
+ return size;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double get(int i) {
+ return data.getOrDefault(i, 0.0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void set(int i, double v) {
+ if (v != 0.0)
+ data.put(i, v);
+ else if (data.containsKey(i))
+ data.remove(i);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeObject(data);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ data = (Map<Integer, Double>) in.readObject();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isSequentialAccess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isRandomAccess() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isDense() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isArrayBased() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isDistributed() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java
index 8400758..f07a16e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java
@@ -22,6 +22,7 @@ import it.unimi.dsi.fastutil.ints.Int2DoubleRBTreeMap;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.util.HashMap;
import java.util.Map;
import org.apache.ignite.ml.math.StorageConstants;
import org.apache.ignite.ml.math.VectorStorage;
@@ -46,6 +47,37 @@ public class SparseLocalOnHeapVectorStorage implements VectorStorage, StorageCon
}
/**
+ *
+ * @param map
+ */
+ public SparseLocalOnHeapVectorStorage(Map<Integer, Double> map, boolean copy) {
+ assert map.size() > 0;
+
+ this.size = map.size();
+
+ if (map instanceof Int2DoubleRBTreeMap)
+ acsMode = SEQUENTIAL_ACCESS_MODE;
+ else
+ if (map instanceof Int2DoubleOpenHashMap)
+ acsMode = RANDOM_ACCESS_MODE;
+ else
+ acsMode = UNKNOWN_STORAGE_MODE;
+
+ if (copy)
+ switch (acsMode) {
+ case SEQUENTIAL_ACCESS_MODE:
+ sto = new Int2DoubleRBTreeMap(map);
+ case RANDOM_ACCESS_MODE:
+ sto = new Int2DoubleOpenHashMap(map);
+ break;
+ default:
+ sto = new HashMap<>(map);
+ }
+ else
+ sto = map;
+ }
+
+ /**
* @param size Vector size.
* @param acsMode Access mode.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java
index c868160..48fbd06 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java
@@ -59,6 +59,11 @@ public class DelegatingVector implements Vector {
this.dlg = dlg;
}
+ /** Get the delegating vector */
+ public Vector getVector() {
+ return dlg;
+ }
+
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
out.writeObject(dlg);
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java
new file mode 100644
index 0000000..729a7ea
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java
@@ -0,0 +1,32 @@
+package org.apache.ignite.ml.math.impls.vector;
+
+import org.apache.ignite.ml.math.Matrix;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.impls.storage.matrix.MapWrapperStorage;
+
+import java.util.Map;
+
+/**
+ * Vector wrapping a given map.
+ */
+public class MapWrapperVector extends AbstractVector {
+
+ /**
+ * Construct a vector wrapping given map.
+ *
+ * @param map Map to wrap.
+ */
+ public MapWrapperVector(Map<Integer, Double> map) {
+ setStorage(new MapWrapperStorage(map));
+ }
+
+ /** {@inheritDoc} */
+ @Override public Vector like(int crd) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Matrix likeMatrix(int rows, int cols) {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java
index e188f70..be5d0f6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ml.math.impls.vector;
+import java.util.Map;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.StorageConstants;
import org.apache.ignite.ml.math.Vector;
@@ -35,6 +36,14 @@ public class SparseLocalVector extends AbstractVector implements StorageConstant
}
/**
+ * @param map Underlying map.
+ * @param copy Should given map be copied.
+ */
+ public SparseLocalVector(Map<Integer, Double> map, boolean copy) {
+ setStorage(new SparseLocalOnHeapVectorStorage(map, copy));
+ }
+
+ /**
* @param size Vector size.
* @param acsMode Vector elements access mode.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java
index f3bd4dd..0988059 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java
@@ -60,6 +60,7 @@ public class VectorView extends AbstractVector {
/** {@inheritDoc} */
@Override public Vector copy() {
+ // TODO: revise this
DelegateVectorStorage sto = storage();
return new VectorView(sto.delegate(), sto.offset(), sto.length());
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java
new file mode 100644
index 0000000..e406b5b
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.statistics;
+
+/**
+ * This class encapsulates calculating variance.
+ */
+public class Variance {
+ /** */
+ private double mean;
+
+ /** */
+ private long n;
+
+ /** */
+ private double m2;
+
+ public Variance() {
+ mean = 0;
+ n = 0;
+ m2 = 0;
+ }
+
+ /** */
+ public Variance update(Double x) {
+ n++;
+ double delta = x - mean;
+ mean += delta / n;
+ double delta2 = x - mean;
+ m2 += delta * delta2;
+ return this;
+ }
+
+ /** */
+ public double getResult() {
+ return m2;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java
new file mode 100644
index 0000000..6c25f0e
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.math.util;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.function.BinaryOperator;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ *
+ */
+public class MapUtil {
+ /** */
+ public static <K, V, M extends Map<K, V>> M mergeMaps(M m1, M m2, BinaryOperator<V> op, Supplier<M> mapSupplier) {
+ return Stream.of(m1, m2)
+ .map(Map::entrySet)
+ .flatMap(Collection::stream)
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, op, mapSupplier));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java
index a06b773..5ef7176 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java
@@ -17,6 +17,9 @@
package org.apache.ignite.ml.math.util;
+import java.util.List;
+
+import org.apache.ignite.internal.util.GridArgumentCheck;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.Vector;
import org.apache.ignite.ml.math.impls.matrix.CacheMatrix;
@@ -24,7 +27,6 @@ import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
import org.apache.ignite.ml.math.impls.matrix.MatrixView;
import org.apache.ignite.ml.math.impls.matrix.PivotedMatrixView;
import org.apache.ignite.ml.math.impls.matrix.RandomMatrix;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
/**
@@ -117,6 +119,38 @@ public class MatrixUtil {
/** */
private static boolean isCopyLikeSupport(Matrix matrix) {
return matrix instanceof RandomMatrix || matrix instanceof MatrixView || matrix instanceof CacheMatrix ||
- matrix instanceof PivotedMatrixView || matrix instanceof SparseDistributedMatrix;
+ matrix instanceof PivotedMatrixView;
+ }
+
+ /** */
+ public static DenseLocalOnHeapMatrix fromList(List<Vector> vecs, boolean entriesAreRows) {
+ GridArgumentCheck.notEmpty(vecs, "vecs");
+
+ int dim = vecs.get(0).size();
+ int vecsSize = vecs.size();
+
+ DenseLocalOnHeapMatrix res = new DenseLocalOnHeapMatrix(entriesAreRows ? vecsSize : dim,
+ entriesAreRows ? dim : vecsSize);
+
+ for (int i = 0; i < vecsSize; i++) {
+ for (int j = 0; j < dim; j++) {
+ int r = entriesAreRows ? i : j;
+ int c = entriesAreRows ? j : i;
+
+ res.setX(r, c, vecs.get(i).get(j));
+ }
+ }
+
+ return res;
+ }
+
+ /** TODO: rewrite in a more optimal way. */
+ public static DenseLocalOnHeapVector localCopyOf(Vector vec) {
+ DenseLocalOnHeapVector res = new DenseLocalOnHeapVector(vec.size());
+
+ for (int i = 0; i < vec.size(); i++)
+ res.setX(i, vec.getX(i));
+
+ return res;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
index 92aa7db..dea3edf 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ml;
+import org.apache.ignite.ml.clustering.ClusteringTesetSuite;
import org.apache.ignite.ml.math.MathImplMainTestSuite;
import org.apache.ignite.ml.regressions.RegressionsTestSuite;
import org.junit.runner.RunWith;
@@ -28,7 +29,8 @@ import org.junit.runners.Suite;
@RunWith(Suite.class)
@Suite.SuiteClasses({
MathImplMainTestSuite.class,
- RegressionsTestSuite.class
+ RegressionsTestSuite.class,
+ ClusteringTesetSuite.class
})
public class IgniteMLTestSuite {
// No-op.
http://git-wip-us.apache.org/repos/asf/ignite/blob/b04b5800/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTesetSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTesetSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTesetSuite.java
new file mode 100644
index 0000000..122d0c1
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTesetSuite.java
@@ -0,0 +1,15 @@
+package org.apache.ignite.ml.clustering;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/**
+ * Test suite for all tests located in org.apache.ignite.ml.clustering package.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ KMeansDistributedClustererTest.class,
+ KMeansLocalClustererTest.class
+})
+public class ClusteringTesetSuite {
+}
[12/12] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-5075
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-5075
# Conflicts:
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/eeba611c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/eeba611c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/eeba611c
Branch: refs/heads/ignite-5075
Commit: eeba611c80e7b63670baf7d4ba05b90d0b86169f
Parents: 5ef6900
Author: sboikov <sb...@gridgain.com>
Authored: Thu Jun 1 11:52:49 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Jun 1 11:52:49 2017 +0300
----------------------------------------------------------------------
.../ignite/internal/processors/cache/GridCacheProcessor.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeba611c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index afb578b..5990c42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -2066,8 +2066,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
Throwable err
) {
ExchangeActions actions = new ExchangeActions(){
- @Override List<DynamicCacheChangeRequest> closeRequests(UUID nodeId) {
- return Collections.singletonList(act.request());
+ @Override List<ActionData> closeRequests(UUID nodeId) {
+ return Collections.singletonList(act);
}
};
@@ -2135,7 +2135,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
}
if (forceClose)
- completeCacheStartFuture(req, false, err);
+ completeCacheStartFuture(req.request(), false, err);
}
}
}
[08/12] ignite git commit: IGNITE-5365 Build release artifacts once
and use them for all editions
Posted by sb...@apache.org.
IGNITE-5365 Build release artifacts once and use them for all editions
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/54f499ce
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/54f499ce
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/54f499ce
Branch: refs/heads/ignite-5075
Commit: 54f499cece98c9cade28f3e0a9cfe89323c64ef0
Parents: b04b580
Author: oleg-ostanin <oo...@gridgain.com>
Authored: Wed May 31 20:02:00 2017 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Wed May 31 20:02:00 2017 +0300
----------------------------------------------------------------------
DEVNOTES.txt | 74 ++++--
assembly/dependencies-fabric-lgpl.xml | 2 +
assembly/dependencies-fabric.xml | 2 +
assembly/release-fabric-lgpl.xml | 1 +
assembly/release-fabric.xml | 1 +
assembly/release-yardstick.xml | 98 +++++++
.../Apache.Ignite.Core.nuspec | 5 +-
modules/platforms/dotnet/build.ps1 | 5 +-
modules/web-console/web-agent/pom.xml | 8 +
parent/pom.xml | 8 +-
pom.xml | 254 ++++++-------------
11 files changed, 246 insertions(+), 212 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/DEVNOTES.txt
----------------------------------------------------------------------
diff --git a/DEVNOTES.txt b/DEVNOTES.txt
index d3c162e..8a689b8 100644
--- a/DEVNOTES.txt
+++ b/DEVNOTES.txt
@@ -1,36 +1,63 @@
Ignite Fabric Maven Build Instructions
======================================
-Without LGPL dependencies (default):
- mvn clean package -DskipTests
+1) Optional: build Apache Ignite.NET as described at modules/platforms/dotnet/DEVNOTES.txt.
-Without LGPL dependencies and Scala 2.10:
- mvn clean package -DskipTests -Dscala-2.10
+2) Compile and package:
-With LGPL dependencies:
- mvn clean package -DskipTests -Prelease,lgpl -Dignite.edition=fabric-lgpl
+ mvn clean package -Pall-java,all-scala,licenses -DskipTests
-With LGPL dependencies and Scala 2.10:
- mvn clean package -DskipTests -Prelease,lgpl -Dignite.edition=fabric-lgpl -Dscala-2.10
+ or if you have built Apache Ignite.NET on the first step use following command:
+ (Note that 'doxygen' should be installed before running this command.)
-With LGPL dependencies and Scala 2.10 and Ignite-Yardstick:
- mvn clean package -DskipTests -Prelease,lgpl,yardstick -Dignite.edition=fabric-lgpl -Dscala-2.10
+ mvn clean package -Pall-java,all-scala,licenses -DskipTests -DclientDocs
-With Apache Ignite.NET:
- Build Apache Ignite.NET as described at modules/platforms/dotnet/DEVNOTES.txt.
+3) Javadoc generation (optional):
- Then build Ignite Fabric with Apache Ignite.NET:
- mvn clean package -DskipTests -DclientDocs
+ mvn initialize -Pjavadoc
- Note that 'doxygen' should be installed before running this command.
+4) Assembly Apache Ignite fabric:
+
+ mvn initialize -Prelease
Look for apache-ignite-fabric-<version>-bin.zip in ./target/bin directory.
+Ignite Fabric with LGPL Maven Build Instructions
+======================================
+1) Optional: build Apache Ignite.NET as described at modules/platforms/dotnet/DEVNOTES.txt.
+
+2) Compile and package:
+
+ mvn clean package -Pall-java,all-scala,licenses -DskipTests
+
+ or if you have built Apache Ignite.NET on the first step use following command:
+ (Note that 'doxygen' should be installed before running this command.)
+
+ mvn clean package -Pall-java,all-scala,licenses -DskipTests -DclientDocs
+
+3) Javadoc generation with LGPL (optional):
+
+ mvn initialize -Pjavadoc,lgpl
+
+4) Assembly Apache Ignite fabric with LGPL dependencies:
+
+ mvn initialize -Prelease,lgpl -Dignite.edition=fabric-lgpl
+
+Look for apache-ignite-fabric-lgpl-<version>-bin.zip in ./target/bin directory.
+
Ignite Hadoop Accelerator Maven Build Instructions
============================================
-mvn clean package -DskipTests -Dignite.edition=hadoop [-Dhadoop.version=X.X.X] [-Dspark.version=x.y.z]
+1) Compile and package:
+
+ mvn clean package -Pall-java,all-scala,licenses -DskipTests
Use 'hadoop.version' parameter to build Ignite against a specific Hadoop version.
Use 'spark.version' parameter to build ignite-spark module for a specific Spark version. Version should be >= 2.0.0.
+For example:
+
+ mvn clean package -Pall-java,all-scala,licenses -DskipTests -Dhadoop.version=2.4.2 -Dspark.version=2.1.1
+
+2) Assembly Hadoop Accelerator:
+ mvn initialize -Prelease -Dignite.edition=hadoop
Look for apache-ignite-hadoop-<version>-bin.zip in ./target/bin directory. Resulting binary
assembly will also include integration module for Apache Spark.
@@ -66,12 +93,17 @@ Maven release plugin release:prepare goal can be used to make release tag.
Deploy Ignite release candidate to maven repository and dev-svn, make tag:
- Following command deploys Ignite to maven repository, prepares sources and fabric edition binaries.
- mvn deploy -P apache-release,gpg,release,scala,lgpl,deploy-ignite-site -Dignite.edition=fabric -DskipTests -B
+ 1) Deploy Ignite to maven repository, prepares sources and fabric edition binaries.
+ mvn deploy -Papache-release,gpg,all-java,all-scala,licenses,deploy-ignite-site -Dignite.edition=fabric -DskipTests
+
+ 2) Javadoc generation:
+ mvn initialize -Pjavadoc
+
+ 3) Assembly Apache Ignite Fabric:
+ mvn initialize -Prelease
- In case you want to release both fabric and hadoop editions you have to build hadoop edition first using command
- mvn package -P apache-release,gpg,release,scala,lgpl -Dignite.edition=hadoop -DskipTests -B
- save /target/bin/*.zip, make "mvn clean" and restore saved files before deploying fabric.
+ 4) Assembly Hadoop Accelerator:
+ mvn initialize -Prelease -Dignite.edition=hadoop
Binary artifact name can be changed by setting additional property -Dignite.zip.pattern. Binary artifact will be
created inside /target/bin folder when release profile is used.
http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/assembly/dependencies-fabric-lgpl.xml
----------------------------------------------------------------------
diff --git a/assembly/dependencies-fabric-lgpl.xml b/assembly/dependencies-fabric-lgpl.xml
index 4cff146..e2fac3c 100644
--- a/assembly/dependencies-fabric-lgpl.xml
+++ b/assembly/dependencies-fabric-lgpl.xml
@@ -131,6 +131,8 @@
<exclude>org.apache.ignite:ignite-websphere-test</exclude>
<exclude>org.apache.ignite:ignite-cassandra</exclude>
<exclude>org.apache.ignite:ignite-yardstick</exclude>
+ <exclude>org.apache.ignite:ignite-benchmarks</exclude>
+ <exclude>org.apache.ignite:ignite-web-agent</exclude>
</excludes>
<sources>
<includeModuleDirectory>true</includeModuleDirectory>
http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/assembly/dependencies-fabric.xml
----------------------------------------------------------------------
diff --git a/assembly/dependencies-fabric.xml b/assembly/dependencies-fabric.xml
index be68d0a..6c4101e 100644
--- a/assembly/dependencies-fabric.xml
+++ b/assembly/dependencies-fabric.xml
@@ -136,6 +136,8 @@
<exclude>org.apache.ignite:ignite-websphere-test</exclude>
<exclude>org.apache.ignite:ignite-cassandra</exclude>
<exclude>org.apache.ignite:ignite-yardstick</exclude>
+ <exclude>org.apache.ignite:ignite-benchmarks</exclude>
+ <exclude>org.apache.ignite:ignite-web-agent</exclude>
</excludes>
<sources>
<includeModuleDirectory>true</includeModuleDirectory>
http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/assembly/release-fabric-lgpl.xml
----------------------------------------------------------------------
diff --git a/assembly/release-fabric-lgpl.xml b/assembly/release-fabric-lgpl.xml
index 1766558..ff4d8c4 100644
--- a/assembly/release-fabric-lgpl.xml
+++ b/assembly/release-fabric-lgpl.xml
@@ -32,6 +32,7 @@
<componentDescriptors>
<componentDescriptor>release-base.xml</componentDescriptor>
<componentDescriptor>release-fabric-base.xml</componentDescriptor>
+ <componentDescriptor>release-yardstick.xml</componentDescriptor>
</componentDescriptors>
<files>
http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/assembly/release-fabric.xml
----------------------------------------------------------------------
diff --git a/assembly/release-fabric.xml b/assembly/release-fabric.xml
index ffde8ec..7536d4e 100644
--- a/assembly/release-fabric.xml
+++ b/assembly/release-fabric.xml
@@ -32,6 +32,7 @@
<componentDescriptors>
<componentDescriptor>release-base.xml</componentDescriptor>
<componentDescriptor>release-fabric-base.xml</componentDescriptor>
+ <componentDescriptor>release-yardstick.xml</componentDescriptor>
</componentDescriptors>
<files>
http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/assembly/release-yardstick.xml
----------------------------------------------------------------------
diff --git a/assembly/release-yardstick.xml b/assembly/release-yardstick.xml
new file mode 100644
index 0000000..0bd6f41
--- /dev/null
+++ b/assembly/release-yardstick.xml
@@ -0,0 +1,98 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<component xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/component/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/component/1.1.2
+ http://maven.apache.org/xsd/component-1.1.2.xsd">
+ <files>
+ <file>
+ <source>modules/yardstick/pom-standalone.xml</source>
+ <outputDirectory>benchmarks/sources</outputDirectory>
+ <destName>pom.xml</destName>
+ </file>
+
+ <file>
+ <source>modules/yardstick/DEVNOTES-standalone.txt</source>
+ <outputDirectory>benchmarks/sources</outputDirectory>
+ <destName>DEVNOTES.txt</destName>
+ </file>
+
+ <file>
+ <source>modules/yardstick/README.txt</source>
+ <outputDirectory>benchmarks</outputDirectory>
+ <destName>README.txt</destName>
+ </file>
+ </files>
+
+ <fileSets>
+ <fileSet>
+ <directory>modules/yardstick/target/assembly/bin</directory>
+ <outputDirectory>benchmarks/bin</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>modules/yardstick/target/assembly/config</directory>
+ <outputDirectory>benchmarks/config</outputDirectory>
+ <includes>
+ <include>benchmark.properties</include>
+ <include>benchmark-remote.properties</include>
+ <include>benchmark-sample.properties</include>
+ <include>benchmark-remote-sample.properties</include>
+ <include>benchmark-multicast.properties</include>
+ <include>ignite-base-config.xml</include>
+ <include>ignite-localhost-config.xml</include>
+ <include>ignite-remote-config.xml</include>
+ <include>ignite-multicast-config.xml</include>
+ </includes>
+ </fileSet>
+
+ <fileSet>
+ <directory>modules/yardstick/target/assembly/libs</directory>
+ <outputDirectory>benchmarks/libs</outputDirectory>
+ <excludes>
+ <exclude>junit-*.jar</exclude>
+ <exclude>ignite-apache-license-gen-*.jar</exclude>
+ <exclude>hamcrest-core-*.jar</exclude>
+ <exclude>tools-*.jar</exclude>
+ </excludes>
+ </fileSet>
+
+ <fileSet>
+ <directory>modules/yardstick/src</directory>
+ <outputDirectory>benchmarks/sources/src</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>modules/yardstick/target/assembly/config</directory>
+ <outputDirectory>benchmarks/sources/config</outputDirectory>
+ <includes>
+ <include>benchmark.properties</include>
+ <include>benchmark-remote.properties</include>
+ <include>benchmark-sample.properties</include>
+ <include>benchmark-remote-sample.properties</include>
+ <include>benchmark-multicast.properties</include>
+ <include>ignite-base-config.xml</include>
+ <include>ignite-localhost-config.xml</include>
+ <include>ignite-remote-config.xml</include>
+ <include>ignite-multicast-config.xml</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+</component>
http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
index 7dc9f9a..9049526 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
@@ -62,10 +62,7 @@ More info: https://apacheignite-net.readme.io/
Library files (jars) should not be included in project, so that NuGet package restore works properly.
We keep jars in NuGet dir and copy them over in PostBuild event.
-->
- <file src="..\..\..\..\target\release-package\libs\*.jar" target="Libs" />
- <file src="..\..\..\..\target\release-package\libs\ignite-spring\*.jar" target="Libs" />
- <file src="..\..\..\..\target\release-package\libs\ignite-indexing\*.jar" target="Libs" />
- <file src="..\..\..\..\target\release-package\libs\licenses\*.*" target="Licenses" />
+ <file src="..\bin\Libs\*.jar" target="Libs" />
<!-- LINQPad samples -->
<file src="NuGet\LINQPad\*.*" target="linqpad-samples" />
http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/modules/platforms/dotnet/build.ps1
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/build.ps1 b/modules/platforms/dotnet/build.ps1
index b8e6a37..222076a 100644
--- a/modules/platforms/dotnet/build.ps1
+++ b/modules/platforms/dotnet/build.ps1
@@ -112,10 +112,7 @@ if (!$skipJava) {
$libsDir = "$PSScriptRoot\bin\Libs"
mkdir -Force $libsDir; del -Force $libsDir\*.*
- copy -Force target\release-package\libs\*.jar $libsDir
- copy -Force target\release-package\libs\ignite-spring\*.jar $libsDir
- copy -Force target\release-package\libs\ignite-indexing\*.jar $libsDir
- copy -Force target\release-package\libs\licenses\*.jar $libsDir
+ ls modules\indexing\target,modules\core\target,modules\spring\target*.jar -recurse -include "ignite-core*","ignite-indexing*","ignite-shmem*","ignite-spring*","lucene*","h2*","cache-api*","commons-*","spring*" -exclude "*-sources*","*-javadoc*","*-tests*" | % { copy -Force $_ $libsDir }
# Restore directory
cd $PSScriptRoot
http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/modules/web-console/web-agent/pom.xml
----------------------------------------------------------------------
diff --git a/modules/web-console/web-agent/pom.xml b/modules/web-console/web-agent/pom.xml
index 2f3969e..e715fbf 100644
--- a/modules/web-console/web-agent/pom.xml
+++ b/modules/web-console/web-agent/pom.xml
@@ -183,6 +183,14 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index 88d4da8..270ece4 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -998,12 +998,8 @@
</profile>
<profile>
- <id>release</id>
- <activation>
- <property>
- <name>!skipDefault</name><!--activeByDefault-->
- </property>
- </activation>
+ <id>licenses</id>
+
<dependencies>
<dependency>
<groupId>org.apache.ignite</groupId>
http://git-wip-us.apache.org/repos/asf/ignite/blob/54f499ce/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7f24613..7fa3eef 100644
--- a/pom.xml
+++ b/pom.xml
@@ -164,29 +164,64 @@
</profile>
<profile>
- <id>dev-libs</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
+ <id>javadoc</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>core-javadoc</id>
+ <goals>
+ <goal>aggregate</goal>
+ </goals>
+ <phase>validate</phase>
+ <configuration>
+ <reportOutputDirectory>${basedir}/target/javadoc</reportOutputDirectory>
+ <destDir>core</destDir>
+ <stylesheetfile>${basedir}/assembly/docfiles/javadoc.css</stylesheetfile>
+ <subpackages>org.apache.ignite -exclude org.apache.ignite.client:org.apache.ignite.codegen:org.apache.ignite.examples:org.apache.ignite.internal:org.apache.ignite.schema:org.apache.ignite.tests:org.apache.ignite.tools:org.apache.ignite.util:org.apache.ignite.spi.discovery.tcp.messages:org.apache.ignite.spi.discovery.tcp.internal:org.apache.ignite.spi.deployment.uri.scanners:org.apache.ignite.spi.deployment.uri.tasks:org.apache.ignite.yardstick:org.apache.ignite.webtest</subpackages>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.7</version>
<inherited>false</inherited>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-tools</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
<executions>
<execution>
- <id>dev-libs</id>
+ <id>javadoc-postprocessing-new</id>
<goals>
<goal>run</goal>
</goals>
- <phase>package</phase>
+ <phase>initialize</phase>
<configuration>
<target>
- <copy todir="${basedir}/libs">
- <fileset dir="${basedir}/target/release-package/libs" />
+ <copy todir="${basedir}/target/javadoc/core">
+ <fileset dir="assembly/docfiles">
+ <include name="img/**" />
+ <include name="*.js" />
+ </fileset>
</copy>
+
+ <taskdef name="doctask" classname="org.apache.ignite.tools.ant.beautifier.GridJavadocAntTask" />
+
+ <doctask css="dotted" dir="target/javadoc/core">
+ <include name="**/*.html" />
+ <exclude name="overview-frame.html" />
+ <exclude name="allclasses-frame.html" />
+ <exclude name="**/class-use/*" />
+ </doctask>
</target>
</configuration>
</execution>
@@ -208,11 +243,6 @@
<profile>
<id>release</id>
- <activation>
- <property>
- <name>!skipDefault</name><!--activeByDefault-->
- </property>
- </activation>
<build>
<plugins>
<plugin><!-- skipping generation of dependencies licenses
@@ -235,26 +265,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <execution>
- <id>core-javadoc</id>
- <goals>
- <goal>aggregate</goal>
- </goals>
- <phase>process-test-classes</phase>
- <configuration>
- <reportOutputDirectory>${basedir}/target/javadoc</reportOutputDirectory>
- <destDir>core</destDir>
- <stylesheetfile>${basedir}/assembly/docfiles/javadoc.css</stylesheetfile>
- <subpackages>org.apache.ignite -exclude org.apache.ignite.client:org.apache.ignite.codegen:org.apache.ignite.examples:org.apache.ignite.internal:org.apache.ignite.schema:org.apache.ignite.tests:org.apache.ignite.tools:org.apache.ignite.util:org.apache.ignite.spi.discovery.tcp.messages:org.apache.ignite.spi.discovery.tcp.internal:org.apache.ignite.spi.deployment.uri.scanners:org.apache.ignite.spi.deployment.uri.tasks:org.apache.ignite.yardstick:org.apache.ignite.webtest</subpackages>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.7</version>
<inherited>false</inherited>
@@ -267,44 +277,17 @@
</dependencies>
<executions>
<execution>
- <id>javadoc-postprocessing</id>
- <goals>
- <goal>run</goal>
- </goals>
- <phase>prepare-package</phase>
- <configuration>
- <target>
- <copy todir="${basedir}/target/javadoc/core">
- <fileset dir="assembly/docfiles">
- <include name="img/**" />
- <include name="*.js" />
- </fileset>
- </copy>
-
- <taskdef name="doctask" classname="org.apache.ignite.tools.ant.beautifier.GridJavadocAntTask" />
-
- <doctask css="dotted" dir="target/javadoc/core">
- <include name="**/*.html" />
- <exclude name="overview-frame.html" />
- <exclude name="allclasses-frame.html" />
- <exclude name="**/class-use/*" />
- </doctask>
- </target>
- </configuration>
- </execution>
-
- <execution>
<id>release-postprocessing</id>
<goals>
<goal>run</goal>
</goals>
- <phase>package</phase>
+ <phase>initialize</phase>
<configuration>
<target>
<replaceregexp byline="true">
<regexp pattern="pushd "%~dp0"/\.\./\.\.(\s*&::.+)?" />
<substitution expression="pushd "%~dp0"/.." />
- <fileset dir="${basedir}/target/release-package/bin">
+ <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
<include name="**/*.bat" />
</fileset>
</replaceregexp>
@@ -312,7 +295,7 @@
<replaceregexp byline="true">
<regexp pattern="IGNITE_HOME_TMP="\$\(dirname "\$\{IGNITE_HOME_TMP\}"\)"(\s*#.*)?" />
<substitution expression="" />
- <fileset dir="${basedir}/target/release-package/bin">
+ <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
<include name="**/*.sh" />
</fileset>
</replaceregexp>
@@ -320,7 +303,7 @@
<replaceregexp byline="true">
<regexp pattern="set SCRIPTS_HOME=%IGNITE_HOME%\\bin(\s*&::.*)?" />
<substitution expression="set SCRIPTS_HOME=%IGNITE_HOME%\\\\bin" />
- <fileset dir="${basedir}/target/release-package/bin">
+ <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
<include name="**/*.bat" />
</fileset>
</replaceregexp>
@@ -328,7 +311,7 @@
<replaceregexp byline="true">
<regexp pattern="SCRIPTS_HOME="\$\{IGNITE_HOME_TMP\}/bin"(\s*#.*)?" />
<substitution expression="SCRIPTS_HOME="$${IGNITE_HOME_TMP}/bin"" />
- <fileset dir="${basedir}/target/release-package/bin">
+ <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
<include name="**/*.sh" />
</fileset>
</replaceregexp>
@@ -336,7 +319,7 @@
<replaceregexp byline="true">
<regexp pattern="\. "\$\{SCRIPTS_HOME\}"/include/build-classpath.sh(\s*#.*)?" />
<substitution expression="" />
- <fileset dir="${basedir}/target/release-package/bin">
+ <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
<include name="**/*.sh" />
</fileset>
</replaceregexp>
@@ -344,7 +327,7 @@
<replaceregexp byline="true">
<regexp pattern="call "%SCRIPTS_HOME%\\include\\build-classpath.bat"(\s*&::.*)?" />
<substitution expression="" />
- <fileset dir="${basedir}/target/release-package/bin">
+ <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
<include name="**/*.bat" />
</fileset>
</replaceregexp>
@@ -352,7 +335,7 @@
<replaceregexp byline="true">
<regexp pattern="ENABLE_ASSERTIONS=.*" />
<substitution expression="ENABLE_ASSERTIONS="0"" />
- <fileset dir="${basedir}/target/release-package/bin">
+ <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
<include name="**/*.sh" />
</fileset>
</replaceregexp>
@@ -360,7 +343,7 @@
<replaceregexp byline="true">
<regexp pattern="ENABLE_ASSERTIONS=.*" />
<substitution expression="ENABLE_ASSERTIONS=0" />
- <fileset dir="${basedir}/target/release-package/bin">
+ <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
<include name="**/*.bat" />
</fileset>
</replaceregexp>
@@ -368,36 +351,41 @@
<replaceregexp byline="true">
<regexp pattern="-DIGNITE_UPDATE_NOTIFIER=false" />
<substitution expression="" />
- <fileset dir="${basedir}/target/release-package/bin">
+ <fileset dir="${basedir}/target/release-package-${ignite.edition}/bin">
<include name="**/*.sh" />
<include name="**/*.bat" />
</fileset>
</replaceregexp>
- <replaceregexp file="${basedir}/target/release-package/examples/pom.xml" byline="true">
+ <replaceregexp file="${basedir}/target/release-package-${ignite.edition}/examples/pom.xml" byline="true">
+ <regexp pattern="to_be_replaced_by_ignite_version" />
+ <substitution expression="${project.version}" />
+ </replaceregexp>
+
+ <replaceregexp file="${basedir}/target/release-package-${ignite.edition}/benchmarks/sources/pom.xml" byline="true">
<regexp pattern="to_be_replaced_by_ignite_version" />
<substitution expression="${project.version}" />
</replaceregexp>
- <chmod dir="${basedir}/target/release-package" perm="755" includes="**/*.sh" />
+ <chmod dir="${basedir}/target/release-package-${ignite.edition}" perm="755" includes="**/*.sh" />
<!--
Line ending bugfix for builds packaged in MS Windows machine to adjust Linux based
end-of-line characters for Linux shell scripts
-->
- <fixcrlf srcdir="${basedir}/target/release-package" eol="lf" eof="remove">
+ <fixcrlf srcdir="${basedir}/target/release-package-${ignite.edition}" eol="lf" eof="remove">
<include name="**/*.sh" />
</fixcrlf>
<zip destfile="${basedir}/target/bin/${ignite.zip.pattern}.zip" encoding="UTF-8">
- <zipfileset dir="${basedir}/target/release-package" prefix="${ignite.zip.pattern}" filemode="755">
+ <zipfileset dir="${basedir}/target/release-package-${ignite.edition}" prefix="${ignite.zip.pattern}" filemode="755">
<include name="**/*.sh" />
<include name="**/configure" />
<include name="**/install-sh" />
<include name="**/compile" />
<include name="**/missing" />
</zipfileset>
- <zipfileset dir="${basedir}/target/release-package" prefix="${ignite.zip.pattern}">
+ <zipfileset dir="${basedir}/target/release-package-${ignite.edition}" prefix="${ignite.zip.pattern}">
<exclude name="**/*.sh" />
<exclude name="**/configure" />
<exclude name="**/install-sh" />
@@ -418,8 +406,8 @@
<inherited>false</inherited>
<executions>
<execution>
- <id>dependencies</id>
- <phase>prepare-package</phase>
+ <id>ignite-dependencies</id>
+ <phase>validate</phase>
<goals>
<goal>single</goal>
</goals>
@@ -427,15 +415,15 @@
<descriptors>
<descriptor>${basedir}/assembly/dependencies-${ignite.edition}.xml</descriptor>
</descriptors>
- <outputDirectory>${basedir}/target/release-package</outputDirectory>
+ <outputDirectory>${basedir}/target/release-package-${ignite.edition}</outputDirectory>
<finalName>libs</finalName>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
</execution>
<execution>
- <id>release</id>
- <phase>prepare-package</phase>
+ <id>ignite-release</id>
+ <phase>validate</phase>
<goals>
<goal>single</goal>
</goals>
@@ -443,14 +431,14 @@
<descriptors>
<descriptor>assembly/release-${ignite.edition}.xml</descriptor>
</descriptors>
- <finalName>release-package</finalName>
+ <finalName>release-package-${ignite.edition}</finalName>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
</execution>
<execution>
<id>dependencies-visor-console</id>
- <phase>prepare-package</phase>
+ <phase>validate</phase>
<goals>
<goal>single</goal>
</goals>
@@ -458,7 +446,7 @@
<descriptors>
<descriptor>assembly/dependencies-visor-console.xml</descriptor>
</descriptors>
- <outputDirectory>target/release-package/bin</outputDirectory>
+ <outputDirectory>target/release-package-${ignite.edition}/bin</outputDirectory>
<finalName>include</finalName>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
@@ -466,7 +454,7 @@
<execution>
<id>scala-scripts</id>
- <phase>prepare-package</phase>
+ <phase>validate</phase>
<goals>
<goal>single</goal>
</goals>
@@ -474,7 +462,7 @@
<descriptors>
<descriptor>assembly/release-scala.xml</descriptor>
</descriptors>
- <outputDirectory>target/release-package</outputDirectory>
+ <outputDirectory>target/release-package-${ignite.edition}</outputDirectory>
<finalName>bin</finalName>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
@@ -498,94 +486,6 @@
<modules>
<module>modules/yardstick</module>
</modules>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <inherited>false</inherited>
- <dependencies>
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-tools</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>release-yardstick</id>
- <goals>
- <goal>run</goal>
- </goals>
- <phase>prepare-package</phase>
- <configuration>
- <target>
- <mkdir dir="${basedir}/target/release-package/benchmarks" />
-
- <copy todir="${basedir}/target/release-package/benchmarks/">
- <fileset dir="${basedir}/modules/yardstick/target/assembly/" />
- </copy>
-
- <!--todo: only required jars should be exported to /benchmarks/libs during compilation-->
- <delete>
- <fileset dir="${basedir}/target/release-package/benchmarks/libs/">
- <include name="junit-*.jar" />
- <include name="ignite-apache-license-gen-*.jar" />
- <include name="hamcrest-core-*.jar" />
- <include name="tools-*.jar" />
- </fileset>
- </delete>
-
- <!--todo: config structure should be refactored to be the same at "sources" and "released sources"-->
- <delete>
- <fileset dir="${basedir}/target/release-package/benchmarks/config/">
- <include name="*.*" />
- <exclude name="benchmark.properties" />
- <exclude name="benchmark-remote.properties" />
- <exclude name="benchmark-sample.properties" />
- <exclude name="benchmark-remote-sample.properties" />
- <exclude name="benchmark-multicast.properties" />
- <exclude name="ignite-base-config.xml" />
- <exclude name="ignite-localhost-config.xml" />
- <exclude name="ignite-remote-config.xml" />
- <exclude name="ignite-multicast-config.xml" />
- </fileset>
- </delete>
-
- <mkdir dir="${basedir}/target/release-package/benchmarks/sources/src" />
-
- <copy todir="${basedir}/target/release-package/benchmarks/sources/src/">
- <fileset dir="${basedir}/modules/yardstick/src" />
- </copy>
-
- <mkdir dir="${basedir}/target/release-package/benchmarks/sources/config" />
-
- <copy todir="${basedir}/target/release-package/benchmarks/sources/config/">
- <fileset dir="${basedir}/target/release-package/benchmarks/config" />
- </copy>
-
- <copy file="${basedir}/modules/yardstick/pom-standalone.xml" tofile="${basedir}/target/release-package/benchmarks/sources/pom.xml" />
-
- <replaceregexp byline="true">
- <regexp pattern="to_be_replaced_by_ignite_version" />
- <substitution expression="${project.version}" />
- <fileset dir="${basedir}/target/release-package/benchmarks/sources/">
- <include name="pom.xml" />
- </fileset>
- </replaceregexp>
-
- <copy file="${basedir}/modules/yardstick/README.txt" tofile="${basedir}/target/release-package/benchmarks/README.txt" overwrite="true">
- </copy>
-
- <copy file="${basedir}/modules/yardstick/DEVNOTES-standalone.txt" tofile="${basedir}/target/release-package/benchmarks/sources/DEVNOTES.txt" />
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
</profile>
<profile>
@@ -658,7 +558,7 @@
</execution>
<execution>
<id>source-release-assembly-ignite</id>
- <phase>package</phase>
+ <phase>initialize</phase>
<goals>
<goal>single</goal>
</goals>
@@ -681,7 +581,7 @@
<executions>
<execution>
<id>attach-artifact</id><!-- allows to sign *bin.zip files -->
- <phase>package</phase>
+ <phase>initialize</phase>
<goals>
<goal>run</goal>
</goals>
@@ -744,7 +644,7 @@
<goals>
<goal>files</goal>
</goals>
- <phase>install</phase>
+ <phase>initialize</phase>
</execution>
</executions>
<configuration>
@@ -776,7 +676,7 @@
<goals>
<goal>run</goal>
</goals>
- <phase>install</phase>
+ <phase>initialize</phase>
<configuration>
<failOnError>false</failOnError>
<target>
[04/12] ignite git commit: IGNITE-5154: Added RemoteFilter to C++
ContinuosQuery example.
Posted by sb...@apache.org.
IGNITE-5154: Added RemoteFilter to C++ ContinuosQuery example.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/41dcce7b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/41dcce7b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/41dcce7b
Branch: refs/heads/ignite-5075
Commit: 41dcce7b8adb86e5a053f62c658e36a9c67c35f5
Parents: abb9d94
Author: Igor Sapego <is...@gridgain.com>
Authored: Wed May 31 18:34:21 2017 +0300
Committer: Igor Sapego <is...@gridgain.com>
Committed: Wed May 31 18:34:21 2017 +0300
----------------------------------------------------------------------
.../src/continuous_query_example.cpp | 125 ++++++++++++++++++-
1 file changed, 119 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/41dcce7b/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp b/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
index b08d4b8..2ca3163 100644
--- a/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
+++ b/modules/platforms/cpp/examples/continuous-query-example/src/continuous_query_example.cpp
@@ -33,14 +33,14 @@ using namespace examples;
/** Cache name. */
const char* CACHE_NAME = "cpp_cache_continuous_query";
-/*
+/**
* Listener class.
*/
template<typename K, typename V>
class Listener : public event::CacheEntryEventListener<K, V>
{
public:
- /*
+ /**
* Default constructor.
*/
Listener()
@@ -65,6 +65,110 @@ public:
}
};
+/**
+ * Range Filter. Only lets through keys from the specified range.
+ */
+template<typename K, typename V>
+struct RangeFilter : event::CacheEntryEventFilter<int32_t, std::string>
+{
+ /**
+ * Default constructor.
+ */
+ RangeFilter() :
+ rangeBegin(0),
+ rangeEnd(0)
+ {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param from Range beginning. Inclusive.
+ * @param to Range end. Not inclusive.
+ */
+ RangeFilter(const K& from, const K& to) :
+ rangeBegin(from),
+ rangeEnd(to)
+ {
+ // No-op.
+ }
+
+ /**
+ * Destructor.
+ */
+ virtual ~RangeFilter()
+ {
+ // No-op.
+ }
+
+ /**
+ * Event callback.
+ *
+ * @param event Event.
+ * @return True if the event passes filter.
+ */
+ virtual bool Process(const CacheEntryEvent<K, V>& event)
+ {
+ return event.GetKey() >= rangeBegin && event.GetKey() < rangeEnd;
+ }
+
+ /** Beginning of the range. */
+ K rangeBegin;
+
+ /** End of the range. */
+ K rangeEnd;
+};
+
+namespace ignite
+{
+ namespace binary
+ {
+ template<>
+ struct BinaryType< RangeFilter<int32_t, std::string> >
+ {
+ static int32_t GetTypeId()
+ {
+ return GetBinaryStringHashCode("RangeFilter<int32_t,std::string>");
+ }
+
+ static void GetTypeName(std::string& dst)
+ {
+ dst = "RangeFilter<int32_t,std::string>";
+
+ }
+
+ static int32_t GetFieldId(const char* name)
+ {
+ return GetBinaryStringHashCode(name);
+ }
+
+ static bool IsNull(const RangeFilter<int32_t, std::string>&)
+ {
+ return false;
+ }
+
+ static void GetNull(RangeFilter<int32_t, std::string>& dst)
+ {
+ dst = RangeFilter<int32_t, std::string>();
+ }
+
+ static void Write(BinaryWriter& writer, const RangeFilter<int32_t, std::string>& obj)
+ {
+ writer.WriteInt32("rangeBegin", obj.rangeBegin);
+ writer.WriteInt32("rangeEnd", obj.rangeEnd);
+ }
+
+ static void Read(BinaryReader& reader, RangeFilter<int32_t, std::string>& dst)
+ {
+ dst.rangeBegin = reader.ReadInt32("rangeBegin");
+ dst.rangeEnd = reader.ReadInt32("rangeEnd");
+ }
+ };
+ }
+}
+
+
int main()
{
IgniteConfiguration cfg;
@@ -80,6 +184,12 @@ int main()
std::cout << ">>> Cache continuous query example started." << std::endl;
std::cout << std::endl;
+ // Get binding.
+ IgniteBinding binding = ignite.GetBinding();
+
+ // Registering remote filter.
+ binding.RegisterCacheEntryEventFilter< RangeFilter<int32_t, std::string> >();
+
// Get cache instance.
Cache<int32_t, std::string> cache = ignite.GetOrCreateCache<int32_t, std::string>(CACHE_NAME);
@@ -97,17 +207,20 @@ int main()
}
// Declaring listener.
- Listener<int, std::string> listener;
+ Listener<int32_t, std::string> listener;
+
+ // Declaring filter.
+ RangeFilter<int32_t, std::string> filter(keyCnt, keyCnt + 5);
// Declaring continuous query.
- continuous::ContinuousQuery<int, std::string> qry(MakeReference(listener));
+ continuous::ContinuousQuery<int32_t, std::string> qry(MakeReference(listener), MakeReference(filter));
{
// Continous query scope. Query is closed when scope is left.
- continuous::ContinuousQueryHandle<int, std::string> handle = cache.QueryContinuous(qry);
+ continuous::ContinuousQueryHandle<int32_t, std::string> handle = cache.QueryContinuous(qry);
// Add a few more keys and watch more query notifications.
- for (int32_t i = keyCnt; i < keyCnt + 5; ++i)
+ for (int32_t i = keyCnt; i < keyCnt + 10; ++i)
{
std::stringstream builder;
[09/12] ignite git commit: IGNITE-4724: SQL: fixed return type of AVG
operation. This closes #1989.
Posted by sb...@apache.org.
IGNITE-4724: SQL: fixed return type of AVG operation. This closes #1989.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c4883113
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c4883113
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c4883113
Branch: refs/heads/ignite-5075
Commit: c4883113dd54a13ea68ed07548f838fde513f9f1
Parents: 54f499c
Author: Sergey Kalashnikov <sk...@gridgain.com>
Authored: Wed May 31 21:39:21 2017 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Wed May 31 21:39:21 2017 +0300
----------------------------------------------------------------------
.../query/h2/sql/GridSqlQuerySplitter.java | 22 +-
...ClientReconnectCacheQueriesFailoverTest.java | 2 +-
.../query/IgniteSqlSplitterSelfTest.java | 256 +++++++++++++++----
3 files changed, 223 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c4883113/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 8fcd7dc..e2b82ab 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -46,6 +46,7 @@ import org.h2.command.Prepared;
import org.h2.command.dml.Query;
import org.h2.command.dml.SelectUnion;
import org.h2.jdbc.JdbcPreparedStatement;
+import org.h2.value.Value;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2CollocationModel.isCollocated;
@@ -1836,12 +1837,12 @@ public class GridSqlQuerySplitter {
/* Note Distinct aggregate can be performed only on reduce phase, so
if query contains distinct aggregate then other aggregates must be processed the same way. */
switch (agg.type()) {
- case AVG: // SUM( AVG(CAST(x AS DOUBLE))*COUNT(x) )/SUM( COUNT(x) ) or AVG(CAST( x AS DOUBLE))
+ case AVG: // SUM( AVG(CAST(x AS DOUBLE))*COUNT(x) )/SUM( COUNT(x) ) or AVG(x)
if (hasDistinctAggregate) /* and has no collocated group by */ {
mapAgg = agg.child();
- rdcAgg = aggregate(agg.distinct(), agg.type()).resultType(GridSqlType.DOUBLE)
- .addChild(function(CAST).resultType(GridSqlType.DOUBLE).addChild(column(mapAggAlias.alias())));
+ rdcAgg = aggregate(agg.distinct(), agg.type()).resultType(agg.resultType())
+ .addChild(column(mapAggAlias.alias()));
}
else {
//-- COUNT(x) map
@@ -1868,7 +1869,13 @@ public class GridSqlQuerySplitter {
GridSqlElement sumDownRdc = aggregate(false, SUM).addChild(column(cntMapAggAlias));
- rdcAgg = op(GridSqlOperationType.DIVIDE, sumUpRdc, sumDownRdc);
+ if (!isFractionalType(agg.resultType().type())) {
+ sumUpRdc = function(CAST).resultType(GridSqlType.BIGINT).addChild(sumUpRdc);
+ sumDownRdc = function(CAST).resultType(GridSqlType.BIGINT).addChild(sumDownRdc);
+ }
+
+ rdcAgg = function(CAST).resultType(agg.resultType())
+ .addChild(op(GridSqlOperationType.DIVIDE, sumUpRdc, sumDownRdc));
}
break;
@@ -1974,6 +1981,13 @@ public class GridSqlQuerySplitter {
}
/**
+ * @param type data type id
+ * @return true if given type is fractional
+ */
+ private static boolean isFractionalType(int type) {
+ return type == Value.DECIMAL || type == Value.FLOAT || type == Value.DOUBLE;
+ }
+ /**
* Simplified tree-like model for a query.
* - SELECT : All the children are list of joined query models in the FROM clause.
* - UNION : All the children are united left and right query models.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c4883113/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientReconnectCacheQueriesFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientReconnectCacheQueriesFailoverTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientReconnectCacheQueriesFailoverTest.java
index 51620b1..39634cb 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientReconnectCacheQueriesFailoverTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientReconnectCacheQueriesFailoverTest.java
@@ -101,7 +101,7 @@ public class IgniteClientReconnectCacheQueriesFailoverTest extends IgniteClientR
assertEquals(1, res.size());
- Double avg = (Double)res.get(0).get(0);
+ Integer avg = (Integer)res.get(0).get(0);
assertEquals(5_000, avg.intValue());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c4883113/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index 9b5c17a..d8f71a3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.query;
import java.io.Serializable;
+import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -1778,6 +1779,98 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
}
}
+ /**
+ * Check results of aggregate functions if no rows are selected.
+ *
+ * @throws Exception If failed,
+ */
+ public void testEmptyCacheAggregates() throws Exception {
+ final String cacheName = "ints";
+
+ IgniteCache<Integer, Value> cache = ignite(0).getOrCreateCache(cacheConfig(cacheName, true,
+ Integer.class, Value.class));
+
+ try (QueryCursor<List<?>> qry = cache.query(new SqlFieldsQuery(
+ "SELECT count(fst), sum(snd), avg(snd), min(snd), max(snd) FROM Value"))) {
+ List<List<?>> result = qry.getAll();
+
+ assertEquals(1, result.size());
+
+ List<?> row = result.get(0);
+
+ assertEquals("count", 0L, ((Number)row.get(0)).longValue());
+ assertEquals("sum", null, row.get(1));
+ assertEquals("avg", null, row.get(2));
+ assertEquals("min", null, row.get(3));
+ assertEquals("max", null, row.get(4));
+ }
+ finally {
+ cache.destroy();
+ }
+ }
+
+ /**
+ * Check avg() with various data types.
+ *
+ * @throws Exception If failed.
+ */
+ public void testAvgVariousDataTypes() throws Exception {
+ final String cacheName = "avgtypes";
+
+ IgniteCache<Integer, AvgDataTypes> cache =
+ ignite(0).getOrCreateCache(cacheConfig(cacheName, true, Integer.class, AvgDataTypes.class));
+
+ // avg 13.125; int avg 13
+ double value[] = new double[] {1, 5, 7, 8, 10.5, 13.5, 20, 40};
+
+ for (int i = 0; i < value.length; i++) {
+ Number v = value[i];
+
+ cache.put(i, new AvgDataTypes(
+ v.byteValue(),
+ v.shortValue(),
+ v.intValue(),
+ v.longValue(),
+ new BigDecimal(v.toString()),
+ v.floatValue(),
+ v.doubleValue()));
+ }
+
+ try {
+ checkAvgWithVariousTypes(cache, false);
+ checkAvgWithVariousTypes(cache, true);
+ }
+ finally {
+ cache.destroy();
+ }
+ }
+
+ /**
+ * Check avg() with various data types.
+ *
+ * @param cache Cache.
+ * @param distinct Distinct flag.
+ */
+ private void checkAvgWithVariousTypes(IgniteCache<Integer, AvgDataTypes> cache, boolean distinct) {
+ String qryText = String.format("select avg(%1$s byteField), avg(%1$s shortField), " +
+ "avg(%1$s intField), avg(%1$s longField), avg(%1$s decimalField), " +
+ "avg(%1$s floatField), avg(%1$s doubleField) from AvgDataTypes", distinct ? "distinct" : "");
+
+ SqlFieldsQuery qry = new SqlFieldsQuery(qryText);
+
+ List<List<?>> result = cache.query(qry).getAll();
+
+ List<?> row = result.get(0);
+
+ assertEquals((byte)13, row.get(0));
+ assertEquals((short)13, row.get(1));
+ assertEquals(13, row.get(2));
+ assertEquals(13L, row.get(3));
+ assertEquals(new BigDecimal("13.125"), row.get(4));
+ assertEquals(13.125f, row.get(5));
+ assertEquals(13.125d, row.get(6));
+ }
+
/** Simple query with aggregates */
private void checkSimpleQueryWithAggr(IgniteCache<Integer, Value> cache) {
try (QueryCursor<List<?>> qry = cache.query(new SqlFieldsQuery(
@@ -1790,7 +1883,7 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
assertEquals("count", 15L, ((Number)row.get(0)).longValue());
assertEquals("sum", 30L, ((Number)row.get(1)).longValue());
- assertEquals("avg", 2.0d, ((Number)row.get(2)).doubleValue(), 0.001);
+ assertEquals("avg", 2, ((Integer)row.get(2)).intValue());
assertEquals("min", 1, ((Integer)row.get(3)).intValue());
assertEquals("max", 3, ((Integer)row.get(4)).intValue());
}
@@ -1809,7 +1902,7 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
assertEquals("count distinct", 6L, ((Number)row.get(0)).longValue());
assertEquals("sum distinct", 6L, ((Number)row.get(1)).longValue());
- assertEquals("avg distinct", 2.0d, ((Number)row.get(2)).doubleValue(), 0.001);
+ assertEquals("avg distinct", 2, ((Integer)row.get(2)).intValue());
assertEquals("min distinct", 1, ((Integer)row.get(3)).intValue());
assertEquals("max distinct", 3, ((Integer)row.get(4)).intValue());
}
@@ -1829,12 +1922,12 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
assertEquals("count", 15L, ((Number)row.get(0)).longValue());
assertEquals("sum", 30L, ((Number)row.get(1)).longValue());
- assertEquals("avg", 2.0d, ((Number)row.get(2)).doubleValue(), 0.001);
+ assertEquals("avg", 2, ((Integer)row.get(2)).intValue());
assertEquals("min", 1, ((Integer)row.get(3)).intValue());
assertEquals("max", 3, ((Integer)row.get(4)).intValue());
assertEquals("count distinct", 6L, ((Number)row.get(5)).longValue());
assertEquals("sum distinct", 6L, ((Number)row.get(6)).longValue());
- assertEquals("avg distinct", 2.0d, ((Number)row.get(7)).doubleValue(), 0.001);
+ assertEquals("avg distinct", 2, ((Integer)row.get(7)).intValue());
assertEquals("min distinct", 1, ((Integer)row.get(8)).intValue());
assertEquals("max distinct", 3, ((Integer)row.get(9)).intValue());
}
@@ -1843,7 +1936,8 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
/** Query with aggregates and groups */
private void checkQueryWithGroupsAndAggrs(IgniteCache<Integer, Value> cache) {
try (QueryCursor<List<?>> qry = cache.query(new SqlFieldsQuery(
- "SELECT fst, count(snd), sum(snd), avg(snd), min(snd), max(snd) FROM Value GROUP BY fst ORDER BY fst"))) {
+ "SELECT fst, count(snd), sum(snd), avg(snd), avg(CAST(snd AS DOUBLE)), " +
+ "min(snd), max(snd) FROM Value GROUP BY fst ORDER BY fst"))) {
List<List<?>> result = qry.getAll();
assertEquals(6, result.size());
@@ -1852,33 +1946,39 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
assertEquals("fst", 1, ((Number)row.get(0)).intValue());
assertEquals("count", 3L, ((Number)row.get(1)).longValue());
assertEquals("sum", 9L, ((Number)row.get(2)).longValue());
- assertEquals("avg", 3.0d, ((Number)row.get(3)).doubleValue(), 0.001);
- assertEquals("min", 3, ((Integer)row.get(4)).intValue());
- assertEquals("max", 3, ((Integer)row.get(5)).intValue());
+ assertEquals("avg", 3, ((Number)row.get(3)).doubleValue(), 0.001);
+ assertEquals("avg dbl", 3d, ((Number)row.get(4)).doubleValue(), 0.001);
+ assertEquals("min", 3, ((Integer)row.get(5)).intValue());
+ assertEquals("max", 3, ((Integer)row.get(6)).intValue());
+
row = result.get(1);
assertEquals("fst", 2, ((Number)row.get(0)).intValue());
assertEquals("count", 3L, ((Number)row.get(1)).longValue());
assertEquals("sum", 6L, ((Number)row.get(2)).longValue());
- assertEquals("avg", 2.0d, ((Number)row.get(3)).doubleValue(), 0.001);
- assertEquals("min", 1, ((Integer)row.get(4)).intValue());
- assertEquals("max", 3, ((Integer)row.get(5)).intValue());
+ assertEquals("avg", 2, ((Number)row.get(3)).doubleValue(), 0.001);
+ assertEquals("avg dbl", 2d, ((Number)row.get(4)).doubleValue(), 0.001);
+ assertEquals("min", 1, ((Integer)row.get(5)).intValue());
+ assertEquals("max", 3, ((Integer)row.get(6)).intValue());
+
row = result.get(2);
assertEquals("fst", 3, ((Number)row.get(0)).intValue());
assertEquals("count", 6L, ((Number)row.get(1)).longValue());
assertEquals("sum", 9L, ((Number)row.get(2)).longValue());
- assertEquals("avg", 1.5d, ((Number)row.get(3)).doubleValue(), 0.001);
- assertEquals("min", 1, ((Integer)row.get(4)).intValue());
- assertEquals("max", 2, ((Integer)row.get(5)).intValue());
+ assertEquals("avg", 1, ((Integer)row.get(3)).intValue());
+ assertEquals("avg dbl", 1.5d, ((Number)row.get(4)).doubleValue(), 0.001);
+ assertEquals("min", 1, ((Integer)row.get(5)).intValue());
+ assertEquals("max", 2, ((Integer)row.get(6)).intValue());
}
}
/** Query with distinct aggregates and groups */
private void checkQueryWithGroupsAndDistinctAggr(IgniteCache<Integer, Value> cache) {
try (QueryCursor<List<?>> qry = cache.query(new SqlFieldsQuery(
- "SELECT count(distinct snd), sum(distinct snd), avg(distinct snd), min(distinct snd), max(distinct snd) " +
- "FROM Value GROUP BY fst"))) {
+ "SELECT count(distinct snd), sum(distinct snd), avg(distinct snd), " +
+ "avg(distinct cast(snd as double)), min(distinct snd), max(distinct snd) " +
+ "FROM Value GROUP BY fst"))) {
List<List<?>> result = qry.getAll();
assertEquals(6, result.size());
@@ -1886,32 +1986,35 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
List<?> row = result.get(0);
assertEquals("count distinct", 1L, ((Number)row.get(0)).longValue());
assertEquals("sum distinct", 3L, ((Number)row.get(1)).longValue());
- assertEquals("avg distinct", 3.0d, ((Number)row.get(2)).doubleValue(), 0.001);
- assertEquals("min distinct", 3, ((Integer)row.get(3)).intValue());
- assertEquals("max distinct", 3, ((Integer)row.get(4)).intValue());
+ assertEquals("avg distinct", 3, ((Integer)row.get(2)).intValue());
+ assertEquals("avg distinct dbl", 3.0d, ((Number)row.get(3)).doubleValue(), 0.001);
+ assertEquals("min distinct", 3, ((Integer)row.get(4)).intValue());
+ assertEquals("max distinct", 3, ((Integer)row.get(5)).intValue());
row = result.get(1);
assertEquals("count distinct", 3L, ((Number)row.get(0)).longValue());
assertEquals("sum distinct", 6L, ((Number)row.get(1)).longValue());
- assertEquals("avg distinct", 2.0d, ((Number)row.get(2)).doubleValue(), 0.001);
- assertEquals("min distinct", 1, ((Integer)row.get(3)).intValue());
- assertEquals("max distinct", 3, ((Integer)row.get(4)).intValue());
+ assertEquals("avg distinct", 2, ((Integer)row.get(2)).intValue());
+ assertEquals("avg distinct dbl", 2.0d, ((Number)row.get(3)).doubleValue(), 0.001);
+ assertEquals("min distinct", 1, ((Integer)row.get(4)).intValue());
+ assertEquals("max distinct", 3, ((Integer)row.get(5)).intValue());
row = result.get(2);
assertEquals("count distinct", 2L, ((Number)row.get(0)).longValue());
assertEquals("sum distinct", 3L, ((Number)row.get(1)).longValue());
- assertEquals("avg distinct", 1.5d, ((Number)row.get(2)).doubleValue(), 0.001);
- assertEquals("min distinct", 1, ((Integer)row.get(3)).intValue());
- assertEquals("max distinct", 2, ((Integer)row.get(4)).intValue());
+ assertEquals("avg distinct", 1, ((Integer)row.get(2)).intValue());
+ assertEquals("avg distinct dbl", 1.5d, ((Number)row.get(3)).doubleValue(), 0.001);
+ assertEquals("min distinct", 1, ((Integer)row.get(4)).intValue());
+ assertEquals("max distinct", 2, ((Integer)row.get(5)).intValue());
}
}
/** Query with distinct aggregates and groups */
private void checkQueryWithGroupsAndAggrMixed(IgniteCache<Integer, Value> cache) {
try (QueryCursor<List<?>> qry = cache.query(new SqlFieldsQuery(
- "SELECT fst, count(snd), sum(snd), avg(snd), min(snd), max(snd)," +
- "count(distinct snd), sum(distinct snd), avg(distinct snd), min(distinct snd), max(distinct snd) " +
- "FROM Value GROUP BY fst"))) {
+ "SELECT fst, count(snd), sum(snd), avg(snd), avg(cast(snd as double)), min(snd), max(snd)," +
+ "count(distinct snd), sum(distinct snd), avg(distinct snd), avg(distinct cast(snd as double)), " +
+ "min(distinct snd), max(distinct snd) FROM Value GROUP BY fst"))) {
List<List<?>> result = qry.getAll();
assertEquals(6, result.size());
@@ -1920,40 +2023,46 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
assertEquals("fst", 1, ((Number)row.get(0)).intValue());
assertEquals("count", 3L, ((Number)row.get(1)).longValue());
assertEquals("sum", 9L, ((Number)row.get(2)).longValue());
- assertEquals("avg", 3.0d, ((Number)row.get(3)).doubleValue(), 0.001);
- assertEquals("min", 3, ((Integer)row.get(4)).intValue());
- assertEquals("max", 3, ((Integer)row.get(5)).intValue());
- assertEquals("count distinct", 1L, ((Number)row.get(6)).longValue());
- assertEquals("sum distinct", 3L, ((Number)row.get(7)).longValue());
- assertEquals("avg distinct", 3.0d, ((Number)row.get(8)).doubleValue(), 0.001);
- assertEquals("min distinct", 3, ((Integer)row.get(9)).intValue());
- assertEquals("max distinct", 3, ((Integer)row.get(10)).intValue());
+ assertEquals("avg", 3, ((Integer)row.get(3)).intValue());
+ assertEquals("avg dbl", 3.0d, ((Number)row.get(4)).doubleValue(), 0.001);
+ assertEquals("min", 3, ((Integer)row.get(5)).intValue());
+ assertEquals("max", 3, ((Integer)row.get(6)).intValue());
+ assertEquals("count distinct", 1L, ((Number)row.get(7)).longValue());
+ assertEquals("sum distinct", 3L, ((Number)row.get(8)).longValue());
+ assertEquals("avg distinct", 3, ((Integer)row.get(9)).intValue());
+ assertEquals("avg distinct dbl", 3.0d, ((Number)row.get(10)).doubleValue(), 0.001);
+ assertEquals("min distinct", 3, ((Integer)row.get(11)).intValue());
+ assertEquals("max distinct", 3, ((Integer)row.get(12)).intValue());
row = result.get(1);
assertEquals("fst", 2, ((Number)row.get(0)).intValue());
assertEquals("count", 3L, ((Number)row.get(1)).longValue());
assertEquals("sum", 6L, ((Number)row.get(2)).longValue());
- assertEquals("avg", 2.0d, ((Number)row.get(3)).doubleValue(), 0.001);
- assertEquals("min", 1, ((Integer)row.get(4)).intValue());
- assertEquals("max", 3, ((Integer)row.get(5)).intValue());
- assertEquals("count distinct", 3L, ((Number)row.get(6)).longValue());
- assertEquals("sum distinct", 6L, ((Number)row.get(7)).longValue());
- assertEquals("avg distinct", 2.0d, ((Number)row.get(8)).doubleValue(), 0.001);
- assertEquals("min distinct", 1, ((Integer)row.get(9)).intValue());
- assertEquals("max distinct", 3, ((Integer)row.get(10)).intValue());
+ assertEquals("avg", 2, ((Integer)row.get(3)).intValue());
+ assertEquals("avg dbl", 2.0d, ((Number)row.get(4)).doubleValue(), 0.001);
+ assertEquals("min", 1, ((Integer)row.get(5)).intValue());
+ assertEquals("max", 3, ((Integer)row.get(6)).intValue());
+ assertEquals("count distinct", 3L, ((Number)row.get(7)).longValue());
+ assertEquals("sum distinct", 6L, ((Number)row.get(8)).longValue());
+ assertEquals("avg distinct", 2, ((Integer)row.get(9)).intValue());
+ assertEquals("avg distinct dbl", 2.0d, ((Number)row.get(10)).doubleValue(), 0.001);
+ assertEquals("min distinct", 1, ((Integer)row.get(11)).intValue());
+ assertEquals("max distinct", 3, ((Integer)row.get(12)).intValue());
row = result.get(2);
assertEquals("fst", 3, ((Number)row.get(0)).intValue());
assertEquals("count", 6L, ((Number)row.get(1)).longValue());
assertEquals("sum", 9L, ((Number)row.get(2)).longValue());
- assertEquals("avg", 1.5d, ((Number)row.get(3)).doubleValue(), 0.001);
- assertEquals("min", 1, ((Integer)row.get(4)).intValue());
- assertEquals("max", 2, ((Integer)row.get(5)).intValue());
- assertEquals("count distinct", 2L, ((Number)row.get(6)).longValue());
- assertEquals("sum distinct", 3L, ((Number)row.get(7)).longValue());
- assertEquals("avg distinct", 1.5d, ((Number)row.get(8)).doubleValue(), 0.001);
- assertEquals("min distinct", 1, ((Integer)row.get(9)).intValue());
- assertEquals("max distinct", 2, ((Integer)row.get(10)).intValue());
+ assertEquals("avg", 1, ((Integer)row.get(3)).intValue());
+ assertEquals("avg dbl", 1.5d, ((Number)row.get(4)).doubleValue(), 0.001);
+ assertEquals("min", 1, ((Integer)row.get(5)).intValue());
+ assertEquals("max", 2, ((Integer)row.get(6)).intValue());
+ assertEquals("count distinct", 2L, ((Number)row.get(7)).longValue());
+ assertEquals("sum distinct", 3L, ((Number)row.get(8)).longValue());
+ assertEquals("avg distinct", 1, ((Integer)row.get(9)).intValue());
+ assertEquals("avg distinct dbl", 1.5d, ((Number)row.get(10)).doubleValue(), 0.001);
+ assertEquals("min distinct", 1, ((Integer)row.get(11)).intValue());
+ assertEquals("max distinct", 2, ((Integer)row.get(12)).intValue());
}
}
@@ -2245,4 +2354,47 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
this.OFFER_ID = offer_Id;
}
}
+
+ /** */
+ public class AvgDataTypes {
+ /** */
+ @QuerySqlField
+ private Byte byteField;
+
+ /** */
+ @QuerySqlField
+ private Short shortField;
+
+ /** */
+ @QuerySqlField
+ private Integer intField;
+
+ /** */
+ @QuerySqlField
+ private Long longField;
+
+ /** */
+ @QuerySqlField
+ private BigDecimal decimalField;
+
+ /** */
+ @QuerySqlField
+ private Float floatField;
+
+ /** */
+ @QuerySqlField
+ private Double doubleField;
+
+ /** */
+ public AvgDataTypes(Byte byteField, Short shortField, Integer intField, Long longField,
+ BigDecimal decimalField, Float floatField, Double doubleField) {
+ this.byteField = byteField;
+ this.shortField = shortField;
+ this.intField = intField;
+ this.longField = longField;
+ this.decimalField = decimalField;
+ this.floatField = floatField;
+ this.doubleField = doubleField;
+ }
+ }
}
[03/12] ignite git commit: IGNITE-5342 - Skip permission check for
TASK_EXECUTE for service jobs
Posted by sb...@apache.org.
IGNITE-5342 - Skip permission check for TASK_EXECUTE for service jobs
(cherry picked from commit d77a134)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/abb9d942
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/abb9d942
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/abb9d942
Branch: refs/heads/ignite-5075
Commit: abb9d9428696ddb3a250e9a491d4abea437d2a38
Parents: 858d11b
Author: dkarachentsev <dk...@gridgain.com>
Authored: Wed May 31 16:57:29 2017 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Wed May 31 16:57:29 2017 +0300
----------------------------------------------------------------------
.../cache/CacheAffinitySharedManager.java | 52 +--
.../processors/cache/GridCacheProcessor.java | 39 +-
.../GridDhtPartitionsExchangeFuture.java | 2 +-
...IgniteClientCacheInitializationFailTest.java | 367 +++++++++++++++++++
.../testsuites/IgniteCacheTestSuite4.java | 2 +
5 files changed, 432 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/abb9d942/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 889eeda..9063a12 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -396,40 +396,48 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
CU.affinityNode(cctx.localNode(), req.startCacheConfiguration().getNodeFilter());
}
- if (startCache) {
- cctx.cache().prepareCacheStart(cacheDesc, nearCfg, fut.topologyVersion());
+ try {
+ if (startCache) {
+ cctx.cache().prepareCacheStart(cacheDesc, nearCfg, fut.topologyVersion());
- if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) {
- if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
- U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+ if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) {
+ if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
+ U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+ }
}
- }
- if (!crd || !lateAffAssign) {
- GridCacheContext cacheCtx = cctx.cacheContext(cacheDesc.cacheId());
+ if (!crd || !lateAffAssign) {
+ GridCacheContext cacheCtx = cctx.cacheContext(cacheDesc.cacheId());
- if (cacheCtx != null && !cacheCtx.isLocal()) {
- boolean clientCacheStarted =
- req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
+ if (cacheCtx != null && !cacheCtx.isLocal()) {
+ boolean clientCacheStarted =
+ req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
- if (clientCacheStarted)
- initAffinity(cacheDesc, cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
- else if (!req.clientStartOnly()) {
- assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
+ if (clientCacheStarted)
+ initAffinity(cacheDesc, cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
+ else if (!req.clientStartOnly()) {
+ assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
- GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
+ GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
- assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
+ assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
- List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
- fut.discoveryEvent(), fut.discoCache());
+ List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
+ fut.discoveryEvent(), fut.discoCache());
- aff.initialize(fut.topologyVersion(), assignment);
+ aff.initialize(fut.topologyVersion(), assignment);
+ }
}
}
+ else
+ initStartedCacheOnCoordinator(fut, cacheDesc.cacheId());
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
+ "[cacheName=" + req.cacheName() + ']', e);
+
+ cctx.cache().forceCloseCache(fut.topologyVersion(), action, e);
}
- else
- initStartedCacheOnCoordinator(fut, cacheDesc.cacheId());
}
for (DynamicCacheChangeRequest req : exchActions.closeRequests(cctx.localNodeId())) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/abb9d942/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index dbcddf4..c7e8670 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1890,8 +1890,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
/**
* @param req Stop request.
*/
- private void prepareCacheStop(DynamicCacheChangeRequest req) {
- assert req.stop() || req.close() : req;
+ private void prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose) {
+ assert req.stop() || req.close() || forceClose : req;
GridCacheAdapter<?, ?> cache = caches.remove(req.cacheName());
@@ -1907,6 +1907,27 @@ public class GridCacheProcessor extends GridProcessorAdapter {
}
/**
+ * Closes cache even if it's not fully initialized (e.g. fail on cache init stage).
+ *
+ * @param topVer Completed topology version.
+ * @param act Exchange action.
+ * @param err Error.
+ */
+ void forceCloseCache(
+ AffinityTopologyVersion topVer,
+ final ExchangeActions.ActionData act,
+ Throwable err
+ ) {
+ ExchangeActions actions = new ExchangeActions(){
+ @Override List<DynamicCacheChangeRequest> closeRequests(UUID nodeId) {
+ return Collections.singletonList(act.request());
+ }
+ };
+
+ onExchangeDone(topVer, actions, err, true);
+ }
+
+ /**
* Callback invoked when first exchange future for dynamic cache is completed.
*
* @param topVer Completed topology version.
@@ -1917,7 +1938,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
public void onExchangeDone(
AffinityTopologyVersion topVer,
@Nullable ExchangeActions exchActions,
- Throwable err
+ Throwable err,
+ boolean forceClose
) {
for (GridCacheAdapter<?, ?> cache : caches.values()) {
GridCacheContext<?, ?> cacheCtx = cache.context();
@@ -1930,11 +1952,11 @@ public class GridCacheProcessor extends GridProcessorAdapter {
}
}
- if (exchActions != null && err == null) {
+ if (exchActions != null && (err == null || forceClose)) {
for (ExchangeActions.ActionData action : exchActions.cacheStopRequests()) {
stopGateway(action.request());
- prepareCacheStop(action.request());
+ prepareCacheStop(action.request(), forceClose);
}
for (DynamicCacheChangeRequest req : exchActions.closeRequests(ctx.localNodeId())) {
@@ -1955,9 +1977,12 @@ public class GridCacheProcessor extends GridProcessorAdapter {
proxy.context().gate().onStopped();
- prepareCacheStop(req);
+ prepareCacheStop(req, forceClose);
}
}
+
+ if (forceClose)
+ completeCacheStartFuture(req, false, err);
}
}
}
@@ -1978,7 +2003,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
* @param req Request to complete future for.
* @param err Error if any.
*/
- void completeCacheStartFuture(DynamicCacheChangeRequest req, boolean success, @Nullable Exception err) {
+ void completeCacheStartFuture(DynamicCacheChangeRequest req, boolean success, @Nullable Throwable err) {
if (req.initiatingNodeId().equals(ctx.localNodeId())) {
DynamicCacheStartFuture fut = (DynamicCacheStartFuture)pendingFuts.get(req.requestId());
http://git-wip-us.apache.org/repos/asf/ignite/blob/abb9d942/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 72c5bbc..2cd5629 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1170,7 +1170,7 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
cacheValidRes = m;
}
- cctx.cache().onExchangeDone(exchId.topologyVersion(), exchActions, err);
+ cctx.cache().onExchangeDone(exchId.topologyVersion(), exchActions, err, false);
cctx.exchange().onExchangeDone(this, err);
http://git-wip-us.apache.org/repos/asf/ignite/blob/abb9d942/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
new file mode 100644
index 0000000..1d616f8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import javax.cache.Cache;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.query.GridQueryCancel;
+import org.apache.ignite.internal.processors.query.GridQueryIndexing;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
+import org.apache.ignite.internal.processors.query.QueryIndexDescriptorImpl;
+import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitor;
+import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.internal.util.lang.GridCloseableIterator;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.spi.indexing.IndexingQueryFilter;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Test checks whether cache initialization error on client side
+ * doesn't causes hangs and doesn't impact other caches.
+ */
+public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractTest {
+ /** Failed cache name. */
+ private static final String CACHE_NAME = "cache";
+
+ /** Atomic cache name. */
+ private static final String ATOMIC_CACHE_NAME = "atomic-cache";
+
+ /** Tx cache name. */
+ private static final String TX_CACHE_NAME = "tx-cache";
+
+ /** Near atomic cache name. */
+ private static final String NEAR_ATOMIC_CACHE_NAME = "near-atomic-cache";
+
+ /** Near tx cache name. */
+ private static final String NEAR_TX_CACHE_NAME = "near-tx-cache";
+
+ /** Failed caches. */
+ private static final Set<String> FAILED_CACHES;
+
+ static {
+ Set<String> set = new HashSet<>();
+
+ set.add(ATOMIC_CACHE_NAME);
+ set.add(TX_CACHE_NAME);
+ set.add(NEAR_ATOMIC_CACHE_NAME);
+ set.add(NEAR_TX_CACHE_NAME);
+
+ FAILED_CACHES = Collections.unmodifiableSet(set);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ startGrid("server");
+ startGrid("client");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ if (gridName.contains("server")) {
+ CacheConfiguration<Integer, String> ccfg1 = new CacheConfiguration<>();
+
+ ccfg1.setIndexedTypes(Integer.class, String.class);
+ ccfg1.setName(ATOMIC_CACHE_NAME);
+ ccfg1.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+ CacheConfiguration<Integer, String> ccfg2 = new CacheConfiguration<>();
+
+ ccfg2.setIndexedTypes(Integer.class, String.class);
+ ccfg2.setName(TX_CACHE_NAME);
+ ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+ cfg.setCacheConfiguration(ccfg1, ccfg2);
+ }
+ else {
+ GridQueryProcessor.idxCls = FailedIndexing.class;
+
+ cfg.setClientMode(true);
+ }
+
+ return cfg;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testAtomicCacheInitialization() throws Exception {
+ checkCacheInitialization(ATOMIC_CACHE_NAME);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testTransactionalCacheInitialization() throws Exception {
+ checkCacheInitialization(TX_CACHE_NAME);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testAtomicNearCacheInitialization() throws Exception {
+ checkCacheInitialization(NEAR_ATOMIC_CACHE_NAME);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testTransactionalNearCacheInitialization() throws Exception {
+ checkCacheInitialization(NEAR_TX_CACHE_NAME);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void checkCacheInitialization(final String cacheName) throws Exception {
+ Ignite client = grid("client");
+
+ checkFailedCache(client, cacheName);
+
+ checkFineCache(client, CACHE_NAME + 1);
+
+ assertNull(client.cache(cacheName));
+ assertNull(client.getOrCreateCache(cacheName));
+
+ checkFineCache(client, CACHE_NAME + 2);
+ }
+
+ /**
+ * @param client Client.
+ * @param cacheName Cache name.
+ */
+ private void checkFineCache(Ignite client, String cacheName) {
+ IgniteCache<Integer, String> cache = client.getOrCreateCache(cacheName);
+
+ cache.put(1, "1");
+
+ assertEquals("1", cache.get(1));
+ }
+
+ /**
+ * @param client Client.
+ */
+ @SuppressWarnings("ThrowableNotThrown")
+ private void checkFailedCache(final Ignite client, final String cacheName) {
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ IgniteCache<Integer, String> cache;
+
+ // Start cache with near enabled.
+ if (NEAR_ATOMIC_CACHE_NAME.equals(cacheName) || NEAR_TX_CACHE_NAME.equals(cacheName)) {
+ CacheConfiguration<Integer, String> ccfg = new CacheConfiguration<Integer, String>(cacheName)
+ .setNearConfiguration(new NearCacheConfiguration<Integer, String>());
+
+ if (NEAR_TX_CACHE_NAME.equals(cacheName))
+ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+ cache = client.getOrCreateCache(ccfg);
+ }
+ else
+ cache = client.cache(cacheName);
+
+
+ cache.put(1, "1");
+
+ assertEquals("1", cache.get(1));
+
+ return null;
+ }
+ }, CacheException.class, null);
+ }
+
+ /**
+ * To fail on cache start.
+ */
+ private static class FailedIndexing implements GridQueryIndexing {
+ /** {@inheritDoc} */
+ @Override public void start(GridKernalContext ctx, GridSpinBusyLock busyLock) throws IgniteCheckedException {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() throws IgniteCheckedException {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> QueryCursor<Cache.Entry<K, V>> queryDistributedSql(String schemaName, SqlQuery qry,
+ boolean keepBinary, int mainCacheId) throws IgniteCheckedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FieldsQueryCursor<List<?>> queryDistributedSqlFields(String schemaName, SqlFieldsQuery qry,
+ boolean keepBinary, GridQueryCancel cancel, @Nullable Integer mainCacheId) throws IgniteCheckedException {
+ return null;
+ }
+
+
+ /** {@inheritDoc} */
+ @Override public long streamUpdateQuery(String spaceName, String qry, @Nullable Object[] params,
+ IgniteDataStreamer<?, ?> streamer) throws IgniteCheckedException {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> QueryCursor<Cache.Entry<K, V>> queryLocalSql(String schemaName, SqlQuery qry,
+ IndexingQueryFilter filter, boolean keepBinary) throws IgniteCheckedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FieldsQueryCursor<List<?>> queryLocalSqlFields(String schemaName, SqlFieldsQuery qry,
+ boolean keepBinary, IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalText(String spaceName, String qry,
+ String typeName, IndexingQueryFilter filter) throws IgniteCheckedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void dynamicIndexCreate(String spaceName, String tblName, QueryIndexDescriptorImpl idxDesc,
+ boolean ifNotExists, SchemaIndexCacheVisitor cacheVisitor) throws IgniteCheckedException {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public void dynamicIndexDrop(String spaceName, String idxName,
+ boolean ifExists) throws IgniteCheckedException {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public void registerCache(String cacheName, String schemaName,
+ GridCacheContext<?, ?> cctx) throws IgniteCheckedException {
+ if (FAILED_CACHES.contains(cctx.name()) && cctx.kernalContext().clientNode())
+ throw new IgniteCheckedException("Test query exception " + cctx.name() + " " + new Random().nextInt());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void unregisterCache(String spaceName) throws IgniteCheckedException {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean registerType(GridCacheContext cctx,
+ GridQueryTypeDescriptor desc) throws IgniteCheckedException {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void store(String cacheName, GridQueryTypeDescriptor type, KeyCacheObject key, int partId,
+ CacheObject val, GridCacheVersion ver, long expirationTime, long link) throws IgniteCheckedException {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public void remove(String spaceName, GridQueryTypeDescriptor type, KeyCacheObject key, int partId,
+ CacheObject val, GridCacheVersion ver) throws IgniteCheckedException {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public void rebuildIndexesFromHash(GridCacheContext cctx, String schemaName,
+ String typeName) throws IgniteCheckedException {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public void markForRebuildFromHash(String cacheName, String typeName) {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public IndexingQueryFilter backupFilter(AffinityTopologyVersion topVer, int[] parts) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onDisconnected(IgniteFuture<?> reconnectFut) {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public PreparedStatement prepareNativeStatement(String space, String sql) throws SQLException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<GridRunningQueryInfo> runningQueries(long duration) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancelQueries(Collection<Long> queries) {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancelAllQueries() {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public String schema(String cacheName) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isInsertStatement(PreparedStatement nativeStmt) {
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/abb9d942/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 6370a10..23f8445 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -71,6 +71,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheTxNearPeekModesTes
import org.apache.ignite.internal.processors.cache.IgniteCacheTxPeekModesTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheTxPreloadNoWriteTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheTxReplicatedPeekModesTest;
+import org.apache.ignite.internal.processors.cache.IgniteClientCacheInitializationFailTest;
import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheFilterTest;
import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartNoExchangeTimeoutTest;
import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartSelfTest;
@@ -227,6 +228,7 @@ public class IgniteCacheTestSuite4 extends TestSuite {
suite.addTestSuite(CacheStartOnJoinTest.class);
suite.addTestSuite(IgniteCacheStartTest.class);
suite.addTestSuite(CacheDiscoveryDataConcurrentJoinTest.class);
+ suite.addTestSuite(IgniteClientCacheInitializationFailTest.class);
suite.addTestSuite(GridCacheTxLoadFromStoreOnLockSelfTest.class);