You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2018/04/23 07:11:57 UTC
[01/50] [abbrv] ignite git commit: IGNITE-8292: Broken yardstick
compilation.
Repository: ignite
Updated Branches:
refs/heads/ignite-6083 ecefdd335 -> bea574b93
IGNITE-8292: Broken yardstick compilation.
this closes #3838
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e76fcb4a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e76fcb4a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e76fcb4a
Branch: refs/heads/ignite-6083
Commit: e76fcb4a538aece58498f43d7e82b2054ca96c51
Parents: 5614621
Author: YuriBabak <y....@gmail.com>
Authored: Tue Apr 17 11:22:14 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Tue Apr 17 11:22:14 2018 +0300
----------------------------------------------------------------------
...uzzyCMeansDistributedClustererBenchmark.java | 130 -------------------
...gniteFuzzyCMeansLocalClustererBenchmark.java | 93 -------------
2 files changed, 223 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e76fcb4a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java
deleted file mode 100644
index e356746..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.clustering;
-
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.clustering.BaseFuzzyCMeansClusterer;
-import org.apache.ignite.ml.clustering.FuzzyCMeansDistributedClusterer;
-import org.apache.ignite.ml.clustering.FuzzyCMeansModel;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-import org.apache.ignite.yardstick.ml.DataChanger;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteFuzzyCMeansDistributedClustererBenchmark extends IgniteAbstractBenchmark {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public boolean test(Map<Object, Object> ctx) throws Exception {
- // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
- // because we create ignite cache internally.
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- this.getClass().getSimpleName(), new Runnable() {
- /** {@inheritDoc} */
- @Override public void run() {
- // IMPL NOTE originally taken from FuzzyCMeansExample.
- // Distance measure that computes distance between two points.
- DistanceMeasure distanceMeasure = new EuclideanDistance();
-
- // "Fuzziness" - specific constant that is used in membership calculation (1.0+-eps ~ K-Means).
- double exponentialWeight = 2.0;
-
- // Condition that indicated when algorithm must stop.
- // In this example algorithm stops if memberships have changed insignificantly.
- BaseFuzzyCMeansClusterer.StopCondition stopCond =
- BaseFuzzyCMeansClusterer.StopCondition.STABLE_MEMBERSHIPS;
-
- // Maximum difference between new and old membership values with which algorithm will continue to work.
- double maxDelta = 0.01;
-
- // The maximum number of FCM iterations.
- int maxIterations = 50;
-
- // Number of steps of primary centers selection (more steps more candidates).
- int initializationSteps = 2;
-
- // Number of K-Means iteration that is used to choose required number of primary centers from candidates.
- int kMeansMaxIterations = 50;
-
- // Create new distributed clusterer with parameters described above.
- FuzzyCMeansDistributedClusterer clusterer = new FuzzyCMeansDistributedClusterer(
- distanceMeasure, exponentialWeight, stopCond, maxDelta, maxIterations,
- null, initializationSteps, kMeansMaxIterations);
-
- // Create sample data.
- double[][] points = shuffle((int)(DataChanger.next()));
-
- // Initialize matrix of data points. Each row contains one point.
- int rows = points.length;
- int cols = points[0].length;
-
- // Create the matrix that contains sample points.
- SparseDistributedMatrix pntMatrix = new SparseDistributedMatrix(rows, cols,
- StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- // Store points into matrix.
- pntMatrix.assign(points);
-
- // Call clusterization method with some number of centers.
- // It returns model that can predict results for new points.
- int numCenters = 4;
- FuzzyCMeansModel mdl = clusterer.cluster(pntMatrix, numCenters);
-
- // Get centers of clusters that is computed by Fuzzy C-Means algorithm.
- mdl.centers();
-
- pntMatrix.destroy();
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
-
- return true;
- }
-
- /** */
- private double[][] shuffle(int off) {
- final double[][] points = new double[][] {
- {-10, -10}, {-9, -11}, {-10, -9}, {-11, -9},
- {10, 10}, {9, 11}, {10, 9}, {11, 9},
- {-10, 10}, {-9, 11}, {-10, 9}, {-11, 9},
- {10, -10}, {9, -11}, {10, -9}, {11, -9}};
-
- final int size = points.length;
-
- final double[][] res = new double[size][];
-
- for (int i = 0; i < size; i++)
- res[i] = points[(i + off) % size];
-
- return res;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e76fcb4a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java
deleted file mode 100644
index 8c4c9ce..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.clustering;
-
-import java.util.Map;
-import org.apache.ignite.ml.clustering.BaseFuzzyCMeansClusterer;
-import org.apache.ignite.ml.clustering.FuzzyCMeansLocalClusterer;
-import org.apache.ignite.ml.clustering.FuzzyCMeansModel;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-import org.apache.ignite.yardstick.ml.DataChanger;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteFuzzyCMeansLocalClustererBenchmark extends IgniteAbstractBenchmark {
- /** {@inheritDoc} */
- @Override public boolean test(Map<Object, Object> ctx) throws Exception {
- // IMPL NOTE originally taken from FuzzyLocalCMeansExample.
- // Distance measure that computes distance between two points.
- DistanceMeasure distanceMeasure = new EuclideanDistance();
-
- // "Fuzziness" - specific constant that is used in membership calculation (1.0+-eps ~ K-Means).
- double exponentialWeight = 2.0;
-
- // Condition that indicated when algorithm must stop.
- // In this example algorithm stops if memberships have changed insignificantly.
- BaseFuzzyCMeansClusterer.StopCondition stopCond =
- BaseFuzzyCMeansClusterer.StopCondition.STABLE_MEMBERSHIPS;
-
- // Maximum difference between new and old membership values with which algorithm will continue to work.
- double maxDelta = 0.01;
-
- // The maximum number of FCM iterations.
- int maxIterations = 50;
-
- // Create new local clusterer with parameters described above.
- FuzzyCMeansLocalClusterer clusterer = new FuzzyCMeansLocalClusterer(distanceMeasure,
- exponentialWeight, stopCond, maxDelta, maxIterations, null);
-
- // Create sample data.
- double[][] points = shuffle((int)(DataChanger.next()));
-
- // Create the matrix that contains sample points.
- DenseLocalOnHeapMatrix pntMatrix = new DenseLocalOnHeapMatrix(points);
-
- // Call clusterization method with some number of centers.
- // It returns model that can predict results for new points.
- int numCenters = 4;
- FuzzyCMeansModel mdl = clusterer.cluster(pntMatrix, numCenters);
-
- // Get centers of clusters that is computed by Fuzzy C-Means algorithm.
- mdl.centers();
-
- return true;
- }
-
- /** */
- private double[][] shuffle(int off) {
- final double[][] points = new double[][] {
- {-10, -10}, {-9, -11}, {-10, -9}, {-11, -9},
- {10, 10}, {9, 11}, {10, 9}, {11, 9},
- {-10, 10}, {-9, 11}, {-10, 9}, {-11, 9},
- {10, -10}, {9, -11}, {10, -9}, {11, -9}};
-
- final int size = points.length;
-
- final double[][] res = new double[size][];
-
- for (int i = 0; i < size; i++)
- res[i] = points[(i + off) % size];
-
- return res;
- }
-}
[37/50] [abbrv] ignite git commit: IGNITE-8325 Compressor thread may
miss notification on stop - Fixes #3877.
Posted by ag...@apache.org.
IGNITE-8325 Compressor thread may miss notification on stop - Fixes #3877.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/80086011
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/80086011
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/80086011
Branch: refs/heads/ignite-6083
Commit: 80086011e9e9105a91ae837268866d4528954e7d
Parents: 631d3a6
Author: Ivan Rakov <iv...@gmail.com>
Authored: Thu Apr 19 19:04:08 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Apr 19 19:04:08 2018 +0300
----------------------------------------------------------------------
.../cache/persistence/wal/FileWriteAheadLogManager.java | 3 +++
.../cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java | 3 +++
2 files changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/80086011/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index a40811b..9917e85 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -1832,6 +1832,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
long segmentToCompress = lastCompressedIdx + 1;
synchronized (this) {
+ if (stopped)
+ return -1;
+
while (segmentToCompress > Math.min(lastAllowedToCompressIdx, archivedMonitor.lastArchivedAbsoluteIndex())) {
wait();
http://git-wip-us.apache.org/repos/asf/ignite/blob/80086011/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index c7d2c11..2305389 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -1666,6 +1666,9 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
long segmentToCompress = lastCompressedIdx + 1;
synchronized (this) {
+ if (stopped)
+ return -1;
+
while (segmentToCompress > Math.min(lastAllowedToCompressIdx, archiver.lastArchivedAbsoluteIndex())) {
wait();
[18/50] [abbrv] ignite git commit: IGNITE-8201 Fixed NPE
Posted by ag...@apache.org.
IGNITE-8201 Fixed NPE
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ced5d276
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ced5d276
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ced5d276
Branch: refs/heads/ignite-6083
Commit: ced5d2768d47504533064000c8ce525d80613ca3
Parents: 69606e4
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Wed Apr 18 10:59:46 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Apr 18 11:00:15 2018 +0300
----------------------------------------------------------------------
.../processors/rest/JettyRestProcessorAbstractSelfTest.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ced5d276/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
index e36447b..0285f3a 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
@@ -985,9 +985,11 @@ public abstract class JettyRestProcessorAbstractSelfTest extends AbstractRestPro
if (dsCfg.getDefaultDataRegionConfiguration().isPersistenceEnabled())
fail("IGNITE-5874");
- for (DataRegionConfiguration dataRegCfg : dsCfg.getDataRegionConfigurations()) {
- if (dataRegCfg.isPersistenceEnabled())
- fail("IGNITE-5874");
+ if (!F.isEmpty(dsCfg.getDataRegionConfigurations())) {
+ for (DataRegionConfiguration dataRegCfg : dsCfg.getDataRegionConfigurations()) {
+ if (dataRegCfg.isPersistenceEnabled())
+ fail("IGNITE-5874");
+ }
}
}
[05/50] [abbrv] ignite git commit: IGNITE-8210 Fixed custom event
handling for baseline topology change - Fixes #3814.
Posted by ag...@apache.org.
IGNITE-8210 Fixed custom event handling for baseline topology change - Fixes #3814.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d79c6409
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d79c6409
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d79c6409
Branch: refs/heads/ignite-6083
Commit: d79c6409bcb8ca3170ce9153db486cac2c537fc4
Parents: 7731669
Author: Sergey Chugunov <se...@gmail.com>
Authored: Tue Apr 17 14:28:47 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Apr 17 14:28:47 2018 +0300
----------------------------------------------------------------------
.../affinity/GridAffinityAssignmentCache.java | 2 +-
.../distributed/CacheBaselineTopologyTest.java | 94 ++++++++++++++++++++
2 files changed, 95 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d79c6409/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
index b1899e3..9d5ce05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
@@ -322,7 +322,7 @@ public class GridAffinityAssignmentCache {
for (DiscoveryEvent event : events.events()) {
boolean affinityNode = CU.affinityNode(event.eventNode(), nodeFilter);
- if (affinityNode) {
+ if (affinityNode || event.type() == EVT_DISCOVERY_CUSTOM_EVT) {
skipCalculation = false;
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/d79c6409/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index 26502ed..0d59a2d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -32,6 +32,7 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.cache.affinity.AffinityFunction;
import org.apache.ignite.cache.affinity.AffinityFunctionContext;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
@@ -54,6 +55,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.Gri
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -81,6 +83,12 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
private boolean delayRebalance;
/** */
+ private Map<String, Object> userAttrs;
+
+ /** */
+ private static final String DATA_NODE = "dataNodeUserAttr";
+
+ /** */
private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
/** {@inheritDoc} */
@@ -129,6 +137,9 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
.setWalMode(WALMode.LOG_ONLY)
);
+ if (userAttrs != null)
+ cfg.setUserAttributes(userAttrs);
+
if (client)
cfg.setClientMode(true);
@@ -139,6 +150,89 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
}
/**
+ * Verifies that rebalance on cache with Node Filter happens when BaselineTopology changes.
+ *
+ * @throws Exception
+ */
+ public void testRebalanceForCacheWithNodeFilter() throws Exception {
+ try {
+ final int EMPTY_NODE_IDX = 2;
+
+ userAttrs = U.newHashMap(1);
+ userAttrs.put(DATA_NODE, true);
+
+ startGrids(2);
+
+ userAttrs.put(DATA_NODE, false);
+
+ IgniteEx ignite = startGrid(2);
+
+ ignite.cluster().active(true);
+
+ awaitPartitionMapExchange();
+
+ IgniteCache<Integer, Integer> cache =
+ ignite.createCache(
+ new CacheConfiguration<Integer, Integer>()
+ .setName(CACHE_NAME)
+ .setCacheMode(PARTITIONED)
+ .setBackups(1)
+ .setPartitionLossPolicy(READ_ONLY_SAFE)
+ .setAffinity(new RendezvousAffinityFunction(32, null))
+ .setNodeFilter(new DataNodeFilter())
+ );
+
+ for (int k = 0; k < 10_000; k++)
+ cache.put(k, k);
+
+ Thread.sleep(500);
+
+ printSizesDataNodes(NODE_COUNT - 1, EMPTY_NODE_IDX);
+
+ userAttrs.put(DATA_NODE, true);
+
+ startGrid(3);
+
+ ignite.cluster().setBaselineTopology(ignite.cluster().topologyVersion());
+
+ awaitPartitionMapExchange();
+
+ Thread.sleep(500);
+
+ printSizesDataNodes(NODE_COUNT, EMPTY_NODE_IDX);
+ }
+ finally {
+ userAttrs = null;
+ }
+ }
+
+ /** */
+ private void printSizesDataNodes(int nodesCnt, int emptyNodeIdx) {
+ for (int i = 0; i < nodesCnt; i++) {
+ IgniteEx ig = grid(i);
+
+ int locSize = ig.cache(CACHE_NAME).localSize(CachePeekMode.PRIMARY);
+
+ if (i == emptyNodeIdx)
+ assertEquals("Cache local size on "
+ + i
+ + " node is expected to be zero", 0, locSize);
+ else
+ assertTrue("Cache local size on "
+ + i
+ + " node is expected to be non zero", locSize > 0);
+ }
+ }
+
+ /** */
+ private static class DataNodeFilter implements IgnitePredicate<ClusterNode> {
+
+ @Override public boolean apply(ClusterNode clusterNode) {
+ return clusterNode.attribute(DATA_NODE);
+ }
+ }
+
+ /**
* @throws Exception If failed.
*/
public void testTopologyChangesWithFixedBaseline() throws Exception {
[21/50] [abbrv] ignite git commit: IGNITE-8122 Restore partition
state from WAL if no checkpoints are done - Fixes #3745.
Posted by ag...@apache.org.
IGNITE-8122 Restore partition state from WAL if no checkpoints are done - Fixes #3745.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/14c032ae
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/14c032ae
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/14c032ae
Branch: refs/heads/ignite-6083
Commit: 14c032ae293b1c85f9b66f9f7a77e0514d04e153
Parents: a050436
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Wed Apr 18 15:57:45 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Apr 18 15:57:45 2018 +0300
----------------------------------------------------------------------
.../affinity/GridAffinityAssignment.java | 4 +-
.../processors/cache/ExchangeActions.java | 2 +-
.../dht/GridDhtPartitionTopology.java | 3 +-
.../dht/GridDhtPartitionTopologyImpl.java | 220 +++++++++----------
.../dht/GridDhtPartitionsStateValidator.java | 61 ++++-
.../GridDhtPartitionsExchangeFuture.java | 6 +-
.../GridDhtPartitionsSingleMessage.java | 18 +-
.../dht/preloader/GridDhtPreloader.java | 4 +-
.../GridCacheDatabaseSharedManager.java | 89 +++-----
.../distributed/CacheBaselineTopologyTest.java | 16 +-
...CacheLoadingConcurrentGridStartSelfTest.java | 3 +
.../GridCachePartitionsStateValidationTest.java | 4 +
...idCachePartitionsStateValidatorSelfTest.java | 47 ++--
.../testsuites/IgniteStandByClusterSuite.java | 2 +-
.../IgniteCacheQueryNodeRestartSelfTest.java | 2 +
...ngBaselineCacheQueryNodeRestartSelfTest.java | 4 +-
16 files changed, 258 insertions(+), 227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
index 6da6aaa..cbec1a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
@@ -149,7 +149,7 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable
* @param part Partition.
* @return Affinity nodes.
*/
- public List<ClusterNode> get(int part) {
+ @Override public List<ClusterNode> get(int part) {
assert part >= 0 && part < assignment.size() : "Affinity partition is out of range" +
" [part=" + part + ", partitions=" + assignment.size() + ']';
@@ -162,7 +162,7 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable
* @param part Partition.
* @return Affinity nodes IDs.
*/
- public HashSet<UUID> getIds(int part) {
+ @Override public HashSet<UUID> getIds(int part) {
assert part >= 0 && part < assignment.size() : "Affinity partition is out of range" +
" [part=" + part + ", partitions=" + assignment.size() + ']';
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
index bcf3f40..c289b6e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
@@ -409,7 +409,7 @@ public class ExchangeActions {
/**
*
*/
- static class CacheGroupActionData {
+ public static class CacheGroupActionData {
/** */
private final CacheGroupDescriptor desc;
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index 6f68dbb..d586a94d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -386,8 +386,9 @@ public interface GridDhtPartitionTopology {
* State of all current owners that aren't contained in the set will be reset to MOVING.
*
* @param p Partition ID.
- * @param updateSeq If should increment sequence when updated.
* @param owners Set of new owners.
+ * @param haveHistory {@code True} if there is WAL history to rebalance given partition.
+ * @param updateSeq If should increment sequence when updated.
* @return Set of node IDs that should reload partitions.
*/
public Set<UUID> setOwners(int p, Set<UUID> owners, boolean haveHistory, boolean updateSeq);
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 740903e..164f0bf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -17,7 +17,6 @@
package org.apache.ignite.internal.processors.cache.distributed.dht;
-import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -31,8 +30,6 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReferenceArray;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
@@ -43,7 +40,6 @@ import org.apache.ignite.events.EventType;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.managers.discovery.DiscoCache;
-import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
@@ -55,8 +51,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.Gri
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.util.F0;
import org.apache.ignite.internal.util.GridAtomicLong;
import org.apache.ignite.internal.util.GridPartitionStateMap;
@@ -324,7 +318,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
long updateSeq = this.updateSeq.incrementAndGet();
- needRefresh = initPartitions0(affVer, exchFut, updateSeq);
+ needRefresh = initPartitions(affVer, grp.affinity().readyAssignments(affVer), exchFut, updateSeq);
consistencyCheck();
}
@@ -340,14 +334,15 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
/**
+ * Creates and initializes partitions using given {@code affVer} and {@code affAssignment}.
+ *
* @param affVer Affinity version to use.
+ * @param affAssignment Affinity assignment to use.
* @param exchFut Exchange future.
* @param updateSeq Update sequence.
* @return {@code True} if partitions must be refreshed.
*/
- private boolean initPartitions0(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
- List<List<ClusterNode>> aff = grp.affinity().readyAssignments(affVer);
-
+ private boolean initPartitions(AffinityTopologyVersion affVer, List<List<ClusterNode>> affAssignment, GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
boolean needRefresh = false;
if (grp.affinityNode()) {
@@ -357,32 +352,24 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
GridDhtPartitionExchangeId exchId = exchFut.exchangeId();
- assert grp.affinity().lastVersion().equals(affVer) :
- "Invalid affinity [topVer=" + grp.affinity().lastVersion() +
- ", grp=" + grp.cacheOrGroupName() +
- ", affVer=" + affVer +
- ", fut=" + exchFut + ']';
-
int num = grp.affinity().partitions();
if (grp.rebalanceEnabled()) {
boolean added = exchFut.cacheGroupAddedOnExchange(grp.groupId(), grp.receivedFrom());
- boolean first = added || (loc.equals(oldest) && loc.id().equals(exchId.nodeId()) && exchId.isJoined());
+ boolean first = added || (loc.equals(oldest) && loc.id().equals(exchId.nodeId()) && exchId.isJoined()) || exchFut.activateCluster();
if (first) {
- assert exchId.isJoined() || added;
+ assert exchId.isJoined() || added || exchFut.activateCluster();
for (int p = 0; p < num; p++) {
- if (localNode(p, aff) || initLocalPartition(p, discoCache)) {
- GridDhtLocalPartition locPart = createPartition(p);
+ if (localNode(p, affAssignment)) {
+ // Partition is created first time, so it's safe to own it.
+ boolean shouldOwn = locParts.get(p) == null;
- if (grp.persistenceEnabled()) {
- GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)grp.shared().database();
+ GridDhtLocalPartition locPart = getOrCreatePartition(p);
- locPart.restoreState(db.readPartitionState(grp, locPart.id()));
- }
- else {
+ if (shouldOwn) {
boolean owned = locPart.own();
assert owned : "Failed to own partition for oldest node [grp=" + grp.cacheOrGroupName() +
@@ -390,7 +377,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (log.isDebugEnabled())
log.debug("Owned partition for oldest node [grp=" + grp.cacheOrGroupName() +
- ", part=" + locPart + ']');
+ ", part=" + locPart + ']');
}
needRefresh = true;
@@ -400,15 +387,15 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
}
else
- createPartitions(affVer, aff, updateSeq);
+ createPartitions(affVer, affAssignment, updateSeq);
}
else {
// If preloader is disabled, then we simply clear out
// the partitions this node is not responsible for.
for (int p = 0; p < num; p++) {
- GridDhtLocalPartition locPart = localPartition0(p, affVer, false, true, false);
+ GridDhtLocalPartition locPart = localPartition0(p, affVer, false, true);
- boolean belongs = localNode(p, aff);
+ boolean belongs = localNode(p, affAssignment);
if (locPart != null) {
if (!belongs) {
@@ -429,7 +416,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
locPart.own();
}
else if (belongs) {
- locPart = createPartition(p);
+ locPart = getOrCreatePartition(p);
locPart.own();
@@ -439,27 +426,14 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
}
- updateRebalanceVersion(aff);
+ updateRebalanceVersion(affAssignment);
return needRefresh;
}
/**
- * @param p Partition ID to restore.
- * @param discoCache Disco cache to use.
- * @return {@code True} if should restore local partition.
- */
- private boolean initLocalPartition(int p, DiscoCache discoCache) {
- IgnitePageStoreManager storeMgr = ctx.pageStore();
-
- return
- grp.persistenceEnabled() &&
- storeMgr instanceof FilePageStoreManager &&
- discoCache.baselineNode(ctx.localNodeId()) &&
- Files.exists(((FilePageStoreManager)storeMgr).getPath(grp.sharedGroup(), grp.cacheOrGroupName(), p));
- }
-
- /**
+ * Creates non-existing partitions belong to given affinity {@code aff}.
+ *
* @param affVer Affinity version.
* @param aff Affinity assignments.
* @param updateSeq Update sequence.
@@ -475,7 +449,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (localNode(p, aff)) {
// This will make sure that all non-existing partitions
// will be created in MOVING state.
- GridDhtLocalPartition locPart = createPartition(p);
+ GridDhtLocalPartition locPart = getOrCreatePartition(p);
updateSeq = updateLocal(p, locPart.state(), updateSeq, affVer);
}
@@ -483,7 +457,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
// If this node's map is empty, we pre-create local partitions,
// so local map will be sent correctly during exchange.
else if (localNode(p, aff))
- createPartition(p);
+ getOrCreatePartition(p);
}
}
@@ -588,18 +562,29 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (grpStarted ||
exchFut.firstEvent().type() == EVT_DISCOVERY_CUSTOM_EVT ||
exchFut.serverNodeDiscoveryEvent()) {
+
+ AffinityTopologyVersion affVer;
+ List<List<ClusterNode>> affAssignment;
+
if (affReady) {
- assert grp.affinity().lastVersion().equals(evts.topologyVersion());
+ affVer = evts.topologyVersion();
- initPartitions0(evts.topologyVersion(), exchFut, updateSeq);
+ assert grp.affinity().lastVersion().equals(affVer) :
+ "Invalid affinity [topVer=" + grp.affinity().lastVersion() +
+ ", grp=" + grp.cacheOrGroupName() +
+ ", affVer=" + affVer +
+ ", fut=" + exchFut + ']';
+
+ affAssignment = grp.affinity().readyAssignments(affVer);
}
else {
assert !exchFut.context().mergeExchanges();
- List<List<ClusterNode>> aff = grp.affinity().idealAssignment();
-
- createPartitions(exchFut.initialVersion(), aff, updateSeq);
+ affVer = exchFut.initialVersion();
+ affAssignment = grp.affinity().idealAssignment();
}
+
+ initPartitions(affVer, affAssignment, exchFut, updateSeq);
}
}
@@ -695,19 +680,12 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
long updateSeq = this.updateSeq.incrementAndGet();
for (int p = 0; p < num; p++) {
- GridDhtLocalPartition locPart = localPartition0(p, topVer, false, false, false);
+ GridDhtLocalPartition locPart = localPartition0(p, topVer, false, true);
if (partitionLocalNode(p, topVer)) {
- // This partition will be created during next topology event,
- // which obviously has not happened at this point.
- if (locPart == null) {
- if (log.isDebugEnabled()) {
- log.debug("Skipping local partition afterExchange (will not create) [" +
- "grp=" + grp.cacheOrGroupName() + ", p=" + p + ']');
- }
-
- continue;
- }
+ // Prepare partition to rebalance if it's not happened on full map update phase.
+ if (locPart == null || locPart.state() == RENTING || locPart.state() == EVICTED)
+ locPart = rebalancePartition(p, false);
GridDhtPartitionState state = locPart.state();
@@ -793,20 +771,23 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
@Nullable @Override public GridDhtLocalPartition localPartition(int p, AffinityTopologyVersion topVer,
boolean create)
throws GridDhtInvalidPartitionException {
- return localPartition0(p, topVer, create, false, true);
+ return localPartition0(p, topVer, create, false);
}
/** {@inheritDoc} */
@Nullable @Override public GridDhtLocalPartition localPartition(int p, AffinityTopologyVersion topVer,
boolean create, boolean showRenting) throws GridDhtInvalidPartitionException {
- return localPartition0(p, topVer, create, showRenting, true);
+ return localPartition0(p, topVer, create, showRenting);
}
/**
+ * Creates partition with id {@code p} if it doesn't exist or evicted.
+ * In other case returns existing partition.
+ *
* @param p Partition number.
* @return Partition.
*/
- private GridDhtLocalPartition createPartition(int p) {
+ private GridDhtLocalPartition getOrCreatePartition(int p) {
assert lock.isWriteLockedByCurrentThread();
assert ctx.database().checkpointLockIsHeldByThread();
@@ -865,16 +846,15 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
/**
* @param p Partition number.
* @param topVer Topology version.
- * @param create Create flag.
- * @param updateSeq Update sequence.
+ * @param create If {@code true} create partition if it doesn't exists or evicted.
+ * @param showRenting If {@code true} return partition in RENTING state if exists.
* @return Local partition.
*/
@SuppressWarnings("TooBroadScope")
private GridDhtLocalPartition localPartition0(int p,
AffinityTopologyVersion topVer,
boolean create,
- boolean showRenting,
- boolean updateSeq) {
+ boolean showRenting) {
GridDhtLocalPartition loc;
loc = locParts.get(p);
@@ -1345,6 +1325,8 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (incomeCntrMap != null) {
// update local counters in partitions
for (int i = 0; i < locParts.length(); i++) {
+ cntrMap.updateCounter(i, incomeCntrMap.updateCounter(i));
+
GridDhtLocalPartition part = locParts.get(i);
if (part == null)
@@ -1511,51 +1493,11 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
}
else if (state == MOVING) {
- GridDhtLocalPartition locPart = locParts.get(p);
-
- if (!partsToReload.contains(p)) {
- if (locPart == null || locPart.state() == EVICTED)
- locPart = createPartition(p);
-
- if (locPart.state() == OWNING) {
- locPart.moving();
+ boolean haveHistory = !partsToReload.contains(p);
- changed = true;
- }
- }
- else {
- if (locPart == null || locPart.state() == EVICTED) {
- createPartition(p);
+ rebalancePartition(p, haveHistory);
- changed = true;
- }
- else if (locPart.state() == OWNING || locPart.state() == MOVING) {
- if (locPart.state() == OWNING)
- locPart.moving();
- locPart.clearAsync();
-
- changed = true;
- }
- else if (locPart.state() == RENTING) {
- // Try to prevent partition eviction.
- if (locPart.reserve()) {
- try {
- locPart.moving();
- locPart.clearAsync();
- } finally {
- locPart.release();
- }
- }
- // In other case just recreate it.
- else {
- assert locPart.state() == EVICTED;
-
- createPartition(p);
- }
-
- changed = true;
- }
- }
+ changed = true;
}
}
}
@@ -2113,7 +2055,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
- @Override public Set<UUID> setOwners(int p, Set<UUID> owners, boolean haveHistory, boolean updateSeq) {
+ @Override public Set<UUID> setOwners(int p, Set<UUID> ownersByUpdCounters, boolean haveHistory, boolean updateSeq) {
Set<UUID> result = haveHistory ? Collections.<UUID>emptySet() : new HashSet<UUID>();
ctx.database().checkpointReadLock();
@@ -2125,17 +2067,15 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
GridDhtLocalPartition locPart = locParts.get(p);
if (locPart != null) {
- if (locPart.state() == OWNING && !owners.contains(ctx.localNodeId())) {
- locPart.moving();
+ if (locPart.state() == OWNING && !ownersByUpdCounters.contains(ctx.localNodeId())) {
+ rebalancePartition(p, haveHistory);
- if (!haveHistory) {
- locPart.clearAsync();
+ if (!haveHistory)
result.add(ctx.localNodeId());
- }
U.warn(log, "Partition has been scheduled for rebalancing due to outdated update counter " +
"[nodeId=" + ctx.localNodeId() + ", grp=" + grp.cacheOrGroupName() +
- ", partId=" + locPart.id() + ", haveHistory=" + haveHistory + "]");
+ ", partId=" + p + ", haveHistory=" + haveHistory + "]");
}
}
@@ -2146,7 +2086,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (!partMap.containsKey(p))
continue;
- if (partMap.get(p) == OWNING && !owners.contains(remoteNodeId)) {
+ if (partMap.get(p) == OWNING && !ownersByUpdCounters.contains(remoteNodeId)) {
partMap.put(p, MOVING);
if (!haveHistory)
@@ -2180,6 +2120,42 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
/**
+ * Prepares given partition {@code p} for rebalance.
+ * Changes partition state to MOVING and starts clearing if needed.
+ * Prevents ongoing renting if required.
+ *
+ * @param p Partition id.
+ * @param haveHistory If {@code true} there is WAL history to rebalance partition,
+ * in other case partition will be cleared for full rebalance.
+ */
+ private GridDhtLocalPartition rebalancePartition(int p, boolean haveHistory) {
+ GridDhtLocalPartition part = getOrCreatePartition(p);
+
+ // Prevent renting.
+ if (part.state() == RENTING) {
+ if (part.reserve()) {
+ part.moving();
+ part.release();
+ }
+ else {
+ assert part.state() == EVICTED : part;
+
+ part = getOrCreatePartition(p);
+ }
+ }
+
+ if (part.state() != MOVING)
+ part.moving();
+
+ if (!haveHistory)
+ part.clearAsync();
+
+ assert part.state() == MOVING : part;
+
+ return part;
+ }
+
+ /**
* Finds local partitions which don't belong to affinity and runs eviction process for such partitions.
*
* @param updateSeq Update sequence.
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
index 92a0584..cc0542c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
@@ -35,6 +35,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.Gri
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.lang.IgniteProductVersion;
+import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
@@ -71,9 +72,9 @@ public class GridDhtPartitionsStateValidator {
public void validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture fut,
GridDhtPartitionTopology top,
Map<UUID, GridDhtPartitionsSingleMessage> messages) throws IgniteCheckedException {
- // Ignore just joined nodes.
final Set<UUID> ignoringNodes = new HashSet<>();
+ // Ignore just joined nodes.
for (DiscoveryEvent evt : fut.events().events())
if (evt.type() == EVT_NODE_JOINED)
ignoringNodes.add(evt.eventNode().id());
@@ -99,6 +100,46 @@ public class GridDhtPartitionsStateValidator {
}
/**
+ * Checks what partitions from given {@code singleMsg} message should be excluded from validation.
+ *
+ * @param top Topology to validate.
+ * @param nodeId Node which sent single message.
+ * @param singleMsg Single message.
+ * @return Set of partition ids should be excluded from validation.
+ */
+ @Nullable private Set<Integer> shouldIgnore(GridDhtPartitionTopology top, UUID nodeId, GridDhtPartitionsSingleMessage singleMsg) {
+ CachePartitionPartialCountersMap countersMap = singleMsg.partitionUpdateCounters(top.groupId(), top.partitions());
+ Map<Integer, Long> sizesMap = singleMsg.partitionSizes(top.groupId());
+
+ Set<Integer> ignore = null;
+
+ for (int p = 0; p < top.partitions(); p++) {
+ if (top.partitionState(nodeId, p) != GridDhtPartitionState.OWNING) {
+ if (ignore == null)
+ ignore = new HashSet<>();
+
+ ignore.add(p);
+
+ continue;
+ }
+
+ int partIdx = countersMap.partitionIndex(p);
+ long updateCounter = partIdx >= 0 ? countersMap.updateCounterAt(partIdx) : 0;
+ long size = sizesMap.containsKey(p) ? sizesMap.get(p) : 0;
+
+ // Do not validate partitions with zero update counter and size.
+ if (updateCounter == 0 && size == 0) {
+ if (ignore == null)
+ ignore = new HashSet<>();
+
+ ignore.add(p);
+ }
+ }
+
+ return ignore;
+ }
+
+ /**
* Validate partitions update counters for given {@code top}.
*
* @param top Topology to validate.
@@ -117,7 +158,10 @@ public class GridDhtPartitionsStateValidator {
// Populate counters statistics from local node partitions.
for (GridDhtLocalPartition part : top.currentLocalPartitions()) {
- if (top.partitionState(cctx.localNodeId(), part.id()) != GridDhtPartitionState.OWNING)
+ if (part.state() != GridDhtPartitionState.OWNING)
+ continue;
+
+ if (part.updateCounter() == 0 && part.fullSize() == 0)
continue;
updateCountersAndNodesByPartitions.put(part.id(), new T2<>(cctx.localNodeId(), part.updateCounter()));
@@ -133,8 +177,10 @@ public class GridDhtPartitionsStateValidator {
CachePartitionPartialCountersMap countersMap = e.getValue().partitionUpdateCounters(top.groupId(), partitions);
+ Set<Integer> ignorePartitions = shouldIgnore(top, nodeId, e.getValue());
+
for (int part = 0; part < partitions; part++) {
- if (top.partitionState(nodeId, part) != GridDhtPartitionState.OWNING)
+ if (ignorePartitions != null && ignorePartitions.contains(part))
continue;
int partIdx = countersMap.partitionIndex(part);
@@ -166,7 +212,10 @@ public class GridDhtPartitionsStateValidator {
// Populate sizes statistics from local node partitions.
for (GridDhtLocalPartition part : top.currentLocalPartitions()) {
- if (top.partitionState(cctx.localNodeId(), part.id()) != GridDhtPartitionState.OWNING)
+ if (part.state() != GridDhtPartitionState.OWNING)
+ continue;
+
+ if (part.updateCounter() == 0 && part.fullSize() == 0)
continue;
sizesAndNodesByPartitions.put(part.id(), new T2<>(cctx.localNodeId(), part.fullSize()));
@@ -182,8 +231,10 @@ public class GridDhtPartitionsStateValidator {
Map<Integer, Long> sizesMap = e.getValue().partitionSizes(top.groupId());
+ Set<Integer> ignorePartitions = shouldIgnore(top, nodeId, e.getValue());
+
for (int part = 0; part < partitions; part++) {
- if (top.partitionState(nodeId, part) != GridDhtPartitionState.OWNING)
+ if (ignorePartitions != null && ignorePartitions.contains(part))
continue;
long currentSize = sizesMap.containsKey(part) ? sizesMap.get(part) : 0L;
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index af5acd6..0d57d48 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -58,6 +58,7 @@ import org.apache.ignite.internal.managers.communication.GridIoPolicy;
import org.apache.ignite.internal.managers.discovery.DiscoCache;
import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
import org.apache.ignite.internal.pagemem.wal.record.ExchangeRecord;
+import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
@@ -2244,6 +2245,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
/**
+ * Collects and determines new owners of partitions for all nodes for given {@code top}.
+ *
* @param top Topology to assign.
*/
private void assignPartitionStates(GridDhtPartitionTopology top) {
@@ -2585,6 +2588,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
detectLostPartitions(resTopVer);
}
+ // Recalculate new affinity based on partitions availability.
if (!exchCtx.mergeExchanges() && forceAffReassignment)
idealAffDiff = cctx.affinity().onCustomEventWithEnforcedAffinityReassignment(this);
@@ -2768,7 +2772,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
validator.validatePartitionCountersAndSizes(this, top, msgs);
}
catch (IgniteCheckedException ex) {
- log.warning("Partition states validation was failed for cache " + grpDesc.cacheOrGroupName(), ex);
+ log.warning("Partition states validation has failed for group: " + grpDesc.cacheOrGroupName() + ". " + ex.getMessage());
// TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index 6ebafac..b60070e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -73,7 +73,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
private Map<Integer, Map<Integer, Long>> partSizes;
/** Serialized partitions counters. */
- private byte[] partSizesBytes;
+ private byte[] partsSizesBytes;
/** Partitions history reservation counters. */
@GridToStringInclude
@@ -324,7 +324,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
boolean marshal = (parts != null && partsBytes == null) ||
(partCntrs != null && partCntrsBytes == null) ||
(partHistCntrs != null && partHistCntrsBytes == null) ||
- (partSizes != null && partSizesBytes == null) ||
+ (partSizes != null && partsSizesBytes == null) ||
(err != null && errBytes == null);
if (marshal) {
@@ -343,7 +343,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
if (partHistCntrs != null && partHistCntrsBytes == null)
partHistCntrsBytes0 = U.marshal(ctx, partHistCntrs);
- if (partSizes != null && partSizesBytes == null)
+ if (partSizes != null && partsSizesBytes == null)
partSizesBytes0 = U.marshal(ctx, partSizes);
if (err != null && errBytes == null)
@@ -375,7 +375,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
partsBytes = partsBytes0;
partCntrsBytes = partCntrsBytes0;
partHistCntrsBytes = partHistCntrsBytes0;
- partSizesBytes = partSizesBytes0;
+ partsSizesBytes = partSizesBytes0;
errBytes = errBytes0;
}
}
@@ -405,11 +405,11 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
partHistCntrs = U.unmarshal(ctx, partHistCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
}
- if (partSizesBytes != null && partSizes == null) {
+ if (partsSizesBytes != null && partSizes == null) {
if (compressed())
- partSizes = U.unmarshalZip(ctx.marshaller(), partSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+ partSizes = U.unmarshalZip(ctx.marshaller(), partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
else
- partSizes = U.unmarshal(ctx, partSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+ partSizes = U.unmarshal(ctx, partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
}
if (errBytes != null && err == null) {
@@ -504,7 +504,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
writer.incrementState();
case 13:
- if (!writer.writeByteArray("partsSizesBytes", partSizesBytes))
+ if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes))
return false;
writer.incrementState();
@@ -589,7 +589,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
reader.incrementState();
case 13:
- partSizesBytes = reader.readByteArray("partsSizesBytes");
+ partsSizesBytes = reader.readByteArray("partsSizesBytes");
if (!reader.isLastRead())
return false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 6ec6ad3..ddcb81e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -202,7 +202,7 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter {
// If partition belongs to local node.
if (aff.get(p).contains(ctx.localNode())) {
- GridDhtLocalPartition part = top.localPartition(p, topVer, true, true);
+ GridDhtLocalPartition part = top.localPartition(p);
assert part != null;
assert part.id() == p;
@@ -226,7 +226,7 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter {
part = top.localPartition(p, topVer, true);
}
- assert part != null && part.state() == MOVING : "Partition has invalid state for rebalance " + part;
+ assert part.state() == MOVING : "Partition has invalid state for rebalance " + aff.topologyVersion() + " " + part;
ClusterNode histSupplier = null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index a371aa8..8f81708 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -60,6 +60,7 @@ import java.util.concurrent.atomic.LongAdder;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import javax.management.ObjectName;
import org.apache.ignite.DataStorageMetrics;
import org.apache.ignite.IgniteCheckedException;
@@ -1190,9 +1191,18 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
boolean clusterInTransitionStateToActive = fut.activateCluster();
- // Before local node join event.
- if (clusterInTransitionStateToActive || (joinEvt && locNode && isSrvNode))
+ // In case of cluster activation or local join restore, restore whole manager state.
+ if (clusterInTransitionStateToActive || (joinEvt && locNode && isSrvNode)) {
restoreState();
+ }
+ // In case of starting groups, restore partition states only for these groups.
+ else if (fut.exchangeActions() != null && !F.isEmpty(fut.exchangeActions().cacheGroupsToStart())) {
+ Set<Integer> restoreGroups = fut.exchangeActions().cacheGroupsToStart().stream()
+ .map(actionData -> actionData.descriptor().groupId())
+ .collect(Collectors.toSet());
+
+ restorePartitionStates(Collections.emptyMap(), restoreGroups);
+ }
if (cctx.kernalContext().query().moduleEnabled()) {
ExchangeActions acts = fut.exchangeActions();
@@ -1432,6 +1442,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
+ * Restores from last checkpoint and applies WAL changes since this checkpoint.
+ *
* @throws IgniteCheckedException If failed to restore database status from WAL.
*/
private void restoreState() throws IgniteCheckedException {
@@ -2193,7 +2205,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
checkpointReadLock();
try {
- restorePartitionState(partStates, Collections.emptySet());
+ restorePartitionStates(partStates, null);
}
finally {
checkpointReadUnlock();
@@ -2303,7 +2315,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
if (!metastoreOnly)
- restorePartitionState(partStates, ignoreGrps);
+ restorePartitionStates(partStates, null);
}
finally {
if (!metastoreOnly)
@@ -2316,15 +2328,17 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
- * @param partStates Partition states.
- * @throws IgniteCheckedException If failed to restore.
+ * Initializes not empty partitions and restores their state from page memory or WAL.
+ * Partition states presented in page memory may be overriden by states restored from WAL {@code partStates}.
+ *
+ * @param partStates Partition states restored from WAL.
+ * @param onlyForGroups If not {@code null} restore states only for specified cache groups.
+ * @throws IgniteCheckedException If failed to restore partition states.
*/
- private void restorePartitionState(
- Map<T2<Integer, Integer>, T2<Integer, Long>> partStates,
- Collection<Integer> ignoreGrps
- ) throws IgniteCheckedException {
+ private void restorePartitionStates(Map<T2<Integer, Integer>, T2<Integer, Long>> partStates,
+ @Nullable Set<Integer> onlyForGroups) throws IgniteCheckedException {
for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
- if (grp.isLocal() || !grp.affinityNode() || ignoreGrps.contains(grp.groupId())) {
+ if (grp.isLocal() || !grp.affinityNode()) {
// Local cache has no partitions and its states.
continue;
}
@@ -2332,6 +2346,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
if (!grp.dataRegion().config().isPersistenceEnabled())
continue;
+ if (onlyForGroups != null && !onlyForGroups.contains(grp.groupId()))
+ continue;
+
int grpId = grp.groupId();
PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
@@ -2415,56 +2432,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
- * @param grpCtx Group context.
- * @param partId Partition ID.
- * @return Partition state.
- */
- public GridDhtPartitionState readPartitionState(CacheGroupContext grpCtx, int partId) {
- int grpId = grpCtx.groupId();
- PageMemoryEx pageMem = (PageMemoryEx)grpCtx.dataRegion().pageMemory();
-
- try {
- if (storeMgr.exists(grpId, partId)) {
- storeMgr.ensure(grpId, partId);
-
- if (storeMgr.pages(grpId, partId) > 1) {
- long partMetaId = pageMem.partitionMetaPageId(grpId, partId);
- long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
-
- try {
- long pageAddr = pageMem.readLock(grpId, partMetaId, partMetaPage);
-
- try {
- if (PageIO.getType(pageAddr) == PageIO.T_PART_META) {
- PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
-
- GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal((int)io.getPartitionState(pageAddr));
-
- if (state == null)
- state = GridDhtPartitionState.MOVING;
-
- return state;
- }
- }
- finally {
- pageMem.readUnlock(grpId, partMetaId, partMetaPage);
- }
- }
- finally {
- pageMem.releasePage(grpId, partMetaId, partMetaPage);
- }
- }
- }
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to read partition state (will default to MOVING) [grp=" + grpCtx +
- ", partId=" + partId + "]", e);
- }
-
- return GridDhtPartitionState.MOVING;
- }
-
- /**
* Wal truncate callBack.
*
* @param highBound WALPointer.
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index 0d59a2d..c3d404b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -83,6 +83,9 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
private boolean delayRebalance;
/** */
+ private boolean disableAutoActivation;
+
+ /** */
private Map<String, Object> userAttrs;
/** */
@@ -107,6 +110,8 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
cleanPersistenceDir();
client = false;
+
+ disableAutoActivation = false;
}
/** {@inheritDoc} */
@@ -120,6 +125,9 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
cfg.setConsistentId(igniteInstanceName);
+ if (disableAutoActivation)
+ cfg.setAutoActivationEnabled(false);
+
cfg.setDataStorageConfiguration(
new DataStorageConfiguration().setDefaultDataRegionConfiguration(
new DataRegionConfiguration()
@@ -884,11 +892,15 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
delayRebalance = true;
+ /* There is a problem with handling simultaneous auto activation after restart and manual activation.
+ To properly catch the moment when cluster activation has finished we temporary disable auto activation. */
+ disableAutoActivation = true;
+
startGrids(4);
ig = grid(0);
- ig.active(true);
+ ig.cluster().active(true);
cache = ig.cache(cacheName);
@@ -990,7 +1002,7 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override public void reset() {
- delegate.reset();;
+ delegate.reset();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java
index 1e046d4..ebc804f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java
@@ -31,6 +31,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cache.store.CacheStoreAdapter;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
@@ -93,6 +94,8 @@ public class CacheLoadingConcurrentGridStartSelfTest extends GridCommonAbstractT
ccfg.setCacheStoreFactory(new FactoryBuilder.SingletonFactory(new TestCacheStoreAdapter()));
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 64));
+
if (getTestIgniteInstanceName(0).equals(igniteInstanceName)) {
if (client)
cfg.setClientMode(true);
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
index 63d772a..fc617bb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
@@ -106,6 +106,10 @@ public class GridCachePartitionsStateValidationTest extends GridCommonAbstractTe
awaitPartitionMapExchange();
+ // Populate cache to increment update counters.
+ for (int i = 0; i < 1000; i++)
+ ignite.cache(CACHE_NAME).put(i, i);
+
// Modify update counter for some partition.
for (GridDhtLocalPartition partition : ignite.cachex(CACHE_NAME).context().topology().localPartitions()) {
partition.updateCounter(100500L);
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
index 9ed8d54..43a2303 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
@@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
import org.junit.Assert;
import org.mockito.Matchers;
import org.mockito.Mockito;
@@ -70,24 +71,21 @@ public class GridCachePartitionsStateValidatorSelfTest extends GridCommonAbstrac
Mockito.when(partitionMock.id()).thenReturn(id);
Mockito.when(partitionMock.updateCounter()).thenReturn(updateCounter);
Mockito.when(partitionMock.fullSize()).thenReturn(size);
+ Mockito.when(partitionMock.state()).thenReturn(GridDhtPartitionState.OWNING);
return partitionMock;
}
/**
- * @return Message containing specified {@code countersMap}.
+ * @param countersMap Update counters map.
+ * @param sizesMap Sizes map.
+ * @return Message with specified {@code countersMap} and {@code sizeMap}.
*/
- private GridDhtPartitionsSingleMessage fromUpdateCounters(Map<Integer, T2<Long, Long>> countersMap) {
+ private GridDhtPartitionsSingleMessage from(@Nullable Map<Integer, T2<Long, Long>> countersMap, @Nullable Map<Integer, Long> sizesMap) {
GridDhtPartitionsSingleMessage msg = new GridDhtPartitionsSingleMessage();
- msg.addPartitionUpdateCounters(0, countersMap);
- return msg;
- }
-
- /**
- * @return Message containing specified {@code sizesMap}.
- */
- private GridDhtPartitionsSingleMessage fromCacheSizes(Map<Integer, Long> sizesMap) {
- GridDhtPartitionsSingleMessage msg = new GridDhtPartitionsSingleMessage();
- msg.addPartitionSizes(0, sizesMap);
+ if (countersMap != null)
+ msg.addPartitionUpdateCounters(0, countersMap);
+ if (sizesMap != null)
+ msg.addPartitionSizes(0, sizesMap);
return msg;
}
@@ -98,15 +96,22 @@ public class GridCachePartitionsStateValidatorSelfTest extends GridCommonAbstrac
UUID remoteNode = UUID.randomUUID();
UUID ignoreNode = UUID.randomUUID();
- // For partitions 0 and 2 (zero counter) we have inconsistent update counters.
+ // For partitions 0 and 2 we have inconsistent update counters.
Map<Integer, T2<Long, Long>> updateCountersMap = new HashMap<>();
updateCountersMap.put(0, new T2<>(2L, 2L));
updateCountersMap.put(1, new T2<>(2L, 2L));
+ updateCountersMap.put(2, new T2<>(5L, 5L));
+
+ // For partitions 0 and 2 we have inconsistent cache sizes.
+ Map<Integer, Long> cacheSizesMap = new HashMap<>();
+ cacheSizesMap.put(0, 2L);
+ cacheSizesMap.put(1, 2L);
+ cacheSizesMap.put(2, 2L);
// Form single messages map.
Map<UUID, GridDhtPartitionsSingleMessage> messages = new HashMap<>();
- messages.put(remoteNode, fromUpdateCounters(updateCountersMap));
- messages.put(ignoreNode, fromUpdateCounters(updateCountersMap));
+ messages.put(remoteNode, from(updateCountersMap, cacheSizesMap));
+ messages.put(ignoreNode, from(updateCountersMap, cacheSizesMap));
GridDhtPartitionsStateValidator validator = new GridDhtPartitionsStateValidator(cctxMock);
@@ -120,7 +125,7 @@ public class GridCachePartitionsStateValidatorSelfTest extends GridCommonAbstrac
Assert.assertTrue(result.get(0).get(localNodeId) == 1L);
Assert.assertTrue(result.get(0).get(remoteNode) == 2L);
Assert.assertTrue(result.get(2).get(localNodeId) == 3L);
- Assert.assertTrue(result.get(2).get(remoteNode) == 0L);
+ Assert.assertTrue(result.get(2).get(remoteNode) == 5L);
}
/**
@@ -130,6 +135,12 @@ public class GridCachePartitionsStateValidatorSelfTest extends GridCommonAbstrac
UUID remoteNode = UUID.randomUUID();
UUID ignoreNode = UUID.randomUUID();
+ // For partitions 0 and 2 we have inconsistent update counters.
+ Map<Integer, T2<Long, Long>> updateCountersMap = new HashMap<>();
+ updateCountersMap.put(0, new T2<>(2L, 2L));
+ updateCountersMap.put(1, new T2<>(2L, 2L));
+ updateCountersMap.put(2, new T2<>(5L, 5L));
+
// For partitions 0 and 2 we have inconsistent cache sizes.
Map<Integer, Long> cacheSizesMap = new HashMap<>();
cacheSizesMap.put(0, 2L);
@@ -138,8 +149,8 @@ public class GridCachePartitionsStateValidatorSelfTest extends GridCommonAbstrac
// Form single messages map.
Map<UUID, GridDhtPartitionsSingleMessage> messages = new HashMap<>();
- messages.put(remoteNode, fromCacheSizes(cacheSizesMap));
- messages.put(ignoreNode, fromCacheSizes(cacheSizesMap));
+ messages.put(remoteNode, from(updateCountersMap, cacheSizesMap));
+ messages.put(ignoreNode, from(updateCountersMap, cacheSizesMap));
GridDhtPartitionsStateValidator validator = new GridDhtPartitionsStateValidator(cctxMock);
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
index 6039ae3..fd124b7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
@@ -47,7 +47,7 @@ public class IgniteStandByClusterSuite extends TestSuite {
* @return Test suite.
*/
public static TestSuite suite() {
- TestSuite suite = new TestSuite("Ignite Activate/DeActivate Cluster Test Suit");
+ TestSuite suite = new TestSuite("Ignite Activate/DeActivate Cluster Test Suite");
suite.addTestSuite(IgniteClusterActivateDeactivateTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
index fc1cea6..dd495cf 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
@@ -73,6 +73,8 @@ public class IgniteCacheQueryNodeRestartSelfTest extends GridCacheAbstractSelfTe
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration c = super.getConfiguration(igniteInstanceName);
+ c.setConsistentId(igniteInstanceName);
+
TcpDiscoverySpi disco = new TcpDiscoverySpi();
disco.setIpFinder(ipFinder);
http://git-wip-us.apache.org/repos/asf/ignite/blob/14c032ae/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
index 8e049ac..3ee19d5 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
@@ -58,7 +58,7 @@ public class IgniteChangingBaselineCacheQueryNodeRestartSelfTest extends IgniteC
initStoreStrategy();
- grid(0).active(true);
+ grid(0).cluster().active(true);
awaitPartitionMapExchange();
}
@@ -74,7 +74,7 @@ public class IgniteChangingBaselineCacheQueryNodeRestartSelfTest extends IgniteC
@Override protected IgniteInternalFuture createRestartAction(final AtomicBoolean done, final AtomicInteger restartCnt) throws Exception {
return multithreadedAsync(new Callable<Object>() {
/** */
- private final long baselineTopChangeInterval = 30 * 1000;
+ private final long baselineTopChangeInterval = 10 * 1000;
/** */
private final int logFreq = 50;
[22/50] [abbrv] ignite git commit: IGNITE-7512 Check for null before
validateKeyAndValue in GridDhtAtomicCache.updateWithBatch - Fixes #3429.
Posted by ag...@apache.org.
IGNITE-7512 Check for null before validateKeyAndValue in GridDhtAtomicCache.updateWithBatch - Fixes #3429.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9d184c9c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9d184c9c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9d184c9c
Branch: refs/heads/ignite-6083
Commit: 9d184c9c08496fa7bd5f37e12dc37d4618576713
Parents: 14c032a
Author: skalashnikov <sk...@gridgain.com>
Authored: Wed Apr 18 16:37:20 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Apr 18 16:37:20 2018 +0300
----------------------------------------------------------------------
.../dht/atomic/GridDhtAtomicCache.java | 3 +-
.../local/atomic/GridLocalAtomicCache.java | 2 +-
.../transactions/IgniteTxLocalAdapter.java | 5 +-
.../query/IgniteSqlNotNullConstraintTest.java | 71 +++++++++++++++++++-
4 files changed, 77 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9d184c9c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index e3ad382..44f2b15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -2071,7 +2071,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
validation = true;
- ctx.validateKeyAndValue(entry.key(), updated);
+ if (updated != null)
+ ctx.validateKeyAndValue(entry.key(), updated);
}
}
catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/9d184c9c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index 6cb50f2..dad1052 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -1074,7 +1074,7 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
if (computed != null)
invokeRes = CacheInvokeResult.fromResult(ctx.unwrapTemporary(computed));
- if (invokeEntry.modified()) {
+ if (invokeEntry.modified() && updated != null) {
validation = true;
ctx.validateKeyAndValue(entry.key(), updated);
http://git-wip-us.apache.org/repos/asf/ignite/blob/9d184c9c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index c2a2842..7a7f65f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1256,7 +1256,10 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
key0 = invokeEntry.key();
}
- ctx.validateKeyAndValue(txEntry.key(), ctx.toCacheObject(val0));
+ val0 = ctx.toCacheObject(val0);
+
+ if (val0 != null)
+ ctx.validateKeyAndValue(txEntry.key(), (CacheObject)val0);
if (res != null)
ret.addEntryProcessResult(ctx, txEntry.key(), key0, res, null, txEntry.keepBinary());
http://git-wip-us.apache.org/repos/asf/ignite/blob/9d184c9c/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java
index 1f4e018..183138b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java
@@ -681,6 +681,72 @@ public class IgniteSqlNotNullConstraintTest extends GridCommonAbstractTest {
}
/** */
+ public void testAtomicOrImplicitTxInvokeDelete() throws Exception {
+ executeWithAllCaches(new TestClosure() {
+ @Override public void run() throws Exception {
+ cache.put(key1, okValue);
+
+ cache.invoke(key1, new TestEntryProcessor(null));
+
+ assertEquals(0, cache.size());
+ }
+ });
+ }
+
+ /** */
+ public void testAtomicOrImplicitTxInvokeAllDelete() throws Exception {
+ executeWithAllCaches(new TestClosure() {
+ @Override public void run() throws Exception {
+ cache.put(key1, okValue);
+ cache.put(key2, okValue);
+
+ cache.invokeAll(F.asMap(
+ key1, new TestEntryProcessor(null),
+ key2, new TestEntryProcessor(null)));
+
+ assertEquals(0, cache.size());
+ }
+ });
+ }
+
+ /** */
+ public void testTxInvokeDelete() throws Exception {
+ executeWithAllTxCaches(new TestClosure() {
+ @Override public void run() throws Exception {
+ cache.put(key1, okValue);
+
+ try (Transaction tx = ignite.transactions().txStart(concurrency, isolation)) {
+ cache.invoke(key1, new TestEntryProcessor(null));
+
+ tx.commit();
+ }
+
+ assertEquals(0, cache.size());
+ }
+ });
+ }
+
+ /** */
+ public void testTxInvokeAllDelete() throws Exception {
+ executeWithAllTxCaches(new TestClosure() {
+ @Override public void run() throws Exception {
+ cache.put(key1, okValue);
+ cache.put(key2, okValue);
+
+ try (Transaction tx = ignite.transactions().txStart(concurrency, isolation)) {
+ cache.invokeAll(F.asMap(
+ key1, new TestEntryProcessor(null),
+ key2, new TestEntryProcessor(null)));
+
+ tx.commit();
+ }
+
+ assertEquals(0, cache.size());
+ }
+ });
+ }
+
+ /** */
public void testDynamicTableCreateNotNullFieldsAllowed() throws Exception {
executeSql("CREATE TABLE test(id INT PRIMARY KEY, field INT NOT NULL)");
@@ -1137,7 +1203,10 @@ public class IgniteSqlNotNullConstraintTest extends GridCommonAbstractTest {
@Override public Object process(MutableEntry<Integer, Person> entry,
Object... objects) throws EntryProcessorException {
- entry.setValue(value);
+ if (value == null)
+ entry.remove();
+ else
+ entry.setValue(value);
return null;
}
[40/50] [abbrv] ignite git commit: IGNITE-7077: Implementation of
Spark query optimization. - Fixes #3397.
Posted by ag...@apache.org.
IGNITE-7077: Implementation of Spark query optimization. - Fixes #3397.
Signed-off-by: Nikolay Izhikov <ni...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/10a4c48b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/10a4c48b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/10a4c48b
Branch: refs/heads/ignite-6083
Commit: 10a4c48bf722f49e7eca09c50cf4af6c2ec59b1b
Parents: 8008601
Author: Nikolay Izhikov <ni...@apache.org>
Authored: Fri Apr 20 11:23:42 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Fri Apr 20 11:23:42 2018 +0300
----------------------------------------------------------------------
.../ignite/spark/IgniteDataFrameSettings.scala | 9 +
.../org/apache/ignite/spark/IgniteRDD.scala | 7 +-
.../spark/impl/IgniteRelationProvider.scala | 20 +-
.../impl/IgniteSQLAccumulatorRelation.scala | 98 ++++
.../spark/impl/IgniteSQLDataFrameRDD.scala | 16 +-
.../ignite/spark/impl/IgniteSQLRelation.scala | 67 +--
.../optimization/AggregateExpressions.scala | 114 ++++
.../optimization/ConditionExpressions.scala | 160 ++++++
.../impl/optimization/DateExpressions.scala | 127 +++++
.../impl/optimization/IgniteQueryContext.scala | 52 ++
.../impl/optimization/MathExpressions.scala | 263 +++++++++
.../impl/optimization/SimpleExpressions.scala | 180 ++++++
.../impl/optimization/StringExpressions.scala | 154 ++++++
.../optimization/SupportedExpressions.scala | 42 ++
.../impl/optimization/SystemExpressions.scala | 122 +++++
.../accumulator/JoinSQLAccumulator.scala | 222 ++++++++
.../accumulator/QueryAccumulator.scala | 70 +++
.../accumulator/SelectAccumulator.scala | 70 +++
.../accumulator/SingleTableSQLAccumulator.scala | 124 +++++
.../accumulator/UnionSQLAccumulator.scala | 63 +++
.../spark/impl/optimization/package.scala | 230 ++++++++
.../org/apache/ignite/spark/impl/package.scala | 48 +-
.../spark/sql/ignite/IgniteOptimization.scala | 436 +++++++++++++++
.../spark/sql/ignite/IgniteSparkSession.scala | 10 +-
.../ignite/spark/AbstractDataFrameSpec.scala | 68 ++-
.../apache/ignite/spark/IgniteCatalogSpec.scala | 7 +-
.../spark/IgniteDataFrameSchemaSpec.scala | 5 +-
.../ignite/spark/IgniteDataFrameSuite.scala | 9 +-
.../IgniteOptimizationAggregationFuncSpec.scala | 189 +++++++
.../spark/IgniteOptimizationDateFuncSpec.scala | 230 ++++++++
.../IgniteOptimizationDisableEnableSpec.scala | 127 +++++
.../spark/IgniteOptimizationJoinSpec.scala | 543 +++++++++++++++++++
.../spark/IgniteOptimizationMathFuncSpec.scala | 358 ++++++++++++
.../ignite/spark/IgniteOptimizationSpec.scala | 305 +++++++++++
.../IgniteOptimizationStringFuncSpec.scala | 313 +++++++++++
.../IgniteOptimizationSystemFuncSpec.scala | 147 +++++
36 files changed, 4924 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteDataFrameSettings.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteDataFrameSettings.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteDataFrameSettings.scala
index 6bff476..9daaec4 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteDataFrameSettings.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteDataFrameSettings.scala
@@ -153,4 +153,13 @@ object IgniteDataFrameSettings {
* @see [[org.apache.ignite.IgniteDataStreamer#perNodeParallelOperations(int)]]
*/
val OPTION_STREAMER_PER_NODE_PARALLEL_OPERATIONS = "streamerPerNodeParallelOperations"
+
+ /**
+ * Option for a [[org.apache.spark.sql.SparkSession]] configuration.
+ * If `true` then all Ignite optimization of Spark SQL statements will be disabled.
+ * Default value is `false`.
+ *
+ * @see [[org.apache.spark.sql.ignite.IgniteOptimization]]
+ */
+ val OPTION_DISABLE_SPARK_SQL_OPTIMIZATION = "ignite.disableSparkSQLOptimization"
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
index d87ea0a..5fb81b6 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
@@ -344,6 +344,11 @@ class IgniteRDD[K, V] (
object IgniteRDD {
/**
+ * Default decimal type.
+ */
+ private[spark] val DECIMAL = DecimalType(DecimalType.MAX_PRECISION, 3)
+
+ /**
* Gets Spark data type based on type name.
*
* @param typeName Type name.
@@ -357,7 +362,7 @@ object IgniteRDD {
case "java.lang.Long" ⇒ LongType
case "java.lang.Float" ⇒ FloatType
case "java.lang.Double" ⇒ DoubleType
- case "java.math.BigDecimal" ⇒ DataTypes.createDecimalType()
+ case "java.math.BigDecimal" ⇒ DECIMAL
case "java.lang.String" ⇒ StringType
case "java.util.Date" ⇒ DateType
case "java.sql.Date" ⇒ DateType
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteRelationProvider.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteRelationProvider.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteRelationProvider.scala
index a9f9f89..e4fa9f7 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteRelationProvider.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteRelationProvider.scala
@@ -26,6 +26,7 @@ import org.apache.ignite.spark.IgniteDataFrameSettings._
import org.apache.ignite.spark.impl.QueryHelper.{createTable, dropTable, ensureCreateTableOptions, saveTable}
import org.apache.spark.sql.SaveMode.{Append, Overwrite}
import org.apache.spark.sql.ignite.IgniteExternalCatalog.{IGNITE_PROTOCOL, OPTION_GRID}
+import org.apache.spark.sql.ignite.IgniteOptimization
import org.apache.spark.sql.sources._
import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode}
@@ -176,11 +177,28 @@ class IgniteRelationProvider extends RelationProvider
* @param sqlCtx SQL context.
* @return Ignite SQL relation.
*/
- private def createRelation(igniteCtx: IgniteContext, tblName: String, sqlCtx: SQLContext): BaseRelation =
+ private def createRelation(igniteCtx: IgniteContext, tblName: String, sqlCtx: SQLContext): BaseRelation = {
+ val optimizationDisabled =
+ sqlCtx.sparkSession.conf.get(OPTION_DISABLE_SPARK_SQL_OPTIMIZATION, "false").toBoolean
+
+ val experimentalMethods = sqlCtx.sparkSession.sessionState.experimentalMethods
+
+ if (optimizationDisabled) {
+ experimentalMethods.extraOptimizations =
+ experimentalMethods.extraOptimizations.filter(_ != IgniteOptimization)
+ }
+ else {
+ val optimizationExists = experimentalMethods.extraOptimizations.contains(IgniteOptimization)
+
+ if (!optimizationExists)
+ experimentalMethods.extraOptimizations = experimentalMethods.extraOptimizations :+ IgniteOptimization
+ }
+
IgniteSQLRelation(
igniteCtx,
tblName,
sqlCtx)
+ }
/**
* @param params Params.
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLAccumulatorRelation.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLAccumulatorRelation.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLAccumulatorRelation.scala
new file mode 100644
index 0000000..6eb600a
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLAccumulatorRelation.scala
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl
+
+import org.apache.ignite.spark.impl
+import org.apache.ignite.spark.impl.optimization.accumulator.{JoinSQLAccumulator, QueryAccumulator}
+import org.apache.ignite.spark.impl.optimization.isSimpleTableAcc
+import org.apache.spark.Partition
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.{Row, SQLContext}
+import org.apache.spark.sql.sources.{BaseRelation, TableScan}
+import org.apache.spark.sql.types.{Metadata, StructField, StructType}
+
+/**
+ * Relation to query data from query generated by <code>QueryAccumulator</code>.
+ * <code>QueryAccumulator</code> is generated by <code>IgniteOptimization</code>.
+ *
+ * @see IgniteOptimization
+ */
+class IgniteSQLAccumulatorRelation[K, V](val acc: QueryAccumulator)
+ (@transient val sqlContext: SQLContext) extends BaseRelation with TableScan {
+
+ /** @inheritdoc */
+ override def schema: StructType =
+ StructType(acc.output.map { c ⇒
+ StructField(
+ name = c.name,
+ dataType = c.dataType,
+ nullable = c.nullable,
+ metadata = Metadata.empty)
+ })
+
+ /** @inheritdoc */
+ override def buildScan(): RDD[Row] =
+ IgniteSQLDataFrameRDD[K, V](
+ acc.igniteQueryContext.igniteContext,
+ acc.igniteQueryContext.cacheName,
+ schema,
+ acc.compileQuery(),
+ List.empty,
+ calcPartitions,
+ isDistributeJoin(acc)
+ )
+
+ /** @inheritdoc */
+ override def toString: String =
+ s"IgniteSQLAccumulatorRelation(columns=[${acc.output.map(_.name).mkString(", ")}], qry=${acc.compileQuery()})"
+
+ /**
+ * @return Collection of spark partition.
+ */
+ private def calcPartitions: Array[Partition] =
+ //If accumulator stores some complex query(join, aggregation, limit, order, etc.).
+ //we has to load data from Ignite as a single Spark partition.
+ if (!isSimpleTableAcc(acc)){
+ val aff = acc.igniteQueryContext.igniteContext.ignite().affinity(acc.igniteQueryContext.cacheName)
+
+ val parts = aff.partitions()
+
+ Array(IgniteDataFramePartition(0, primary = null, igniteParts = (0 until parts).toList))
+ }
+ else
+ impl.calcPartitions(acc.igniteQueryContext.igniteContext, acc.igniteQueryContext.cacheName)
+
+ /**
+ * @param acc Plan.
+ * @return True if plan of one or its children are `JoinSQLAccumulator`, false otherwise.
+ */
+ private def isDistributeJoin(acc: LogicalPlan): Boolean =
+ acc match {
+ case _: JoinSQLAccumulator ⇒
+ true
+
+ case _ ⇒
+ acc.children.exists(isDistributeJoin)
+ }
+}
+
+object IgniteSQLAccumulatorRelation {
+ def apply[K, V](acc: QueryAccumulator): IgniteSQLAccumulatorRelation[K, V] =
+ new IgniteSQLAccumulatorRelation[K, V](acc)(acc.igniteQueryContext.sqlContext)
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLDataFrameRDD.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLDataFrameRDD.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLDataFrameRDD.scala
index 93ef529..ec502fc 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLDataFrameRDD.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLDataFrameRDD.scala
@@ -36,7 +36,8 @@ class IgniteSQLDataFrameRDD[K, V](
schema: StructType,
qryStr: String,
args: List[_],
- parts: Array[Partition]) extends
+ parts: Array[Partition],
+ distributedJoin: Boolean) extends
IgniteSqlRDD[Row, JList[_], K, V](
ic,
cacheName,
@@ -56,13 +57,17 @@ class IgniteSQLDataFrameRDD[K, V](
override def compute(partition: Partition, context: TaskContext): Iterator[Row] = {
val qry0 = new SqlFieldsQuery(qryStr)
+ qry0.setDistributedJoins(distributedJoin)
+
if (args.nonEmpty)
qry0.setArgs(args.map(_.asInstanceOf[Object]): _*)
val ccfg = ic.ignite().cache[K, V](cacheName).getConfiguration(classOf[CacheConfiguration[K, V]])
- if (ccfg.getCacheMode != CacheMode.REPLICATED)
- qry0.setPartitions(partition.asInstanceOf[IgniteDataFramePartition].igniteParts: _*)
+ val ignitePartition = partition.asInstanceOf[IgniteDataFramePartition]
+
+ if (ccfg.getCacheMode != CacheMode.REPLICATED && ignitePartition.igniteParts.nonEmpty && !distributedJoin)
+ qry0.setPartitions(ignitePartition.igniteParts: _*)
qry = qry0
@@ -76,7 +81,8 @@ object IgniteSQLDataFrameRDD {
schema: StructType,
qryStr: String,
args: List[_],
- parts: Array[Partition] = Array(IgnitePartition(0))) = {
- new IgniteSQLDataFrameRDD(ic, cacheName, schema, qryStr, args, parts)
+ parts: Array[Partition] = Array(IgnitePartition(0)),
+ distributedJoin: Boolean = false): IgniteSQLDataFrameRDD[K, V] = {
+ new IgniteSQLDataFrameRDD[K, V](ic, cacheName, schema, qryStr, args, parts, distributedJoin)
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLRelation.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLRelation.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLRelation.scala
index 1fb8de7..485ddf6 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLRelation.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLRelation.scala
@@ -18,10 +18,8 @@
package org.apache.ignite.spark.impl
import org.apache.ignite.IgniteException
-import org.apache.ignite.cache.{CacheMode, QueryEntity}
-import org.apache.ignite.cluster.ClusterNode
-import org.apache.ignite.configuration.CacheConfiguration
-import org.apache.ignite.spark.{IgniteContext, IgniteRDD}
+import org.apache.ignite.cache.QueryEntity
+import org.apache.ignite.spark.{IgniteContext, IgniteRDD, impl}
import org.apache.spark.Partition
import org.apache.spark.internal.Logging
import org.apache.spark.rdd.RDD
@@ -30,14 +28,13 @@ import org.apache.spark.sql.types._
import org.apache.spark.sql.{Row, SQLContext}
import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
/**
* Apache Ignite implementation of Spark BaseRelation with PrunedFilteredScan for Ignite SQL Tables
*/
class IgniteSQLRelation[K, V](
- private[spark] val ic: IgniteContext,
- private[spark] val tableName: String)
+ private[apache] val ic: IgniteContext,
+ private[apache] val tableName: String)
(@transient val sqlContext: SQLContext) extends BaseRelation with PrunedFilteredScan with Logging {
/**
@@ -92,41 +89,13 @@ class IgniteSQLRelation[K, V](
qryAndArgs
}
- private def calcPartitions(filters: Array[Filter]): Array[Partition] = {
- val cache = ic.ignite().cache[K, V](cacheName)
-
- val ccfg = cache.getConfiguration(classOf[CacheConfiguration[K, V]])
-
- if (ccfg.getCacheMode == CacheMode.REPLICATED) {
- val serverNodes = ic.ignite().cluster().forCacheNodes(cacheName).forServers().nodes()
-
- Array(IgniteDataFramePartition(0, serverNodes.head, Stream.from(0).take(1024).toList))
- }
- else {
- val aff = ic.ignite().affinity(cacheName)
-
- val parts = aff.partitions()
-
- val nodesToParts = (0 until parts).foldLeft(Map[ClusterNode, ArrayBuffer[Int]]()) {
- case (nodeToParts, ignitePartIdx) ⇒
- val primary = aff.mapPartitionToPrimaryAndBackups(ignitePartIdx).head
-
- if (nodeToParts.contains(primary)) {
- nodeToParts(primary) += ignitePartIdx
-
- nodeToParts
- }
- else
- nodeToParts + (primary → ArrayBuffer[Int](ignitePartIdx))
- }
-
- val partitions = nodesToParts.zipWithIndex.map { case ((node, nodesParts), i) ⇒
- IgniteDataFramePartition(i, node, nodesParts.toList)
- }
-
- partitions.toArray
- }
- }
+ /**
+ * Computes spark partitions for this relation.
+ *
+ * @return Array of IgniteDataFramPartition.
+ */
+ private def calcPartitions(filters: Array[Filter]): Array[Partition] =
+ impl.calcPartitions(ic, cacheName)
/**
* Cache name for a table name.
@@ -134,20 +103,6 @@ class IgniteSQLRelation[K, V](
private lazy val cacheName: String =
sqlCacheName(ic.ignite(), tableName)
.getOrElse(throw new IgniteException(s"Unknown table $tableName"))
-
- /**
- * Utility method to add clause to sql WHERE string.
- *
- * @param filterStr Current filter string
- * @param clause Clause to add.
- * @return Filter string.
- */
- private def addStrClause(filterStr: String, clause: String) =
- if (filterStr.isEmpty)
- clause
- else
- filterStr + " AND " + clause
-
}
object IgniteSQLRelation {
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/AggregateExpressions.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/AggregateExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/AggregateExpressions.scala
new file mode 100644
index 0000000..421a9a9
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/AggregateExpressions.scala
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.spark.sql.catalyst.expressions.aggregate._
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.types._
+
+/**
+ * Object to support aggregate expressions like `sum` or `avg`.
+ */
+private[optimization] object AggregateExpressions extends SupportedExpressions {
+ /** @inheritdoc */
+ def apply(expr: Expression, checkChild: (Expression) ⇒ Boolean): Boolean = expr match {
+ case AggregateExpression(aggregateFunction, _, _, _) ⇒
+ checkChild(aggregateFunction)
+
+ case Average(child) ⇒
+ checkChild(child)
+
+ case Count(children) ⇒
+ children.forall(checkChild)
+
+ case Max(child) ⇒
+ checkChild(child)
+
+ case Min(child) ⇒
+ checkChild(child)
+
+ case Sum(child) ⇒
+ checkChild(child)
+
+ case _ ⇒
+ false
+ }
+
+ /** @inheritdoc */
+ override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+ useAlias: Boolean): Option[String] = expr match {
+ case AggregateExpression(aggregateFunction, _, isDistinct, _) ⇒
+ aggregateFunction match {
+ case Count(children) ⇒
+ if (isDistinct)
+ Some(s"COUNT(DISTINCT ${children.map(childToString(_)).mkString(" ")})")
+ else
+ Some(s"COUNT(${children.map(childToString(_)).mkString(" ")})")
+
+ case sum: Sum ⇒
+ if (isDistinct)
+ Some(castSum(
+ s"SUM(DISTINCT ${sum.children.map(childToString(_)).mkString(" ")})", sum.dataType))
+ else
+ Some(castSum(s"SUM(${sum.children.map(childToString(_)).mkString(" ")})", sum.dataType))
+
+ case _ ⇒
+ Some(childToString(aggregateFunction))
+ }
+
+ case Average(child) ⇒
+ child.dataType match {
+ case DecimalType() | DoubleType ⇒
+ Some(s"AVG(${childToString(child)})")
+
+ case _ ⇒
+ //Spark `AVG` return type is always a double or a decimal.
+ //See [[org.apache.spark.sql.catalyst.expressions.aggregate.Average]]
+ //But Ignite `AVG` return type for a integral types is integral.
+ //To preserve query correct results has to cast column to double.
+ Some(s"AVG(CAST(${childToString(child)} AS DOUBLE))")
+ }
+
+
+ case Count(children) ⇒
+ Some(s"COUNT(${children.map(childToString(_)).mkString(" ")})")
+
+ case Max(child) ⇒
+ Some(s"MAX(${childToString(child)})")
+
+ case Min(child) ⇒
+ Some(s"MIN(${childToString(child)})")
+
+ case sum: Sum ⇒
+ Some(castSum(s"SUM(${childToString(sum.child)})", sum.dataType))
+
+ case _ ⇒
+ None
+ }
+
+ /**
+ * Ignite returns BigDecimal but Spark expects BIGINT.
+ */
+ private def castSum(sumSql: String, dataType: DataType): String = dataType match {
+ case LongType ⇒
+ s"CAST($sumSql AS BIGINT)"
+
+ case _ ⇒
+ s"$sumSql"
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/ConditionExpressions.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/ConditionExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/ConditionExpressions.scala
new file mode 100644
index 0000000..fbfbd64
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/ConditionExpressions.scala
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, _}
+
+/**
+ * Object to support condition expression. Like `and` or `in` operators.
+ */
+private[optimization] object ConditionExpressions extends SupportedExpressions {
+ /** @inheritdoc */
+ def apply(expr: Expression, checkChild: (Expression) ⇒ Boolean): Boolean = expr match {
+ case EqualTo(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case EqualNullSafe(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case GreaterThan(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case GreaterThanOrEqual(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case LessThan(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case LessThanOrEqual(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case InSet(child, set) if set.forall(_.isInstanceOf[Literal]) ⇒
+ checkChild(child)
+
+ case In(child, list) if list.forall(_.isInstanceOf[Literal]) ⇒
+ checkChild(child)
+
+ case IsNull(child) ⇒
+ checkChild(child)
+
+ case IsNotNull(child) ⇒
+ checkChild(child)
+
+ case And(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case Or(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case Not(child) ⇒
+ checkChild(child)
+
+ case StartsWith(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case EndsWith(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case Contains(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case _ ⇒
+ false
+ }
+
+ /** @inheritdoc */
+ override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+ useAlias: Boolean): Option[String] = expr match {
+ case EqualTo(left, right) ⇒
+ Some(s"${childToString(left)} = ${childToString(right)}")
+
+ case EqualNullSafe(left, right) ⇒
+ Some(s"(${childToString(left)} IS NULL OR ${childToString(left)} = ${childToString(right)})")
+
+ case GreaterThan(left, right) ⇒
+ Some(s"${childToString(left)} > ${childToString(right)}")
+
+ case GreaterThanOrEqual(left, right) ⇒
+ Some(s"${childToString(left)} >= ${childToString(right)}")
+
+ case LessThan(left, right) ⇒
+ Some(s"${childToString(left)} < ${childToString(right)}")
+
+ case LessThanOrEqual(left, right) ⇒
+ Some(s"${childToString(left)} <= ${childToString(right)}")
+
+ case In(attr, values) ⇒
+ Some(s"${childToString(attr)} IN (${values.map(childToString(_)).mkString(", ")})")
+
+ case IsNull(child) ⇒
+ Some(s"${childToString(child)} IS NULL")
+
+ case IsNotNull(child) ⇒
+ Some(s"${childToString(child)} IS NOT NULL")
+
+ case And(left, right) ⇒
+ Some(s"${childToString(left)} AND ${childToString(right)}")
+
+ case Or(left, right) ⇒
+ Some(s"${childToString(left)} OR ${childToString(right)}")
+
+ case Not(child) ⇒
+ Some(s"NOT ${childToString(child)}")
+
+ case StartsWith(attr, value) ⇒ {
+ //Expecting string literal here.
+ //To add % sign it's required to remove quotes.
+ val valStr = removeQuotes(childToString(value))
+
+ Some(s"${childToString(attr)} LIKE '$valStr%'")
+ }
+
+ case EndsWith(attr, value) ⇒ {
+ //Expecting string literal here.
+ //To add % sign it's required to remove quotes.
+ val valStr = removeQuotes(childToString(value))
+
+ Some(s"${childToString(attr)} LIKE '%$valStr'")
+ }
+
+ case Contains(attr, value) ⇒ {
+ //Expecting string literal here.
+ //To add % signs it's required to remove quotes.
+ val valStr = removeQuotes(childToString(value))
+
+ Some(s"${childToString(attr)} LIKE '%$valStr%'")
+ }
+
+ case _ ⇒
+ None
+ }
+
+ /**
+ * @param str String to process.
+ * @return Str without surrounding quotes.
+ */
+ private def removeQuotes(str: String): String =
+ if (str.length < 2)
+ str
+ else
+ str match {
+ case quoted if quoted.startsWith("'") && quoted.endsWith("'") ⇒
+ quoted.substring(1, quoted.length-1)
+
+ case _ ⇒ str
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/DateExpressions.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/DateExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/DateExpressions.scala
new file mode 100644
index 0000000..d075bf0
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/DateExpressions.scala
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, _}
+
+/**
+ * Object to support expressions to work with date/timestamp.
+ */
+private[optimization] object DateExpressions extends SupportedExpressions {
+ /** @inheritdoc */
+ def apply(expr: Expression, checkChild: (Expression) ⇒ Boolean): Boolean = expr match {
+ case CurrentDate(None) ⇒
+ true
+
+ case CurrentTimestamp() ⇒
+ true
+
+ case DateAdd(startDate, days) ⇒
+ checkChild(startDate) && checkChild(days)
+
+ case DateDiff(date1, date2) ⇒
+ checkChild(date1) && checkChild(date2)
+
+ case DayOfMonth(date) ⇒
+ checkChild(date)
+
+ case DayOfYear(date) ⇒
+ checkChild(date)
+
+ case Hour(date, _) ⇒
+ checkChild(date)
+
+ case Minute(date, _) ⇒
+ checkChild(date)
+
+ case Month(date) ⇒
+ checkChild(date)
+
+ case ParseToDate(left, format, child) ⇒
+ checkChild(left) && (format.isEmpty || checkChild(format.get)) && checkChild(child)
+
+ case Quarter(date) ⇒
+ checkChild(date)
+
+ case Second(date, _) ⇒
+ checkChild(date)
+
+ case WeekOfYear(date) ⇒
+ checkChild(date)
+
+ case Year(date) ⇒
+ checkChild(date)
+
+ case _ ⇒
+ false
+ }
+
+ /** @inheritdoc */
+ override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+ useAlias: Boolean): Option[String] = expr match {
+ case CurrentDate(_) ⇒
+ Some(s"CURRENT_DATE()")
+
+ case CurrentTimestamp() ⇒
+ Some(s"CURRENT_TIMESTAMP()")
+
+ case DateAdd(startDate, days) ⇒
+ Some(s"CAST(DATEADD('DAY', ${childToString(days)}, ${childToString(startDate)}) AS DATE)")
+
+ case DateDiff(date1, date2) ⇒
+ Some(s"CAST(DATEDIFF('DAY', ${childToString(date1)}, ${childToString(date2)}) AS INT)")
+
+ case DayOfMonth(date) ⇒
+ Some(s"DAY_OF_MONTH(${childToString(date)})")
+
+ case DayOfYear(date) ⇒
+ Some(s"DAY_OF_YEAR(${childToString(date)})")
+
+ case Hour(date, _) ⇒
+ Some(s"HOUR(${childToString(date)})")
+
+ case Minute(date, _) ⇒
+ Some(s"MINUTE(${childToString(date)})")
+
+ case Month(date) ⇒
+ Some(s"MINUTE(${childToString(date)})")
+
+ case ParseToDate(left, formatOption, _) ⇒
+ formatOption match {
+ case Some(format) ⇒
+ Some(s"PARSEDATETIME(${childToString(left)}, ${childToString(format)})")
+ case None ⇒
+ Some(s"PARSEDATETIME(${childToString(left)})")
+ }
+
+ case Quarter(date) ⇒
+ Some(s"QUARTER(${childToString(date)})")
+
+ case Second(date, _) ⇒
+ Some(s"SECOND(${childToString(date)})")
+
+ case WeekOfYear(date) ⇒
+ Some(s"WEEK(${childToString(date)})")
+
+ case Year(date) ⇒
+ Some(s"YEAR(${childToString(date)})")
+
+ case _ ⇒
+ None
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/IgniteQueryContext.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/IgniteQueryContext.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/IgniteQueryContext.scala
new file mode 100644
index 0000000..c5a7f34
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/IgniteQueryContext.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.ignite.spark.IgniteContext
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.sql.catalyst.catalog.CatalogTable
+import org.apache.spark.sql.catalyst.expressions.NamedExpression
+
+/**
+ * Class to store Ignite query info during optimization process.
+ *
+ * @param igniteContext IgniteContext.
+ * @param sqlContext SQLContext.
+ * @param cacheName Cache name.
+ * @param aliasIndex Iterator to generate indexes for auto-generated aliases.
+ * @param catalogTable CatalogTable from source relation.
+ */
+case class IgniteQueryContext(
+ igniteContext: IgniteContext,
+ sqlContext: SQLContext,
+ cacheName: String,
+ aliasIndex: Iterator[Int],
+ catalogTable: Option[CatalogTable] = None,
+ distributeJoin: Boolean = false
+) {
+ /**
+ * @return Unique table alias.
+ */
+ def uniqueTableAlias: String = "table" + aliasIndex.next
+
+ /**
+ * @param col Column
+ * @return Unique column alias.
+ */
+ def uniqueColumnAlias(col: NamedExpression): String = col.name + "_" + aliasIndex.next
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/MathExpressions.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/MathExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/MathExpressions.scala
new file mode 100644
index 0000000..dc05e95
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/MathExpressions.scala
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, _}
+
+/**
+ * Object to support math expressions.
+ */
+private[optimization] object MathExpressions extends SupportedExpressions {
+ /** @inheritdoc */
+ def apply(expr: Expression, checkChild: (Expression) ⇒ Boolean): Boolean = expr match {
+ case Abs(child) ⇒
+ checkChild(child)
+
+ case Acos(child) ⇒
+ checkChild(child)
+
+ case Asin(child) ⇒
+ checkChild(child)
+
+ case Atan(child) ⇒
+ checkChild(child)
+
+ case Cos(child) ⇒
+ checkChild(child)
+
+ case Cosh(child) ⇒
+ checkChild(child)
+
+ case Sin(child) ⇒
+ checkChild(child)
+
+ case Sinh(child) ⇒
+ checkChild(child)
+
+ case Tan(child) ⇒
+ checkChild(child)
+
+ case Tanh(child) ⇒
+ checkChild(child)
+
+ case Atan2(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case BitwiseAnd(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case BitwiseOr(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case BitwiseXor(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case Ceil(child) ⇒
+ checkChild(child)
+
+ case ToDegrees(child) ⇒
+ checkChild(child)
+
+ case Exp(child) ⇒
+ checkChild(child)
+
+ case Floor(child) ⇒
+ checkChild(child)
+
+ case Log(child) ⇒
+ checkChild(child)
+
+ case Log10(child) ⇒
+ checkChild(child)
+
+ case Logarithm(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case ToRadians(child) ⇒
+ checkChild(child)
+
+ case Sqrt(child) ⇒
+ checkChild(child)
+
+ case _: Pi ⇒
+ true
+
+ case _: EulerNumber ⇒
+ true
+
+ case Pow(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case Rand(child) ⇒
+ checkChild(child)
+
+ case Round(child, scale) ⇒
+ checkChild(child) && checkChild(scale)
+
+ case Signum(child) ⇒
+ checkChild(child)
+
+ case Remainder(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case Divide(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case Multiply(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case Subtract(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case Add(left, right) ⇒
+ checkChild(left) && checkChild(right)
+
+ case UnaryMinus(child) ⇒
+ checkChild(child)
+
+ case UnaryPositive(child) ⇒
+ checkChild(child)
+
+ case _ ⇒ false
+ }
+
+ /** @inheritdoc */
+ override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+ useAlias: Boolean): Option[String] = expr match {
+ case Abs(child) ⇒
+ Some(s"ABS(${childToString(child)})")
+
+ case Acos(child) ⇒
+ Some(s"ACOS(${childToString(child)})")
+
+ case Asin(child) ⇒
+ Some(s"ASIN(${childToString(child)})")
+
+ case Atan(child) ⇒
+ Some(s"ATAN(${childToString(child)})")
+
+ case Cos(child) ⇒
+ Some(s"COS(${childToString(child)})")
+
+ case Cosh(child) ⇒
+ Some(s"COSH(${childToString(child)})")
+
+ case Sin(child) ⇒
+ Some(s"SIN(${childToString(child)})")
+
+ case Sinh(child) ⇒
+ Some(s"SINH(${childToString(child)})")
+
+ case Tan(child) ⇒
+ Some(s"TAN(${childToString(child)})")
+
+ case Tanh(child) ⇒
+ Some(s"TANH(${childToString(child)})")
+
+ case Atan2(left, right) ⇒
+ Some(s"ATAN2(${childToString(left)}, ${childToString(right)})")
+
+ case BitwiseAnd(left, right) ⇒
+ Some(s"BITAND(${childToString(left)}, ${childToString(right)})")
+
+ case BitwiseOr(left, right) ⇒
+ Some(s"BITOR(${childToString(left)}, ${childToString(right)})")
+
+ case BitwiseXor(left, right) ⇒
+ Some(s"BITXOR(${childToString(left)}, ${childToString(right)})")
+
+ case Ceil(child) ⇒
+ Some(s"CAST(CEIL(${childToString(child)}) AS LONG)")
+
+ case ToDegrees(child) ⇒
+ Some(s"DEGREES(${childToString(child)})")
+
+ case Exp(child) ⇒
+ Some(s"EXP(${childToString(child)})")
+
+ case Floor(child) ⇒
+ Some(s"CAST(FLOOR(${childToString(child)}) AS LONG)")
+
+ case Log(child) ⇒
+ Some(s"LOG(${childToString(child)})")
+
+ case Log10(child) ⇒
+ Some(s"LOG10(${childToString(child)})")
+
+ case Logarithm(base, arg) ⇒
+ childToString(base) match {
+ //Spark internally converts LN(XXX) to LOG(2.718281828459045, XXX).
+ //Because H2 doesn't have builtin function for a free base logarithm
+ //I want to prevent usage of log(a, b) = ln(a)/ln(b) when possible.
+ case "2.718281828459045" ⇒
+ Some(s"LOG(${childToString(arg)})")
+ case "10" ⇒
+ Some(s"LOG10(${childToString(arg)})")
+ case argStr ⇒
+ Some(s"(LOG(${childToString(arg)})/LOG($argStr))")
+ }
+
+ case ToRadians(child) ⇒
+ Some(s"RADIANS(${childToString(child)})")
+
+ case Sqrt(child) ⇒
+ Some(s"SQRT(${childToString(child)})")
+
+ case _: Pi ⇒
+ Some("PI()")
+
+ case _: EulerNumber ⇒
+ Some("E()")
+
+ case Pow(left, right) ⇒
+ Some(s"POWER(${childToString(left)}, ${childToString(right)})")
+
+ case Rand(child) ⇒
+ Some(s"RAND(${childToString(child)})")
+
+ case Round(child, scale) ⇒
+ Some(s"ROUND(${childToString(child)}, ${childToString(scale)})")
+
+ case Signum(child) ⇒
+ Some(s"SIGN(${childToString(child)})")
+
+ case Remainder(left, right) ⇒
+ Some(s"${childToString(left)} % ${childToString(right)}")
+
+ case Divide(left, right) ⇒
+ Some(s"${childToString(left)} / ${childToString(right)}")
+
+ case Multiply(left, right) ⇒
+ Some(s"${childToString(left)} * ${childToString(right)}")
+
+ case Subtract(left, right) ⇒
+ Some(s"${childToString(left)} - ${childToString(right)}")
+
+ case Add(left, right) ⇒
+ Some(s"${childToString(left)} + ${childToString(right)}")
+
+ case UnaryMinus(child) ⇒
+ Some(s"-${childToString(child)}")
+
+ case UnaryPositive(child) ⇒
+ Some(s"+${childToString(child)}")
+
+ case _ ⇒
+ None
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
new file mode 100644
index 0000000..a1c9458
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
@@ -0,0 +1,180 @@
+package org.apache.ignite.spark.impl.optimization
+
+import java.text.SimpleDateFormat
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, _}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.types._
+
+/**
+ * Object to support some 'simple' expressions like aliases.
+ */
+private[optimization] object SimpleExpressions extends SupportedExpressions {
+ /** @inheritdoc */
+ override def apply(expr: Expression, checkChild: Expression ⇒ Boolean): Boolean = expr match {
+ case Literal(_, _) ⇒
+ true
+
+ case _: Attribute ⇒
+ true
+
+ case Alias(child, _) ⇒
+ checkChild(child)
+
+ case Cast(child, dataType, _) ⇒
+ checkChild(child) && castSupported(from = child.dataType, to = dataType)
+
+ case _ ⇒
+ false
+ }
+
+ /** @inheritdoc */
+ override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+ useAlias: Boolean): Option[String] = expr match {
+ case l: Literal ⇒ l.dataType match {
+ case StringType ⇒
+ Some("'" + l.value.toString + "'")
+
+ case TimestampType ⇒
+ l.value match {
+ //Internal representation of TimestampType is Long.
+ //So we converting from internal spark representation to CAST call.
+ case date: Long ⇒
+ Some(s"CAST('${timestampFormat.get.format(DateTimeUtils.toJavaTimestamp(date))}' AS TIMESTAMP)")
+
+ case _ ⇒
+ Some(l.value.toString)
+ }
+
+ case DateType ⇒
+ l.value match {
+ //Internal representation of DateType is Int.
+ //So we converting from internal spark representation to CAST call.
+ case days: Integer ⇒
+ val date = new java.util.Date(DateTimeUtils.daysToMillis(days))
+
+ Some(s"CAST('${dateFormat.get.format(date)}' AS DATE)")
+
+ case _ ⇒
+ Some(l.value.toString)
+ }
+
+ case _ ⇒
+ if (l.value == null)
+ Some("null")
+ else
+ Some(l.value.toString)
+ }
+
+ case ar: AttributeReference ⇒
+ val name =
+ if (useQualifier)
+ ar.qualifier.map(_ + "." + ar.name).getOrElse(ar.name)
+ else
+ ar.name
+
+ if (ar.metadata.contains(ALIAS) && !isAliasEqualColumnName(ar.metadata.getString(ALIAS), ar.name) && useAlias)
+ Some(aliasToString(name, ar.metadata.getString(ALIAS)))
+ else
+ Some(name)
+
+ case Alias(child, name) ⇒
+ if (useAlias)
+ Some(childToString(child)).map(aliasToString(_, name))
+ else
+ Some(childToString(child))
+
+ case Cast(child, dataType, _) ⇒
+ Some(s"CAST(${childToString(child)} AS ${toSqlType(dataType)})")
+
+ case SortOrder(child, direction, _, _) ⇒
+ Some(s"${childToString(child)}${if(direction==Descending) " DESC" else ""}")
+
+ case _ ⇒
+ None
+ }
+
+ /**
+ * @param column Column name.
+ * @param alias Alias.
+ * @return SQL String for column with alias.
+ */
+ private def aliasToString(column: String, alias: String): String =
+ if (isAliasEqualColumnName(alias, column))
+ column
+ else if (alias.matches("[A-Za-z_][0-9A-Za-z_]*"))
+ s"$column AS $alias"
+ else
+ s"""$column AS "$alias""""
+
+ /**
+ * @param alias Alias.
+ * @param column Column.
+ * @return True if name equals to alias, false otherwise.
+ */
+ private def isAliasEqualColumnName(alias: String, column: String): Boolean =
+ alias.compareToIgnoreCase(column.replaceAll("'", "")) == 0
+
+ /**
+ * @param from From type conversion.
+ * @param to To type conversion.
+ * @return True if cast support for types, false otherwise.
+ */
+ private def castSupported(from: DataType, to: DataType): Boolean = from match {
+ case BooleanType ⇒
+ Set[DataType](BooleanType, StringType)(to)
+
+ case ByteType ⇒
+ Set(ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType, StringType, DecimalType(_, _), StringType)(to)
+
+ case ShortType ⇒
+ Set(ShortType, IntegerType, LongType, FloatType, DoubleType, StringType, DecimalType(_, _))(to)
+
+ case IntegerType ⇒
+ Set(IntegerType, LongType, FloatType, DoubleType, StringType, DecimalType(_, _))(to)
+
+ case LongType ⇒
+ Set(LongType, FloatType, DoubleType, StringType, DecimalType(_, _))(to)
+
+ case FloatType ⇒
+ Set(FloatType, DoubleType, StringType, DecimalType(_, _))(to)
+
+ case DoubleType ⇒
+ Set(DoubleType, StringType, DecimalType(_, _))(to)
+
+ case DecimalType() ⇒
+ Set(StringType, DecimalType(_, _))(to)
+
+ case DateType ⇒
+ Set[DataType](DateType, StringType, LongType, TimestampType)(to)
+
+ case TimestampType ⇒
+ Set[DataType](TimestampType, DateType, StringType, LongType)(to)
+
+ case StringType ⇒
+ Set(BooleanType, ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType,
+ DecimalType(_, _), DateType, TimestampType, StringType)(to)
+
+ case BinaryType ⇒
+ false
+
+ case ArrayType(_, _) ⇒
+ false
+ }
+
+ /**
+ * Date format built-in Ignite.
+ */
+ private val dateFormat: ThreadLocal[SimpleDateFormat] = new ThreadLocal[SimpleDateFormat] {
+ override def initialValue(): SimpleDateFormat =
+ new SimpleDateFormat("yyyy-MM-dd")
+ }
+
+ /**
+ * Timestamp format built-in Ignite.
+ */
+ private val timestampFormat: ThreadLocal[SimpleDateFormat] = new ThreadLocal[SimpleDateFormat] {
+ override def initialValue(): SimpleDateFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS")
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/StringExpressions.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/StringExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/StringExpressions.scala
new file mode 100644
index 0000000..1ecab2c
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/StringExpressions.scala
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, _}
+
+/**
+ * Object to support expressions to work with strings like `length` or `trim`.
+ */
+private[optimization] object StringExpressions extends SupportedExpressions {
+ /** @inheritdoc */
+ def apply(expr: Expression, checkChild: (Expression) ⇒ Boolean): Boolean = expr match {
+ case Ascii(child) ⇒
+ checkChild(child)
+
+ case Length(child) ⇒
+ checkChild(child)
+
+ case Concat(children) ⇒
+ children.forall(checkChild)
+
+ case ConcatWs(children) ⇒
+ children.forall(checkChild)
+
+ case StringInstr(str, substr) ⇒
+ checkChild(str) && checkChild(substr)
+
+ case Lower(child) ⇒
+ checkChild(child)
+
+ case Upper(child) ⇒
+ checkChild(child)
+
+ case StringLocate(substr, str, start) ⇒
+ checkChild(substr) && checkChild(str) && checkChild(start)
+
+ case StringLPad(str, len, pad) ⇒
+ checkChild(str) && checkChild(len) && checkChild(pad)
+
+ case StringRPad(str, len, pad) ⇒
+ checkChild(str) && checkChild(len) && checkChild(pad)
+
+ case StringTrimLeft(child) ⇒
+ checkChild(child)
+
+ case StringTrimRight(child) ⇒
+ checkChild(child)
+
+ case StringTrim(child) ⇒
+ checkChild(child)
+
+ case RegExpReplace(subject, regexp, rep) ⇒
+ checkChild(subject) && checkChild(regexp) && checkChild(rep)
+
+ case StringRepeat(str, times) ⇒
+ checkChild(str) && checkChild(times)
+
+ case SoundEx(child) ⇒
+ checkChild(child)
+
+ case StringSpace(child) ⇒
+ checkChild(child)
+
+ case Substring(str, pos, len) ⇒
+ checkChild(str) && checkChild(pos) && checkChild(len)
+
+ case Substring(str, pos, len) ⇒
+ checkChild(str) && checkChild(pos) && checkChild(len)
+
+ case StringTranslate(str, strMatch, strReplace) ⇒
+ checkChild(str) && checkChild(strMatch) && checkChild(strReplace)
+
+ case _ ⇒ false
+ }
+
+ /** @inheritdoc */
+ override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+ useAlias: Boolean): Option[String] = expr match {
+ case Ascii(child) ⇒
+ Some(s"ASCII(${childToString(child)})")
+
+ case Length(child) ⇒
+ Some(s"CAST(LENGTH(${childToString(child)}) AS INTEGER)")
+
+ case Concat(children) ⇒
+ Some(s"CONCAT(${children.map(childToString(_)).mkString(", ")})")
+
+ case ConcatWs(children) ⇒
+ Some(s"CONCAT_WS(${children.map(childToString(_)).mkString(", ")})")
+
+ case StringInstr(str, substr) ⇒
+ Some(s"POSITION(${childToString(substr)}, ${childToString(str)})")
+
+ case Lower(child) ⇒
+ Some(s"LOWER(${childToString(child)})")
+
+ case Upper(child) ⇒
+ Some(s"UPPER(${childToString(child)})")
+
+ case StringLocate(substr, str, start) ⇒
+ Some(s"LOCATE(${childToString(substr)}, ${childToString(str)}, ${childToString(start)})")
+
+ case StringLPad(str, len, pad) ⇒
+ Some(s"LPAD(${childToString(str)}, ${childToString(len)}, ${childToString(pad)})")
+
+ case StringRPad(str, len, pad) ⇒
+ Some(s"RPAD(${childToString(str)}, ${childToString(len)}, ${childToString(pad)})")
+
+ case StringTrimLeft(child) ⇒
+ Some(s"LTRIM(${childToString(child)})")
+
+ case StringTrimRight(child) ⇒
+ Some(s"RTRIM(${childToString(child)})")
+
+ case StringTrim(child) ⇒
+ Some(s"TRIM(${childToString(child)})")
+
+ case RegExpReplace(subject, regexp, rep) ⇒
+ Some(s"REGEXP_REPLACE(${childToString(subject)}, ${childToString(regexp)}, ${childToString(rep)})")
+
+ case StringRepeat(str, times) ⇒
+ Some(s"REPEAT(${childToString(str)}, ${childToString(times)})")
+
+ case SoundEx(child) ⇒
+ Some(s"SOUND_EX(${childToString(child)})")
+
+ case StringSpace(child) ⇒
+ Some(s"SPACE(${childToString(child)})")
+
+ case Substring(str, pos, len) ⇒
+ Some(s"SUBSTR(${childToString(str)}, ${childToString(pos)}, ${childToString(len)})")
+
+ case StringTranslate(str, strMatch, strReplace) ⇒
+ Some(s"TRANSLATE(${childToString(str)}, ${childToString(strMatch)}, ${childToString(strReplace)})")
+
+ case _ ⇒
+ None
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SupportedExpressions.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SupportedExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SupportedExpressions.scala
new file mode 100644
index 0000000..f46eb72
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SupportedExpressions.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+/**
+ * Provides methods to work with Spark SQL expression that supported by Ignite SQL syntax.
+ */
+private[optimization] trait SupportedExpressions {
+ /**
+ * @param expr Expression to check.
+ * @param checkChild Closure to check child expression.
+ * @return True if `expr` are supported, false otherwise.
+ */
+ def apply(expr: Expression, checkChild: (Expression) ⇒ Boolean): Boolean
+
+ /**
+ * @param expr Expression to convert to string.
+ * @param childToString Closure to convert children expressions.
+ * @param useQualifier If true `expr` should be printed using qualifier. `Table1.id` for example.
+ * @param useAlias If true `expr` should be printed with alias. `name as person_name` for example.
+ * @return SQL representation of `expr` if it supported. `None` otherwise.
+ */
+ def toString(expr: Expression, childToString: (Expression) ⇒ String, useQualifier: Boolean,
+ useAlias: Boolean): Option[String]
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SystemExpressions.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SystemExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SystemExpressions.scala
new file mode 100644
index 0000000..40e4e29
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SystemExpressions.scala
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.ignite.IgniteException
+import org.apache.spark.sql.catalyst.expressions.{Coalesce, EqualTo, Expression, Greatest, If, IfNull, IsNotNull, IsNull, Least, Literal, NullIf, Nvl2}
+
+/**
+ * Object to support some built-in expressions like `nvl2` or `coalesce`.
+ */
+private[optimization] object SystemExpressions extends SupportedExpressions {
+ /** @inheritdoc */
+ override def apply(expr: Expression, checkChild: Expression ⇒ Boolean): Boolean = expr match {
+ case Coalesce(children) ⇒
+ children.forall(checkChild)
+
+ case Greatest(children) ⇒
+ children.forall(checkChild)
+
+ case IfNull(left, right, _) ⇒
+ checkChild(left) && checkChild(right)
+
+ case Least(children) ⇒
+ children.forall(checkChild)
+
+ case NullIf(left, right, _) ⇒
+ checkChild(left) && checkChild(right)
+
+ case Nvl2(expr1, expr2, expr3, _) ⇒
+ checkChild(expr1) && checkChild(expr2) && checkChild(expr3)
+
+ case If(predicate, trueValue, falseValue) ⇒
+ predicate match {
+ case IsNotNull(child) ⇒
+ checkChild(child) && checkChild(trueValue) && checkChild(falseValue)
+
+ case IsNull(child) ⇒
+ checkChild(child) && checkChild(trueValue) && checkChild(falseValue)
+
+ case EqualTo(left, right) ⇒
+ trueValue match {
+ case Literal(null, _) ⇒
+ (left == falseValue || right == falseValue) && checkChild(left) && checkChild(right)
+
+ case _ ⇒
+ false
+ }
+
+ case _ ⇒
+ false
+ }
+
+ case _ ⇒
+ false
+ }
+
+ /** @inheritdoc */
+ override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+ useAlias: Boolean): Option[String] = expr match {
+ case Coalesce(children) ⇒
+ Some(s"COALESCE(${children.map(childToString(_)).mkString(", ")})")
+
+ case Greatest(children) ⇒
+ Some(s"GREATEST(${children.map(childToString(_)).mkString(", ")})")
+
+ case IfNull(left, right, _) ⇒
+ Some(s"IFNULL(${childToString(left)}, ${childToString(right)})")
+
+ case Least(children) ⇒
+ Some(s"LEAST(${children.map(childToString(_)).mkString(", ")})")
+
+ case NullIf(left, right, _) ⇒
+ Some(s"NULLIF(${childToString(left)}, ${childToString(right)})")
+
+ case Nvl2(expr1, expr2, expr3, _) ⇒
+ Some(s"NVL2(${childToString(expr1)}, ${childToString(expr2)}, ${childToString(expr3)})")
+
+ case If(predicate, trueValue, falseValue) ⇒
+ predicate match {
+ case IsNotNull(child) ⇒
+ Some(s"NVL2(${childToString(child)}, ${childToString(trueValue)}, ${childToString(falseValue)})")
+
+ case IsNull(child) ⇒
+ Some(s"NVL2(${childToString(child)}, ${childToString(falseValue)}, ${childToString(trueValue)})")
+
+ case EqualTo(left, right) ⇒
+ trueValue match {
+ case Literal(null, _) ⇒
+ if (left == falseValue)
+ Some(s"NULLIF(${childToString(left)}, ${childToString(right)})")
+ else if (right == falseValue)
+ Some(s"NULLIF(${childToString(right)}, ${childToString(left)})")
+ else
+ throw new IgniteException(s"Expression not supported. $expr")
+
+ case _ ⇒
+ throw new IgniteException(s"Expression not supported. $expr")
+ }
+
+ case _ ⇒
+ throw new IgniteException(s"Expression not supported. $expr")
+ }
+
+ case _ ⇒
+ None
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/JoinSQLAccumulator.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/JoinSQLAccumulator.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/JoinSQLAccumulator.scala
new file mode 100644
index 0000000..7ae5e70
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/JoinSQLAccumulator.scala
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization.accumulator
+
+import org.apache.ignite.IgniteException
+import org.apache.ignite.spark.impl.optimization._
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, NamedExpression, SortOrder}
+import org.apache.spark.sql.catalyst.plans.logical.BinaryNode
+import org.apache.spark.sql.catalyst.plans.{Inner, JoinType, LeftOuter, RightOuter}
+
+/**
+ * Accumulator to store information about join query.
+ */
+private[apache] case class JoinSQLAccumulator(
+ igniteQueryContext: IgniteQueryContext,
+ left: QueryAccumulator,
+ right: QueryAccumulator,
+ joinType: JoinType,
+ outputExpressions: Seq[NamedExpression],
+ condition: Option[Expression],
+ leftAlias: Option[String],
+ rightAlias: Option[String],
+ distinct: Boolean = false,
+ where: Option[Seq[Expression]] = None,
+ groupBy: Option[Seq[Expression]] = None,
+ having: Option[Seq[Expression]] = None,
+ limit: Option[Expression] = None,
+ localLimit: Option[Expression] = None,
+ orderBy: Option[Seq[SortOrder]] = None
+) extends BinaryNode with SelectAccumulator {
+ /** @inheritdoc */
+ override def compileQuery(prettyPrint: Boolean = false): String = {
+ val delim = if (prettyPrint) "\n" else " "
+ val tab = if (prettyPrint) " " else ""
+
+ var sql = s"SELECT$delim$tab" +
+ s"${fixQualifier(outputExpressions).map(exprToString(_, useQualifier = true)).mkString(", ")}$delim" +
+ s"FROM$delim$tab$compileJoinExpr"
+
+ if (allFilters.nonEmpty)
+ sql += s"${delim}WHERE$delim$tab" +
+ s"${fixQualifier(allFilters).map(exprToString(_, useQualifier = true)).mkString(s" AND$delim$tab")}"
+
+ if (groupBy.exists(_.nonEmpty))
+ sql += s"${delim}GROUP BY " +
+ s"${fixQualifier(groupBy.get).map(exprToString(_, useQualifier = true)).mkString(s",$delim$tab")}"
+
+ if (having.exists(_.nonEmpty))
+ sql += s"${delim}HAVING " +
+ s"${fixQualifier(having.get).map(exprToString(_, useQualifier = true)).mkString(s" AND$delim$tab")}"
+
+ if (orderBy.exists(_.nonEmpty))
+ sql += s"${delim}ORDER BY " +
+ s"${fixQualifier(orderBy.get).map(exprToString(_, useQualifier = true)).mkString(s",$delim$tab")}"
+
+ if (limit.isDefined)
+ sql += s" LIMIT ${exprToString(fixQualifier0(limit.get), useQualifier = true)}"
+
+ sql
+ }
+
+ /**
+ * @return Filters for this query.
+ */
+ private def allFilters: Seq[Expression] = {
+ val leftFilters =
+ if (isSimpleTableAcc(left))
+ left.asInstanceOf[SingleTableSQLAccumulator].where.getOrElse(Seq.empty)
+ else
+ Seq.empty
+
+ val rightFilters =
+ if (isSimpleTableAcc(right))
+ right.asInstanceOf[SingleTableSQLAccumulator].where.getOrElse(Seq.empty)
+ else Seq.empty
+
+ where.getOrElse(Seq.empty) ++ leftFilters ++ rightFilters
+ }
+
+ /**
+ * @return `table1 LEFT JOIN (SELECT....FROM...) table2` part of join query.
+ */
+ private def compileJoinExpr: String = {
+ val leftJoinSql =
+ if (isSimpleTableAcc(left))
+ left.asInstanceOf[SingleTableSQLAccumulator].table.get
+ else
+ s"(${left.compileQuery()}) ${leftAlias.get}"
+
+ val rightJoinSql = {
+ val leftTableName =
+ if (isSimpleTableAcc(left))
+ left.qualifier
+ else
+ leftAlias.get
+
+ if (isSimpleTableAcc(right)) {
+ val rightTableName = right.asInstanceOf[SingleTableSQLAccumulator].table.get
+
+ if (leftTableName == rightTableName)
+ s"$rightTableName as ${rightAlias.get}"
+ else
+ rightTableName
+ } else
+ s"(${right.compileQuery()}) ${rightAlias.get}"
+ }
+
+ s"$leftJoinSql $joinTypeSQL $rightJoinSql" +
+ s"${condition.map(expr ⇒ s" ON ${exprToString(fixQualifier0(expr), useQualifier = true)}").getOrElse("")}"
+ }
+
+ /**
+ * @return SQL string representing specific join type.
+ */
+ private def joinTypeSQL = joinType match {
+ case Inner ⇒
+ "JOIN"
+ case LeftOuter ⇒
+ "LEFT JOIN"
+
+ case RightOuter ⇒
+ "RIGHT JOIN"
+
+ case _ ⇒
+ throw new IgniteException(s"Unsupported join type $joinType")
+ }
+
+ /**
+ * Changes table qualifier in case of embedded query.
+ *
+ * @param exprs Expressions to fix.
+ * @tparam T type of input expression.
+ * @return copy of `exprs` with fixed qualifier.
+ */
+ private def fixQualifier[T <: Expression](exprs: Seq[T]): Seq[T] =
+ exprs.map(fixQualifier0)
+
+ /**
+ * Changes table qualifier for single expression.
+ *
+ * @param expr Expression to fix.
+ * @tparam T type of input expression.
+ * @return copy of `expr` with fixed qualifier.
+ */
+ private def fixQualifier0[T <: Expression](expr: T): T = expr match {
+ case attr: AttributeReference ⇒
+ attr.withQualifier(Some(findQualifier(attr))).asInstanceOf[T]
+
+ case _ ⇒
+ expr.withNewChildren(fixQualifier(expr.children)).asInstanceOf[T]
+ }
+
+ /**
+ * Find right qualifier for a `attr`.
+ *
+ * @param attr Attribute to fix qualifier in
+ * @return Right qualifier for a `attr`
+ */
+ private def findQualifier(attr: AttributeReference): String = {
+ val leftTableName =
+ if (isSimpleTableAcc(left))
+ left.qualifier
+ else
+ leftAlias.get
+
+ if (left.outputExpressions.exists(_.exprId == attr.exprId))
+ leftTableName
+ else if (isSimpleTableAcc(right) && right.qualifier != leftTableName)
+ right.qualifier
+ else
+ rightAlias.get
+ }
+
+ /** @inheritdoc */
+ override def simpleString: String =
+ s"JoinSQLAccumulator(joinType: $joinType, columns: $outputExpressions, condition: $condition)"
+
+ /** @inheritdoc */
+ override def withOutputExpressions(outputExpressions: Seq[NamedExpression]): SelectAccumulator = copy(outputExpressions= outputExpressions)
+
+ /** @inheritdoc */
+ override def withDistinct(distinct: Boolean): JoinSQLAccumulator = copy(distinct = distinct)
+
+ /** @inheritdoc */
+ override def withWhere(where: Seq[Expression]): JoinSQLAccumulator = copy(where = Some(where))
+
+ /** @inheritdoc */
+ override def withGroupBy(groupBy: Seq[Expression]): JoinSQLAccumulator = copy(groupBy = Some(groupBy))
+
+ /** @inheritdoc */
+ override def withHaving(having: Seq[Expression]): JoinSQLAccumulator = copy(having = Some(having))
+
+ /** @inheritdoc */
+ override def withLimit(limit: Expression): JoinSQLAccumulator = copy(limit = Some(limit))
+
+ /** @inheritdoc */
+ override def withLocalLimit(localLimit: Expression): JoinSQLAccumulator = copy(localLimit = Some(localLimit))
+
+ /** @inheritdoc */
+ override def withOrderBy(orderBy: Seq[SortOrder]): JoinSQLAccumulator = copy(orderBy = Some(orderBy))
+
+ /** @inheritdoc */
+ override def output: Seq[Attribute] = outputExpressions.map(toAttributeReference(_, Seq.empty))
+
+ /** @inheritdoc */
+ override lazy val qualifier: String = igniteQueryContext.uniqueTableAlias
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/QueryAccumulator.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/QueryAccumulator.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/QueryAccumulator.scala
new file mode 100644
index 0000000..133d355
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/QueryAccumulator.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization.accumulator
+
+import org.apache.ignite.spark.impl.optimization.IgniteQueryContext
+import org.apache.spark.sql.catalyst.expressions.{NamedExpression, SortOrder}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+/**
+ * Generic query info accumulator interface.
+ */
+private[apache] trait QueryAccumulator extends LogicalPlan {
+ /**
+ * @return Ignite query context.
+ */
+ def igniteQueryContext: IgniteQueryContext
+
+ /**
+ * @return Generated output.
+ */
+ def outputExpressions: Seq[NamedExpression]
+
+ /**
+ * @return Ordering info.
+ */
+ def orderBy: Option[Seq[SortOrder]]
+
+ /**
+ * @param outputExpressions New output expressions.
+ * @return Copy of this accumulator with new output.
+ */
+ def withOutputExpressions(outputExpressions: Seq[NamedExpression]): QueryAccumulator
+
+ /**
+ * @param orderBy New ordering.
+ * @return Copy of this accumulator with new order.
+ */
+ def withOrderBy(orderBy: Seq[SortOrder]): QueryAccumulator
+
+ /**
+ * @param prettyPrint If true human readable query will be generated.
+ * @return SQL query.
+ */
+ def compileQuery(prettyPrint: Boolean = false): String
+
+ /**
+ * @return Qualifier that should be use to select data from this accumulator.
+ */
+ def qualifier: String
+
+ /**
+ * All expressions are resolved when extra optimization executed.
+ */
+ override lazy val resolved = true
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SelectAccumulator.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SelectAccumulator.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SelectAccumulator.scala
new file mode 100644
index 0000000..c1db6f9
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SelectAccumulator.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization.accumulator
+
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+/**
+ * Generic interface for a SELECT query.
+ */
+private[apache] trait SelectAccumulator extends QueryAccumulator {
+ /**
+ * @return Expression for HAVING part of query.
+ */
+ def having: Option[Seq[Expression]]
+
+ /**
+ * @return Expression for WHERE part of query.
+ */
+ def where: Option[Seq[Expression]]
+
+ /**
+ * @return Expression for GROUP BY part of query.
+ */
+ def groupBy: Option[Seq[Expression]]
+
+ /**
+ * @return Copy of this accumulator with `distinct` flag.
+ */
+ def withDistinct(distinct: Boolean): SelectAccumulator
+
+ /**
+ * @return Copy of this accumulator with `where` expressions.
+ */
+ def withWhere(where: Seq[Expression]): SelectAccumulator
+
+ /**
+ * @return Copy of this accumulator with `groupBy` expressions.
+ */
+ def withGroupBy(groupBy: Seq[Expression]): SelectAccumulator
+
+ /**
+ * @return Copy of this accumulator with `having` expressions.
+ */
+ def withHaving(having: Seq[Expression]): SelectAccumulator
+
+ /**
+ * @return Copy of this accumulator with `limit` expression.
+ */
+ def withLimit(limit: Expression): SelectAccumulator
+
+ /**
+ * @return Copy of this accumulator with `localLimit` expression.
+ */
+ def withLocalLimit(localLimit: Expression): SelectAccumulator
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SingleTableSQLAccumulator.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SingleTableSQLAccumulator.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SingleTableSQLAccumulator.scala
new file mode 100644
index 0000000..47035b9
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SingleTableSQLAccumulator.scala
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization.accumulator
+
+import org.apache.ignite.IgniteException
+import org.apache.ignite.spark.impl.optimization._
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, NamedExpression, SortOrder}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+/**
+ * Class for accumulating parts of SQL query to a single Ignite table.
+ *
+ * See <a href="http://www.h2database.com/html/grammar.html#select">select syntax of H2</a>.
+ */
+private[apache] case class SingleTableSQLAccumulator(
+ igniteQueryContext: IgniteQueryContext,
+ table: Option[String],
+ tableExpression: Option[(QueryAccumulator, String)],
+ outputExpressions: Seq[NamedExpression],
+ distinct: Boolean = false,
+ all: Boolean = false,
+ where: Option[Seq[Expression]] = None,
+ groupBy: Option[Seq[Expression]] = None,
+ having: Option[Seq[Expression]] = None,
+ limit: Option[Expression] = None,
+ localLimit: Option[Expression] = None,
+ orderBy: Option[Seq[SortOrder]] = None
+) extends SelectAccumulator {
+ /** @inheritdoc */
+ override def compileQuery(prettyPrint: Boolean = false): String = {
+ val delim = if (prettyPrint) "\n" else " "
+ val tab = if (prettyPrint) " " else ""
+
+ var sql = s"SELECT$delim$tab${outputExpressions.map(exprToString(_)).mkString(", ")}${delim}" +
+ s"FROM$delim$tab$compiledTableExpression"
+
+ if (where.exists(_.nonEmpty))
+ sql += s"${delim}WHERE$delim$tab${where.get.map(exprToString(_)).mkString(s" AND$delim$tab")}"
+
+ if (groupBy.exists(_.nonEmpty))
+ sql += s"${delim}GROUP BY ${groupBy.get.map(exprToString(_)).mkString(s",$delim$tab")}"
+
+ if (having.exists(_.nonEmpty))
+ sql += s"${delim}HAVING ${having.get.map(exprToString(_)).mkString(s" AND$delim$tab")}"
+
+ if (orderBy.exists(_.nonEmpty))
+ sql += s"${delim}ORDER BY ${orderBy.get.map(exprToString(_)).mkString(s",$delim$tab")}"
+
+ if (limit.isDefined)
+ sql += s" LIMIT ${limit.map(exprToString(_)).get}"
+
+ sql
+ }
+
+ /**
+ * @return From table SQL query part.
+ */
+ private def compiledTableExpression: String = table match {
+ case Some(tableName) ⇒
+ tableName
+
+ case None ⇒ tableExpression match {
+ case Some((acc, alias)) ⇒
+ s"(${acc.compileQuery()}) $alias"
+
+ case None ⇒
+ throw new IgniteException("Unknown table.")
+ }
+ }
+
+ /** @inheritdoc */
+ override def simpleString: String =
+ s"IgniteSQLAccumulator(table: $table, columns: $outputExpressions, distinct: $distinct, all: $all, " +
+ s"where: $where, groupBy: $groupBy, having: $having, limit: $limit, orderBy: $orderBy)"
+
+ /** @inheritdoc */
+ override def withOutputExpressions(outputExpressions: Seq[NamedExpression]): SelectAccumulator =
+ copy(outputExpressions= outputExpressions)
+
+ /** @inheritdoc */
+ override def withDistinct(distinct: Boolean): SingleTableSQLAccumulator = copy(distinct = distinct)
+
+ /** @inheritdoc */
+ override def withWhere(where: Seq[Expression]): SingleTableSQLAccumulator = copy(where = Some(where))
+
+ /** @inheritdoc */
+ override def withGroupBy(groupBy: Seq[Expression]): SingleTableSQLAccumulator = copy(groupBy = Some(groupBy))
+
+ /** @inheritdoc */
+ override def withHaving(having: Seq[Expression]): SingleTableSQLAccumulator = copy(having = Some(having))
+
+ /** @inheritdoc */
+ override def withLimit(limit: Expression): SingleTableSQLAccumulator = copy(limit = Some(limit))
+
+ /** @inheritdoc */
+ override def withLocalLimit(localLimit: Expression): SingleTableSQLAccumulator = copy(localLimit = Some(localLimit))
+
+ /** @inheritdoc */
+ override def withOrderBy(orderBy: Seq[SortOrder]): SingleTableSQLAccumulator = copy(orderBy = Some(orderBy))
+
+ /** @inheritdoc */
+ override def output: Seq[Attribute] = outputExpressions.map(toAttributeReference(_, Seq.empty))
+
+ /** @inheritdoc */
+ override def qualifier: String = table.getOrElse(tableExpression.get._2)
+
+ /** @inheritdoc */
+ override def children: Seq[LogicalPlan] = tableExpression.map(te ⇒ Seq(te._1)).getOrElse(Nil)
+}
[10/50] [abbrv] ignite git commit: IGNITE-8021 Delete cache config
files when cache is destroyed - Fixes #3697.
Posted by ag...@apache.org.
IGNITE-8021 Delete cache config files when cache is destroyed - Fixes #3697.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2edcb22f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2edcb22f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2edcb22f
Branch: refs/heads/ignite-6083
Commit: 2edcb22fbb566981097733af6470ed6dde8e786b
Parents: 1b3a292
Author: Ivan Daschinskiy <iv...@gmail.com>
Authored: Tue Apr 17 18:05:42 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Apr 17 18:05:42 2018 +0300
----------------------------------------------------------------------
.../pagemem/store/IgnitePageStoreManager.java | 9 +
.../processors/cache/GridCacheProcessor.java | 11 +
.../persistence/file/FilePageStoreManager.java | 47 +++
...eConfigurationDataAfterDestroyCacheTest.java | 326 +++++++++++++++++++
.../pagemem/NoOpPageStoreManager.java | 5 +
.../ignite/testsuites/IgnitePdsTestSuite.java | 2 +
6 files changed, 400 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2edcb22f/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index 1b46bf9..0fc9f94 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -193,6 +193,15 @@ public interface IgnitePageStoreManager extends GridCacheSharedManager, IgniteCh
* @throws IgniteCheckedException If failed.
*/
public void storeCacheData(StoredCacheData cacheData, boolean overwrite) throws IgniteCheckedException;
+
+ /**
+ * Remove cache configuration data file.
+ *
+ * @param cacheData Cache configuration.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void removeCacheData(StoredCacheData cacheData) throws IgniteCheckedException;
+
/**
* @param grpId Cache group ID.
* @return {@code True} if index store for given cache group existed before node started.
http://git-wip-us.apache.org/repos/asf/ignite/blob/2edcb22f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 36edd72..bceb8c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1284,6 +1284,17 @@ public class GridCacheProcessor extends GridProcessorAdapter {
U.stopLifecycleAware(log, lifecycleAwares(ctx.group(), cache.configuration(), ctx.store().configuredStore()));
+ IgnitePageStoreManager pageStore;
+
+ if (destroy && (pageStore = sharedCtx.pageStore()) != null) {
+ try {
+ pageStore.removeCacheData(new StoredCacheData(ctx.config()));
+ } catch (IgniteCheckedException e) {
+ U.error(log, "Failed to delete cache configuration data while destroying cache" +
+ "[cache=" + ctx.name() + "]", e);
+ }
+ }
+
if (log.isInfoEnabled()) {
if (ctx.group().sharedGroup())
log.info("Stopped cache [cacheName=" + cache.name() + ", group=" + ctx.group().name() + ']');
http://git-wip-us.apache.org/repos/asf/ignite/blob/2edcb22f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 6313eac..837f3d0 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
+import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
@@ -281,6 +282,9 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
IgniteCheckedException ex = shutdown(old, /*clean files if destroy*/destroy, null);
+ if (destroy)
+ removeCacheGroupConfigurationData(grp);
+
if (ex != null)
throw ex;
}
@@ -746,6 +750,49 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
}
/**
+ * Delete caches' configuration data files of cache group.
+ *
+ * @param ctx Cache group context.
+ * @throws IgniteCheckedException If fails.
+ */
+ private void removeCacheGroupConfigurationData(CacheGroupContext ctx) throws IgniteCheckedException {
+ File cacheGrpDir = cacheWorkDir(ctx.sharedGroup(), ctx.cacheOrGroupName());
+
+ if (cacheGrpDir != null && cacheGrpDir.exists()) {
+ DirectoryStream.Filter<Path> cacheCfgFileFilter = new DirectoryStream.Filter<Path>() {
+ @Override public boolean accept(Path path) {
+ return Files.isRegularFile(path) && path.getFileName().toString().endsWith(CACHE_DATA_FILENAME);
+ }
+ };
+
+ try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(cacheGrpDir.toPath(), cacheCfgFileFilter)) {
+ for(Path path: dirStream)
+ Files.deleteIfExists(path);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to delete cache configurations of group: " + ctx.toString(), e);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void removeCacheData(StoredCacheData cacheData) throws IgniteCheckedException {
+ CacheConfiguration cacheCfg = cacheData.config();
+ File cacheWorkDir = cacheWorkDir(cacheCfg);
+ File file;
+
+ if (cacheData.config().getGroupName() != null)
+ file = new File(cacheWorkDir, cacheCfg.getName() + CACHE_DATA_FILENAME);
+ else
+ file = new File(cacheWorkDir, CACHE_DATA_FILENAME);
+
+ if (file.exists()) {
+ if (!file.delete())
+ throw new IgniteCheckedException("Failed to delete cache configuration:" + cacheCfg.getName());
+ }
+ }
+
+ /**
* @param store Store to shutdown.
* @param cleanFile {@code True} if files should be cleaned.
* @param aggr Aggregating exception.
http://git-wip-us.apache.org/repos/asf/ignite/blob/2edcb22f/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java
new file mode 100644
index 0000000..d2767d4
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test correct clean up cache configuration data after destroying cache.
+ */
+public class IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final int CACHES = 3;
+
+ /** */
+ private static final int NODES = 3;
+
+ /** */
+ private static final int NUM_OF_KEYS = 100;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ return cfg.setDiscoverySpi(new TcpDiscoverySpi()
+ .setIpFinder(IP_FINDER))
+ .setDataStorageConfiguration(new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setMaxSize(200 * 1024 * 1024)
+ .setPersistenceEnabled(true)));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+
+ super.afterTest();
+ }
+
+ /**
+ * {@inheritDoc}
+ * @returns always {@code true} in order to be able to kill nodes when checkpointer thread hangs.
+ */
+ @Override protected boolean isMultiJvm() {
+ return true;
+ }
+
+ /**
+ * Test destroy non grouped caches.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDestroyCaches() throws Exception {
+ Ignite ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ startCachesDynamically(ignite);
+
+ checkDestroyCaches(ignite);
+ }
+
+ /**
+ * Test destroy grouped caches.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDestroyGroupCaches() throws Exception {
+ Ignite ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ startGroupCachesDynamically(ignite);
+
+ checkDestroyCaches(ignite);
+ }
+
+ /**
+ * Test destroy caches with disabled checkpoints.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDestroyCachesAbruptlyWithoutCheckpoints() throws Exception {
+ Ignite ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ startCachesDynamically(ignite);
+
+ enableCheckpoints(false);
+
+ checkDestroyCachesAbruptly(ignite);
+ }
+
+ /**
+ * Test destroy group caches with disabled checkpoints.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDestroyGroupCachesAbruptlyWithoutCheckpoints() throws Exception {
+ Ignite ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ startGroupCachesDynamically(ignite);
+
+ enableCheckpoints(false);
+
+ checkDestroyCachesAbruptly(ignite);
+ }
+
+ /**
+ * Test destroy caches abruptly with checkpoints.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDestroyCachesAbruptly() throws Exception {
+ Ignite ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ startCachesDynamically(ignite);
+
+ checkDestroyCachesAbruptly(ignite);
+ }
+
+
+ /**
+ * Test destroy group caches abruptly with checkpoints.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDestroyGroupCachesAbruptly() throws Exception {
+ Ignite ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ startGroupCachesDynamically(ignite);
+
+ checkDestroyCachesAbruptly(ignite);
+ }
+
+ /**
+ * @param ignite Ignite.
+ */
+ private void loadCaches(Ignite ignite) {
+ for (int i = 0; i < CACHES; i++) {
+ try (IgniteDataStreamer<Object, Object> s = ignite.dataStreamer(cacheName(i))) {
+ s.allowOverwrite(true);
+
+ for (int j = 0; j < NUM_OF_KEYS; j++)
+ s.addData(j, "cache: " + i + " data: " + j);
+
+ s.flush();
+ }
+ }
+ }
+
+ /**
+ * @param ignite Ignite.
+ */
+ private void checkDestroyCaches(Ignite ignite) throws Exception {
+ loadCaches(ignite);
+
+ log.warning("destroying caches....");
+
+ ignite.cache(cacheName(0)).destroy();
+ ignite.cache(cacheName(1)).destroy();
+
+ assertEquals(CACHES - 2, ignite.cacheNames().size());
+
+ log.warning("Stopping grid");
+
+ stopAllGrids();
+
+ log.warning("Grid stopped");
+
+ log.warning("Starting grid");
+
+ ignite = startGrids(NODES);
+
+ log.warning("Grid started");
+
+ assertEquals("Check that caches don't survived", CACHES - 2, ignite.cacheNames().size());
+
+ for(Ignite ig: G.allGrids()) {
+ IgniteCache cache = ig.cache(cacheName(2));
+
+ for (int j = 0; j < NUM_OF_KEYS; j++)
+ assertNotNull("Check that cache2 contains key: " + j + " node: " + ignite.name(), cache.get(j));
+ }
+ }
+
+
+ /**
+ * @param ignite Ignite instance.
+ */
+ private void checkDestroyCachesAbruptly(Ignite ignite) throws Exception {
+ loadCaches(ignite);
+
+ log.warning("Destroying caches");
+
+ ((GatewayProtectedCacheProxy)ignite.cache(cacheName(0))).destroyAsync();
+ ((GatewayProtectedCacheProxy)ignite.cache(cacheName(1))).destroyAsync();
+
+ log.warning("Stopping grid");
+
+ stopAllGrids();
+
+ log.warning("Grid stopped");
+
+ log.warning("Starting grid");
+
+ startGrids(NODES);
+
+ log.warning("Grid started");
+
+ for(Ignite ig: G.allGrids()) {
+ assertTrue(ig.cacheNames().contains(cacheName(2)));
+
+ IgniteCache cache = ig.cache(cacheName(2));
+
+ for (int j = 0; j < NUM_OF_KEYS; j++)
+ assertNotNull("Check that survived cache cache2 contains key: " + j + " node: " + ig.name(), cache.get(j));
+ }
+ }
+
+ /**
+ * @param ignite Ignite.
+ */
+ private void startCachesDynamically(Ignite ignite) {
+ List<CacheConfiguration> ccfg = new ArrayList<>(CACHES);
+
+ for (int i = 0; i < CACHES; i++)
+ ccfg.add(new CacheConfiguration<>(cacheName(i))
+ .setBackups(1)
+ .setAffinity(new RendezvousAffinityFunction(false, 32)));
+
+ ignite.createCaches(ccfg);
+ }
+
+ /**
+ * @param ignite Ignite instance.
+ */
+ private void startGroupCachesDynamically(Ignite ignite) {
+ List<CacheConfiguration> ccfg = new ArrayList<>(CACHES);
+
+ for (int i = 0; i < CACHES; i++)
+ ccfg.add(new CacheConfiguration<>(cacheName(i))
+ .setGroupName(i % 2 == 0 ? "grp-even" : "grp-odd")
+ .setBackups(1)
+ .setAffinity(new RendezvousAffinityFunction(false, 32)));
+
+ ignite.createCaches(ccfg);
+ }
+
+
+ /**
+ * Generate cache name from idx.
+ *
+ * @param idx Index.
+ */
+ private String cacheName(int idx) {
+ return "cache" + idx;
+ }
+
+ /**
+ * Enable/disable checkpoints on multi JVM nodes only.
+ *
+ * @param enabled Enabled flag.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void enableCheckpoints(boolean enabled) throws IgniteCheckedException {
+ for (Ignite ignite : G.allGrids()) {
+ assert !ignite.cluster().localNode().isClient();
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.enableCheckpoints(enabled).get();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/2edcb22f/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
index 64acf02..be40c90 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
@@ -187,6 +187,11 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
}
/** {@inheritDoc} */
+ @Override public void removeCacheData(StoredCacheData cacheData) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Override public boolean hasIndexStore(int grpId) {
return false;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/2edcb22f/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index a9668e7..af0b7ad 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -19,6 +19,7 @@ package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistence;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest;
@@ -113,6 +114,7 @@ public class IgnitePdsTestSuite extends TestSuite {
suite.addTestSuite(IgnitePdsCacheRestoreTest.class);
suite.addTestSuite(IgnitePdsDataRegionMetricsTest.class);
+ suite.addTestSuite(IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.class);
suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
[03/50] [abbrv] ignite git commit: IGNITE-8292: Broken yardstick
compilation.
Posted by ag...@apache.org.
IGNITE-8292: Broken yardstick compilation.
this closes #3840
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3cebf912
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3cebf912
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3cebf912
Branch: refs/heads/ignite-6083
Commit: 3cebf9123ed1161822ece503169b361f49293358
Parents: 8c80dce
Author: YuriBabak <y....@gmail.com>
Authored: Tue Apr 17 11:54:41 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Tue Apr 17 11:54:41 2018 +0300
----------------------------------------------------------------------
...niteKMeansDistributedClustererBenchmark.java | 75 --------------------
.../IgniteKMeansLocalClustererBenchmark.java | 50 -------------
.../yardstick/ml/clustering/package-info.java | 22 ------
3 files changed, 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3cebf912/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansDistributedClustererBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansDistributedClustererBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansDistributedClustererBenchmark.java
deleted file mode 100644
index de928e8..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansDistributedClustererBenchmark.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.clustering;
-
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.clustering.KMeansDistributedClusterer;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-import org.apache.ignite.yardstick.ml.DataChanger;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteKMeansDistributedClustererBenchmark extends IgniteAbstractBenchmark {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public boolean test(Map<Object, Object> ctx) throws Exception {
- final DataChanger.Scale scale = new DataChanger.Scale();
-
- // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
- // because we create ignite cache internally.
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- this.getClass().getSimpleName(), new Runnable() {
- /** {@inheritDoc} */
- @Override public void run() {
- // IMPL NOTE originally taken from KMeansDistributedClustererTest
- KMeansDistributedClusterer clusterer = new KMeansDistributedClusterer(
- new EuclideanDistance(), 1, 1, 1L);
-
- double[] v1 = scale.mutate(new double[] {1959, 325100});
- double[] v2 = scale.mutate(new double[] {1960, 373200});
-
- SparseDistributedMatrix points = new SparseDistributedMatrix(
- 2, 2, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
- points.setRow(0, v1);
- points.setRow(1, v2);
-
- clusterer.cluster(points, 1);
-
- points.destroy();
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
-
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3cebf912/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansLocalClustererBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansLocalClustererBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansLocalClustererBenchmark.java
deleted file mode 100644
index d68fc6d..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansLocalClustererBenchmark.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.clustering;
-
-import java.util.Map;
-import org.apache.ignite.ml.clustering.KMeansLocalClusterer;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-import org.apache.ignite.yardstick.ml.DataChanger;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteKMeansLocalClustererBenchmark extends IgniteAbstractBenchmark {
- /** {@inheritDoc} */
- @Override public boolean test(Map<Object, Object> ctx) throws Exception {
- final DataChanger.Scale scale = new DataChanger.Scale();
-
- // IMPL NOTE originally taken from KMeansLocalClustererTest
- KMeansLocalClusterer clusterer = new KMeansLocalClusterer(new EuclideanDistance(), 1, 1L);
-
- double[] v1 = scale.mutate(new double[] {1959, 325100});
- double[] v2 = scale.mutate(new double[] {1960, 373200});
-
- DenseLocalOnHeapMatrix points = new DenseLocalOnHeapMatrix(new double[][] {
- v1,
- v2});
-
- clusterer.cluster(points, 1);
-
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3cebf912/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/package-info.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/package-info.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/package-info.java
deleted file mode 100644
index af217d2..0000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * ML Grid clustering benchmarks.
- */
-package org.apache.ignite.yardstick.ml.clustering;
\ No newline at end of file
[36/50] [abbrv] ignite git commit: IGNITE-8327 Removed delaying
communication SPI from tests
Posted by ag...@apache.org.
IGNITE-8327 Removed delaying communication SPI from tests
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/631d3a62
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/631d3a62
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/631d3a62
Branch: refs/heads/ignite-6083
Commit: 631d3a621222443cebbc71def35b72e48f0d0115
Parents: 9796ec4
Author: Ilya Lantukh <il...@gridgain.com>
Authored: Thu Apr 19 17:53:26 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Apr 19 17:53:26 2018 +0300
----------------------------------------------------------------------
.../distributed/CacheBaselineTopologyTest.java | 35 +-------------------
1 file changed, 1 insertion(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/631d3a62/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index c3d404b..22cf205 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -46,17 +46,12 @@ import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.TestDelayingCommunicationSpi;
-import org.apache.ignite.internal.managers.communication.GridIoMessage;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
@@ -80,9 +75,6 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
private static final int NODE_COUNT = 4;
/** */
- private boolean delayRebalance;
-
- /** */
private boolean disableAutoActivation;
/** */
@@ -151,16 +143,13 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
if (client)
cfg.setClientMode(true);
- if (delayRebalance)
- cfg.setCommunicationSpi(new DelayRebalanceCommunicationSpi());
-
return cfg;
}
/**
* Verifies that rebalance on cache with Node Filter happens when BaselineTopology changes.
*
- * @throws Exception
+ * @throws Exception If failed.
*/
public void testRebalanceForCacheWithNodeFilter() throws Exception {
try {
@@ -846,8 +835,6 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
* @throws Exception if failed.
*/
public void testAffinityAssignmentChangedAfterRestart() throws Exception {
- delayRebalance = false;
-
int parts = 32;
final List<Integer> partMapping = new ArrayList<>();
@@ -890,8 +877,6 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
Collections.shuffle(TestAffinityFunction.partsAffMapping, new Random(1));
- delayRebalance = true;
-
/* There is a problem with handling simultaneous auto activation after restart and manual activation.
To properly catch the moment when cluster activation has finished we temporary disable auto activation. */
disableAutoActivation = true;
@@ -1035,22 +1020,4 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
delegate.removeNode(nodeId);
}
}
-
- /**
- *
- */
- private static class DelayRebalanceCommunicationSpi extends TestDelayingCommunicationSpi {
- /** {@inheritDoc} */
- @Override protected boolean delayMessage(Message msg, GridIoMessage ioMsg) {
- if (msg != null && (msg instanceof GridDhtPartitionDemandMessage || msg instanceof GridDhtPartitionSupplyMessage))
- return true;
-
- return false;
- }
-
- /** {@inheritDoc} */
- @Override protected int delayMillis() {
- return 1_000_000;
- }
- }
}
[25/50] [abbrv] ignite git commit: IGNITE-8276 Fixed incorrect
assertion during initialValue - Fixes #3827.
Posted by ag...@apache.org.
IGNITE-8276 Fixed incorrect assertion during initialValue - Fixes #3827.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b058bf65
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b058bf65
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b058bf65
Branch: refs/heads/ignite-6083
Commit: b058bf6577bd19bff15307c1e748fb15e9f60445
Parents: 8826fa1
Author: Ilya Lantukh <il...@gridgain.com>
Authored: Wed Apr 18 19:04:39 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Apr 18 19:04:39 2018 +0300
----------------------------------------------------------------------
.../ignite/internal/processors/cache/GridCacheMapEntry.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b058bf65/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index a6ef0d2..9f3686a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -3571,12 +3571,10 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
* @param oldRow Old row if available.
* @throws IgniteCheckedException If update failed.
*/
- protected boolean storeValue(CacheObject val,
+ protected boolean storeValue(@Nullable CacheObject val,
long expireTime,
GridCacheVersion ver,
@Nullable CacheDataRow oldRow) throws IgniteCheckedException {
- assert val != null : "null values in update for key: " + key;
-
return storeValue(val, expireTime, ver, oldRow, null);
}
[43/50] [abbrv] ignite git commit: IGNITE-8077 Introduced additional
transaction metrics - Fixes #3879.
Posted by ag...@apache.org.
IGNITE-8077 Introduced additional transaction metrics - Fixes #3879.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/358d0390
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/358d0390
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/358d0390
Branch: refs/heads/ignite-6083
Commit: 358d0390d9b0eb9f7fa9f5cb66c55be682b1cbcc
Parents: ae18817
Author: Dmitriy Govorukhin <dm...@gmail.com>
Authored: Fri Apr 20 16:20:00 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Apr 20 16:20:00 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/IgniteKernal.java | 56 ++--
.../internal/TransactionMetricsMxBeanImpl.java | 104 +++++++
.../cache/GridCacheSharedContext.java | 6 +-
.../transactions/TransactionMetricsAdapter.java | 206 ++++++++++++-
.../ignite/mxbean/TransactionMetricsMxBean.java | 86 ++++++
.../ignite/transactions/TransactionMetrics.java | 52 ++++
.../TransactionMetricsMxBeanImplTest.java | 302 +++++++++++++++++++
.../IgniteCacheMetricsSelfTestSuite.java | 3 +
.../ApiParity/TransactionMetricsParityTest.cs | 14 +-
9 files changed, 786 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/358d0390/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 019938f..53fcc83 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -201,6 +201,7 @@ import org.apache.ignite.mxbean.IgniteMXBean;
import org.apache.ignite.mxbean.StripedExecutorMXBean;
import org.apache.ignite.mxbean.WorkersControlMXBean;
import org.apache.ignite.mxbean.ThreadPoolMXBean;
+import org.apache.ignite.mxbean.TransactionMetricsMxBean;
import org.apache.ignite.plugin.IgnitePlugin;
import org.apache.ignite.plugin.PluginNotFoundException;
import org.apache.ignite.plugin.PluginProvider;
@@ -650,7 +651,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/**
- * @param name New attribute name.
+ * @param name New attribute name.
* @param val New attribute value.
* @throws IgniteCheckedException If duplicated SPI name found.
*/
@@ -750,8 +751,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
@Nullable final Map<String, ? extends ExecutorService> customExecSvcs,
GridAbsClosure errHnd
)
- throws IgniteCheckedException
- {
+ throws IgniteCheckedException {
gw.compareAndSet(null, new GridKernalGatewayImpl(cfg.getIgniteInstanceName()));
GridKernalGateway gw = this.gw.get();
@@ -954,7 +954,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
try {
startProcessor(new PdsConsistentIdProcessor(ctx));
startProcessor(createComponent(DiscoveryNodeValidationProcessor.class, ctx));
- startProcessor(new GridAffinityProcessor(ctx));
+ startProcessor(new GridAffinityProcessor(ctx));
startProcessor(createComponent(GridSegmentationProcessor.class, ctx));
startProcessor(createComponent(IgniteCacheObjectProcessor.class, ctx));
startProcessor(createComponent(IGridClusterStateProcessor.class, ctx));
@@ -1720,6 +1720,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
/**
* Returns class name filter for marshaller.
+ *
* @return Class name filter for marshaller.
*/
private IgnitePredicate<String> classNameFilter() throws IgniteCheckedException {
@@ -1774,9 +1775,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
return clsSet;
}
-
/**
* Reads class names from resource referred by given system property name and returns set of classes.
+ *
* @param fileName File name containing list of classes.
* @param clsSet Class set for update.
* @return Set of classes.
@@ -1805,7 +1806,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
catch (IllegalArgumentException e) {
throw new IgniteCheckedException("Exception occurred while reading list of classes" +
- "[path=" + fileName + ", row=" + i + ", line=" + s + ']', e);
+ "[path=" + fileName + ", row=" + i + ", line=" + s + ']', e);
}
}
}
@@ -1836,7 +1837,6 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/**
- *
* @return Whether or not REST is enabled.
*/
private boolean isRestEnabled() {
@@ -1903,7 +1903,6 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
log.info("Config URL: " + System.getProperty(IGNITE_CONFIG_URL, "n/a"));
}
-
/**
* Acks configuration.
*/
@@ -1921,7 +1920,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
assert log != null;
if (log.isInfoEnabled())
- log.info("Logger: " + log.getLoggerInfo() );
+ log.info("Logger: " + log.getLoggerInfo());
}
/**
@@ -2345,8 +2344,8 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
* USED ONLY FOR TESTING.
*
* @param name Cache name.
- * @param <K> Key type.
- * @param <V> Value type.
+ * @param <K> Key type.
+ * @param <V> Value type.
* @return Internal cache instance.
*/
/*@java.test.only*/
@@ -2410,7 +2409,6 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/**
- *
* @return {@code True} is this node is daemon.
*/
private boolean isDaemon() {
@@ -2420,11 +2418,8 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/**
- * Whether or not remote JMX management is enabled for this node. Remote JMX management is
- * enabled when the following system property is set:
- * <ul>
- * <li>{@code com.sun.management.jmxremote}</li>
- * </ul>
+ * Whether or not remote JMX management is enabled for this node. Remote JMX management is enabled when the
+ * following system property is set: <ul> <li>{@code com.sun.management.jmxremote}</li> </ul>
*
* @return {@code True} if remote JMX management is enabled - {@code false} otherwise.
*/
@@ -2433,9 +2428,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/**
- * Whether or not node restart is enabled. Node restart us supported when this node was started
- * with {@code bin/ignite.{sh|bat}} script using {@code -r} argument. Node can be
- * programmatically restarted using {@link Ignition#restart(boolean)}} method.
+ * Whether or not node restart is enabled. Node restart us supported when this node was started with {@code
+ * bin/ignite.{sh|bat}} script using {@code -r} argument. Node can be programmatically restarted using {@link
+ * Ignition#restart(boolean)}} method.
*
* @return {@code True} if restart mode is enabled, {@code false} otherwise.
* @see Ignition#restart(boolean)
@@ -2812,7 +2807,6 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
}
-
/** {@inheritDoc} */
@Override public Collection<IgniteCache> createCaches(Collection<CacheConfiguration> cacheCfgs) {
A.notNull(cacheCfgs, "cacheCfgs");
@@ -2886,12 +2880,12 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
if (ctx.cache().cache(cacheCfg.getName()) == null) {
res =
sql ? ctx.cache().dynamicStartSqlCache(cacheCfg).get() :
- ctx.cache().dynamicStartCache(cacheCfg,
- cacheCfg.getName(),
- null,
- false,
- true,
- true).get();
+ ctx.cache().dynamicStartCache(cacheCfg,
+ cacheCfg.getName(),
+ null,
+ false,
+ true,
+ true).get();
}
return new IgniteBiTuple<>((IgniteCache<K, V>)ctx.cache().publicJCache(cacheCfg.getName()), res);
@@ -4022,7 +4016,6 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
/**
* @return IgniteKernal instance.
- *
* @throws ObjectStreamException If failed.
*/
protected Object readResolve() throws ObjectStreamException {
@@ -4159,7 +4152,6 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
* @param qryExecSvc Query executor service
* @param schemaExecSvc Schema executor service
* @param customExecSvcs Custom named executors
- *
* @throws IgniteCheckedException if fails to register any of the MBeans
*/
private void registerAllMBeans(
@@ -4193,6 +4185,10 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
ClusterMetricsMXBean metricsBean = new ClusterMetricsMXBeanImpl(cluster());
registerMBean("Kernal", metricsBean.getClass().getSimpleName(), metricsBean, ClusterMetricsMXBean.class);
+ // Transactions
+ TransactionMetricsMxBean txMXBean = new TransactionMetricsMxBeanImpl(ctx.cache().transactions().metrics());
+ registerMBean("Transactions", txMXBean.getClass().getSimpleName(), txMXBean, TransactionMetricsMxBean.class);
+
// Executors
registerExecutorMBean("GridUtilityCacheExecutor", utilityCachePool);
registerExecutorMBean("GridExecutionExecutor", execSvc);
@@ -4239,7 +4235,6 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
*
* @param name name of the bean to register
* @param exec executor to register a bean for
- *
* @throws IgniteCheckedException if registration fails.
*/
private void registerExecutorMBean(String name, ExecutorService exec) throws IgniteCheckedException {
@@ -4254,7 +4249,6 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
* @param impl bean implementation
* @param itf bean interface
* @param <T> bean type
- *
* @throws IgniteCheckedException if registration fails
*/
private <T> void registerMBean(String grp, String name, T impl, Class<T> itf) throws IgniteCheckedException {
http://git-wip-us.apache.org/repos/asf/ignite/blob/358d0390/modules/core/src/main/java/org/apache/ignite/internal/TransactionMetricsMxBeanImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/TransactionMetricsMxBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/TransactionMetricsMxBeanImpl.java
new file mode 100644
index 0000000..715bd27
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/TransactionMetricsMxBeanImpl.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.Map;
+import org.apache.ignite.mxbean.TransactionMetricsMxBean;
+import org.apache.ignite.transactions.TransactionMetrics;
+
+/**
+ * Transactions MXBean implementation.
+ */
+public class TransactionMetricsMxBeanImpl implements TransactionMetricsMxBean {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final TransactionMetrics transactionMetrics;
+
+ /**
+ * Create TransactionMetricsMxBeanImpl.
+ */
+ public TransactionMetricsMxBeanImpl() {
+ this(null);
+ }
+
+ /**
+ * @param transactionMetrics Transaction metrics.
+ */
+ TransactionMetricsMxBeanImpl(TransactionMetrics transactionMetrics) {
+ this.transactionMetrics = transactionMetrics;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long commitTime() {
+ return transactionMetrics.commitTime();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long rollbackTime() {
+ return transactionMetrics.rollbackTime();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int txCommits() {
+ return transactionMetrics.txCommits();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int txRollbacks() {
+ return transactionMetrics.txRollbacks();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, String> getAllOwnerTransactions() {
+ return transactionMetrics.getAllOwnerTransactions();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, String> getLongRunningOwnerTransactions(int duration) {
+ return transactionMetrics.getLongRunningOwnerTransactions(duration);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTransactionsCommittedNumber() {
+ return transactionMetrics.getTransactionsCommittedNumber();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTransactionsRolledBackNumber() {
+ return transactionMetrics.getTransactionsRolledBackNumber();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTransactionsHoldingLockNumber() {
+ return transactionMetrics.getTransactionsHoldingLockNumber();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLockedKeysNumber() {
+ return transactionMetrics.getLockedKeysNumber();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getOwnerTransactionsNumber() {
+ return transactionMetrics.getOwnerTransactionsNumber();
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/ignite/blob/358d0390/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index b3b4f0d..b5c4096 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -129,7 +129,7 @@ public class GridCacheSharedContext<K, V> {
private ConcurrentHashMap<Integer, GridCacheContext<K, V>> ctxMap;
/** Tx metrics. */
- private volatile TransactionMetricsAdapter txMetrics;
+ private final TransactionMetricsAdapter txMetrics;
/** Store session listeners. */
private Collection<CacheStoreSessionListener> storeSesLsnrs;
@@ -208,7 +208,7 @@ public class GridCacheSharedContext<K, V> {
this.storeSesLsnrs = storeSesLsnrs;
- txMetrics = new TransactionMetricsAdapter();
+ txMetrics = new TransactionMetricsAdapter(kernalCtx);
ctxMap = new ConcurrentHashMap<>();
@@ -621,7 +621,7 @@ public class GridCacheSharedContext<K, V> {
* Resets tx metrics.
*/
public void resetTxMetrics() {
- txMetrics = new TransactionMetricsAdapter();
+ txMetrics.reset();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/358d0390/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java
index 783796b..c537bb8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java
@@ -21,14 +21,32 @@ import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.GridKernalContextImpl;
+import org.apache.ignite.internal.processors.cache.GridCacheMvccManager;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.util.GridStringBuilder;
+import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.transactions.TransactionMetrics;
+import org.apache.ignite.transactions.TransactionState;
/**
* Tx metrics adapter.
*/
public class TransactionMetricsAdapter implements TransactionMetrics, Externalizable {
+ /** Grid kernal context. */
+ private final GridKernalContext gridKernalCtx;
+
/** */
private static final long serialVersionUID = 0L;
@@ -45,20 +63,17 @@ public class TransactionMetricsAdapter implements TransactionMetrics, Externaliz
private volatile long rollbackTime;
/**
- *
+ * Create TransactionMetricsAdapter.
*/
public TransactionMetricsAdapter() {
-
+ this(null);
}
/**
- * @param m Transaction metrics to copy.
+ * @param ctx Kernal context.
*/
- public TransactionMetricsAdapter(TransactionMetrics m) {
- commitTime = m.commitTime();
- rollbackTime = m.rollbackTime();
- txCommits = m.txCommits();
- txRollbacks = m.txRollbacks();
+ public TransactionMetricsAdapter(GridKernalContext ctx) {
+ gridKernalCtx = ctx;
}
/** {@inheritDoc} */
@@ -81,6 +96,41 @@ public class TransactionMetricsAdapter implements TransactionMetrics, Externaliz
return txRollbacks;
}
+ /** {@inheritDoc} */
+ @Override public Map<String, String> getAllOwnerTransactions() {
+ return getNearTxs(0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, String> getLongRunningOwnerTransactions(final int duration) {
+ return getNearTxs(duration);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTransactionsCommittedNumber() {
+ return gridKernalCtx.cache().context().txMetrics().txCommits();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTransactionsRolledBackNumber() {
+ return gridKernalCtx.cache().context().txMetrics().txRollbacks();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTransactionsHoldingLockNumber() {
+ return txHoldingLockNum();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLockedKeysNumber() {
+ return txLockedKeysNum();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getOwnerTransactionsNumber() {
+ return nearTxNum();
+ }
+
/**
* Transaction commit callback.
*/
@@ -99,6 +149,146 @@ public class TransactionMetricsAdapter implements TransactionMetrics, Externaliz
txRollbacks++;
}
+ /**
+ * Reset.
+ */
+ public void reset() {
+ commitTime = 0;
+ txCommits = 0;
+ rollbackTime = 0;
+ txRollbacks = 0;
+ }
+
+ /**
+ * @param duration Duration.
+ */
+ private Map<String, String> getNearTxs(long duration) {
+ final Collection<GridNearTxLocal> txs = nearTxs(duration);
+
+ final HashMap<String, String> res = new HashMap<>(txs.size());
+
+ for (GridNearTxLocal tx : txs)
+ res.put(tx.xid().toString(), composeTx(tx));
+
+ return res;
+ }
+
+ /**
+ * @param id Id.
+ */
+ private String composeNodeInfo(final UUID id) {
+ final ClusterNode node = gridKernalCtx.discovery().node(id);
+ if (node == null)
+ return "";
+
+ return String.format("%s %s",
+ node.id(),
+ node.hostNames());
+ }
+
+ /**
+ * @param ids Ids.
+ */
+ private String composeNodeInfo(final Set<UUID> ids) {
+ final GridStringBuilder sb = new GridStringBuilder();
+
+ sb.a("[");
+
+ String delim = "";
+
+ for (UUID id : ids) {
+ sb
+ .a(delim)
+ .a(composeNodeInfo(id));
+ delim = ", ";
+ }
+
+ sb.a("]");
+
+ return sb.toString();
+ }
+
+ /**
+ * @param tx Transaction.
+ */
+ private String composeTx(final GridNearTxLocal tx) {
+ final TransactionState txState = tx.state();
+
+ String top = txState + ", NEAR, ";
+
+ if (txState == TransactionState.PREPARING) {
+ final Map<UUID, Collection<UUID>> transactionNodes = tx.transactionNodes();
+ if (!F.isEmpty(transactionNodes)) {
+ final Set<UUID> primaryNodes = transactionNodes.keySet();
+ if (!F.isEmpty(primaryNodes))
+ top += "PRIMARY: " + composeNodeInfo(primaryNodes) + ", ";
+ }
+ }
+
+ final Long duration = System.currentTimeMillis() - tx.startTime();
+
+ return top + "DURATION: " + duration;
+ }
+
+ /**
+ *
+ */
+ private Collection<GridNearTxLocal> nearTxs(long duration) {
+ final long start = System.currentTimeMillis();
+ IgniteClosure<IgniteInternalTx, GridNearTxLocal> c = new IgniteClosure<IgniteInternalTx, GridNearTxLocal>() {
+ @Override public GridNearTxLocal apply(IgniteInternalTx tx) {
+ return ((GridNearTxLocal)tx);
+ }
+ };
+
+ IgnitePredicate<IgniteInternalTx> pred = new IgnitePredicate<IgniteInternalTx>() {
+ @Override public boolean apply(IgniteInternalTx tx) {
+ return tx.local() && tx.near() && start - tx.startTime() >= duration;
+ }
+ };
+
+ return F.viewReadOnly(gridKernalCtx.cache().context().tm().activeTransactions(), c, pred);
+ }
+
+ /**
+ *
+ */
+ private long nearTxNum() {
+ IgnitePredicate<IgniteInternalTx> pred = new IgnitePredicate<IgniteInternalTx>() {
+ @Override public boolean apply(IgniteInternalTx tx) {
+ return tx.local() && tx.near();
+ }
+ };
+
+ return F.size(gridKernalCtx.cache().context().tm().activeTransactions(), pred);
+ }
+
+ /**
+ * Count total number of holding locks on local node.
+ */
+ private long txHoldingLockNum() {
+ long holdingLockCounter = 0;
+
+ IgniteTxManager tm = gridKernalCtx.cache().context().tm();
+ for (IgniteInternalTx tx : tm.activeTransactions()) {
+ if ((tx.optimistic() && tx.state() == TransactionState.ACTIVE) || tx.empty() || !tx.local())
+ continue;
+
+ holdingLockCounter++;
+ }
+
+ return holdingLockCounter;
+ }
+
+ /**
+ * Count total number of locked keys on local node.
+ */
+ private long txLockedKeysNum() {
+ GridCacheMvccManager mvccManager = gridKernalCtx.cache().context().mvcc();
+
+ return mvccManager.lockedKeys().size() + mvccManager.nearLockedKeys().size();
+ }
+
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
out.writeLong(commitTime);
http://git-wip-us.apache.org/repos/asf/ignite/blob/358d0390/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionMetricsMxBean.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionMetricsMxBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionMetricsMxBean.java
new file mode 100644
index 0000000..effe28f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionMetricsMxBean.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mxbean;
+
+import java.util.Map;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.transactions.TransactionMetrics;
+
+/**
+ * Transactions MXBean interface.
+ */
+@MXBeanDescription("MBean that provides access to Ignite transactions.")
+public interface TransactionMetricsMxBean extends TransactionMetrics {
+ /**
+ * All near transactions
+ *
+ * @return near transactions.
+ */
+ @MXBeanDescription("All near transactions.")
+ @Override public Map<String, String> getAllOwnerTransactions();
+
+ /**
+ * Long running near transactions
+ *
+ * @return near transactions.
+ */
+ @MXBeanDescription("Long running near transactions.")
+ @MXBeanParametersNames("duration")
+ @MXBeanParametersDescriptions("Duration, at least (ms).")
+ @Override public Map<String, String> getLongRunningOwnerTransactions(int duration);
+
+ /**
+ * The number of transactions which were committed.
+ *
+ * @return number of transactions which were committed.
+ */
+ @MXBeanDescription("The number of transactions which were committed.")
+ @Override public long getTransactionsCommittedNumber();
+
+ /**
+ * The number of transactions which were rollback.
+ *
+ * @return number of transactions which were rollback.
+ */
+ @MXBeanDescription("The number of transactions which were rollback.")
+ @Override public long getTransactionsRolledBackNumber();
+
+ /**
+ * The number of active transactions holding at least one key lock.
+ *
+ * @return number of active transactions holding at least one key lock.
+ */
+ @MXBeanDescription("The number of active transactions holding at least one key lock.")
+ @Override public long getTransactionsHoldingLockNumber();
+
+ /**
+ * The number of keys locked on the node.
+ *
+ * @return number of keys locked on the node.
+ */
+ @MXBeanDescription("The number of keys locked on the node.")
+ @Override public long getLockedKeysNumber();
+
+ /**
+ * The number of active transactions for which this node is the initiator.
+ *
+ * @return number of active transactions for which this node is the initiator.
+ */
+ @MXBeanDescription("The number of active transactions for which this node is the initiator.")
+ @Override public long getOwnerTransactionsNumber();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/358d0390/modules/core/src/main/java/org/apache/ignite/transactions/TransactionMetrics.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/transactions/TransactionMetrics.java b/modules/core/src/main/java/org/apache/ignite/transactions/TransactionMetrics.java
index a5af51e..fa2ec22 100644
--- a/modules/core/src/main/java/org/apache/ignite/transactions/TransactionMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/transactions/TransactionMetrics.java
@@ -18,6 +18,7 @@
package org.apache.ignite.transactions;
import java.io.Serializable;
+import java.util.Map;
/**
* Transaction metrics, shared across all caches.
@@ -50,4 +51,55 @@ public interface TransactionMetrics extends Serializable {
* @return Number of transaction rollbacks.
*/
public int txRollbacks();
+
+ /**
+ * Gets a map of all transactions for which the local node is the originating node.
+ *
+ * @return Map of local node owning transactions.
+ */
+ public Map<String, String> getAllOwnerTransactions();
+
+ /**
+ * Gets a map of all transactions for which the local node is the originating node and which duration
+ * exceeds the given duration.
+ *
+ * @return Map of local node owning transactions which duration is longer than {@code duration}.
+ */
+ public Map<String, String> getLongRunningOwnerTransactions(int duration);
+
+ /**
+ * The number of transactions which were committed on the local node.
+ *
+ * @return The number of transactions which were committed on the local node.
+ */
+ public long getTransactionsCommittedNumber();
+
+ /**
+ * The number of transactions which were rolled back on the local node.
+ *
+ * @return The number of transactions which were rolled back on the local node.
+ */
+ public long getTransactionsRolledBackNumber();
+
+ /**
+ * The number of active transactions on the local node holding at least one key lock.
+ *
+ * @return The number of active transactions holding at least one key lock.
+ */
+ public long getTransactionsHoldingLockNumber();
+
+ /**
+ * The number of keys locked on the node.
+ *
+ * @return The number of keys locked on the node.
+ */
+ public long getLockedKeysNumber();
+
+ /**
+ * The number of active transactions for which this node is the initiator. Effectively, this method is
+ * semantically equivalent to {@code getAllOwnerTransactions.size()}.
+ *
+ * @return The number of active transactions for which this node is the initiator.
+ */
+ public long getOwnerTransactionsNumber();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/358d0390/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java
new file mode 100644
index 0000000..80b0dc0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import javax.management.MBeanServer;
+import javax.management.MBeanServerInvocationHandler;
+import javax.management.ObjectName;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.mxbean.TransactionMetricsMxBean;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ *
+ */
+public class TransactionMetricsMxBeanImplTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final int TRANSACTIONS = 10;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+ final IgniteConfiguration cfg = super.getConfiguration(name);
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+ cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+ cfg.setLocalHost("127.0.0.1");
+
+ final CacheConfiguration cCfg = new CacheConfiguration()
+ .setName(DEFAULT_CACHE_NAME)
+ .setAffinity(new RendezvousAffinityFunction(false, 32))
+ .setBackups(1)
+ .setAtomicityMode(TRANSACTIONAL)
+ .setRebalanceMode(CacheRebalanceMode.ASYNC)
+ .setWriteSynchronizationMode(FULL_SYNC);
+
+ cfg.setCacheConfiguration(cCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ super.afterTest();
+ }
+
+ /**
+ *
+ */
+ public void testTxMetric() throws Exception {
+ //given:
+ int keysNumber = 10;
+
+ IgniteEx ignite = startGrid(0);
+
+ startGrid(1);
+
+ IgniteConfiguration clientConf = getConfiguration(getTestIgniteInstanceName(2));
+
+ clientConf.setClientMode(true);
+
+ IgniteEx client = startGrid(clientConf);
+
+ awaitPartitionMapExchange();
+
+ TransactionMetricsMxBean txMXBean = txMXBean(0);
+
+ final IgniteCache<Integer, String> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+ //when: one transaction commit
+ ignite.transactions().txStart().commit();
+
+ //then:
+ assertEquals(1, txMXBean.getTransactionsCommittedNumber());
+
+ //when: transaction is opening
+ final Transaction tx1 = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ);
+
+ int localKeysNum = 0;
+
+ for (int i = 0; i < keysNumber; i++) {
+ cache.put(i, "");
+
+ if (affinity(cache).isPrimary(ignite.localNode(), i))
+ localKeysNum++;
+ }
+
+ //then:
+ assertEquals(localKeysNum, txMXBean.getLockedKeysNumber());
+ assertEquals(1, txMXBean.getTransactionsHoldingLockNumber());
+ assertEquals(1, txMXBean.getOwnerTransactionsNumber());
+
+ //when: transaction rollback
+ tx1.rollback();
+
+ //then:
+ assertEquals(1, txMXBean.getTransactionsRolledBackNumber());
+ assertEquals(0, txMXBean.getLockedKeysNumber());
+ assertEquals(0, txMXBean.getTransactionsHoldingLockNumber());
+ assertEquals(0, txMXBean.getOwnerTransactionsNumber());
+
+ //when: keysNumber transactions from MXbean owner node + keysNumber transactions from client.
+ CountDownLatch commitAllower = new CountDownLatch(1);
+ CountDownLatch transactionStarter = new CountDownLatch(keysNumber + keysNumber);
+
+ int txNumFromOwner = 0;
+
+ for (int i = 0; i < keysNumber; i++) {
+ new Thread(new TxThread(commitAllower, transactionStarter, ignite, i, i)).start();
+
+ if (affinity(cache).isPrimary(ignite.localNode(), i))
+ txNumFromOwner++;
+ }
+
+ int txNumFromClient = 0;
+
+ for (int i = keysNumber; i < keysNumber * 2; i++) {
+ new Thread(new TxThread(commitAllower, transactionStarter, client, i, i)).start();
+
+ if (affinity(cache).isPrimary(ignite.localNode(), i))
+ txNumFromClient++;
+ }
+
+ transactionStarter.await();
+
+ //then:
+ assertEquals(txNumFromOwner + txNumFromClient, txMXBean.getLockedKeysNumber());
+ assertEquals(keysNumber + txNumFromClient, txMXBean.getTransactionsHoldingLockNumber());
+ assertEquals(keysNumber, txMXBean.getOwnerTransactionsNumber());
+
+ commitAllower.countDown();
+ }
+
+ /**
+ *
+ */
+ public void testNearTxInfo() throws Exception {
+ IgniteEx primaryNode1 = startGrid(0);
+ IgniteEx primaryNode2 = startGrid(1);
+ IgniteEx nearNode = startGrid(2);
+
+ TransactionMetricsMxBean txMXBeanBackup = txMXBean(2);
+
+ awaitPartitionMapExchange();
+
+ final IgniteCache<Integer, String> primaryCache1 = primaryNode1.cache(DEFAULT_CACHE_NAME);
+ final IgniteCache<Integer, String> primaryCache2 = primaryNode2.cache(DEFAULT_CACHE_NAME);
+
+ final List<Integer> primaryKeys1 = primaryKeys(primaryCache1, TRANSACTIONS);
+ final List<Integer> primaryKeys2 = primaryKeys(primaryCache2, TRANSACTIONS);
+
+ CountDownLatch commitAllower = new CountDownLatch(1);
+ CountDownLatch transactionStarter = new CountDownLatch(primaryKeys1.size());
+
+ for (int i = 0; i < primaryKeys1.size(); i++)
+ new Thread(new TxThread(
+ commitAllower,
+ transactionStarter,
+ nearNode,
+ primaryKeys1.get(i),
+ primaryKeys2.get(i)
+ )).start();
+
+ transactionStarter.await();
+
+ final Map<String, String> transactions = txMXBeanBackup.getAllOwnerTransactions();
+
+ assertEquals(TRANSACTIONS, transactions.size());
+
+ int match = 0;
+
+ for (String txInfo : transactions.values()) {
+ if (txInfo.contains("ACTIVE")
+ && txInfo.contains("NEAR")
+ && !txInfo.contains("REMOTE"))
+ match++;
+ }
+
+ assertEquals(TRANSACTIONS, match);
+
+ commitAllower.countDown();
+ }
+
+ /**
+ *
+ */
+ private static class TxThread implements Runnable {
+ /** */
+ private CountDownLatch commitAllowLatch;
+ /** */
+ private CountDownLatch transactionStartLatch;
+ /** */
+ private Ignite ignite;
+ /** */
+ private int key1;
+ /** */
+ private int key2;
+
+ /**
+ * Create TxThread.
+ */
+ private TxThread(
+ CountDownLatch commitAllowLatch,
+ CountDownLatch transactionStartLatch,
+ final Ignite ignite,
+ final int key1,
+ final int key2
+ ) {
+ this.commitAllowLatch = commitAllowLatch;
+ this.transactionStartLatch = transactionStartLatch;
+ this.ignite = ignite;
+ this.key1 = key1;
+ this.key2 = key2;
+ }
+
+ /**
+ * @param ignite Ignite.
+ * @param key1 key 1.
+ * @param key2 key 2.
+ */
+ private TxThread(final Ignite ignite, final int key1, final int key2) {
+ commitAllowLatch = new CountDownLatch(0);
+ transactionStartLatch = new CountDownLatch(1);
+
+ this.ignite = ignite;
+ this.key1 = key1;
+ this.key2 = key2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+ ignite.cache(DEFAULT_CACHE_NAME).put(key1, Thread.currentThread().getName());
+ ignite.cache(DEFAULT_CACHE_NAME).put(key2, Thread.currentThread().getName());
+
+ transactionStartLatch.countDown();
+
+ commitAllowLatch.await();
+
+ tx.commit();
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private TransactionMetricsMxBean txMXBean(int igniteInt) throws Exception {
+ ObjectName mbeanName = U.makeMBeanName(
+ getTestIgniteInstanceName(igniteInt),
+ "Transactions",
+ TransactionMetricsMxBeanImpl.class.getSimpleName()
+ );
+
+ MBeanServer mbeanSrv = ManagementFactory.getPlatformMBeanServer();
+
+ if (!mbeanSrv.isRegistered(mbeanName))
+ fail("MBean is not registered: " + mbeanName.getCanonicalName());
+
+ return MBeanServerInvocationHandler.newProxyInstance(mbeanSrv, mbeanName, TransactionMetricsMxBean.class, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/358d0390/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
index ac7dca2..237f7e1 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
@@ -18,6 +18,7 @@
package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
+import org.apache.ignite.internal.TransactionMetricsMxBeanImplTest;
import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMBeanTest;
import org.apache.ignite.internal.processors.cache.CacheGroupsMetricsRebalanceTest;
import org.apache.ignite.internal.processors.cache.CacheMetricsEnableRuntimeTest;
@@ -74,6 +75,8 @@ public class IgniteCacheMetricsSelfTestSuite extends TestSuite {
suite.addTestSuite(CacheMetricsForClusterGroupSelfTest.class);
suite.addTestSuite(OffheapCacheMetricsForClusterGroupSelfTest.class);
+ suite.addTestSuite(TransactionMetricsMxBeanImplTest.class);
+
return suite;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/358d0390/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionMetricsParityTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionMetricsParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionMetricsParityTest.cs
index 0327e3a..6740eb9 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionMetricsParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionMetricsParityTest.cs
@@ -25,6 +25,18 @@ namespace Apache.Ignite.Core.Tests.ApiParity
/// </summary>
public class TransactionMetricsParityTest
{
+ /** Properties that are missing on .NET side. */
+ private static readonly string[] MissingProperties =
+ {
+ "AllOwnerTransactions",
+ "LongRunningOwnerTransactions",
+ "TransactionsCommittedNumber",
+ "TransactionsRolledBackNumber",
+ "TransactionsHoldingLockNumber",
+ "LockedKeysNumber",
+ "OwnerTransactionsNumber",
+ };
+
/// <summary>
/// Tests the API parity.
/// </summary>
@@ -33,7 +45,7 @@ namespace Apache.Ignite.Core.Tests.ApiParity
{
ParityTest.CheckInterfaceParity(
@"modules\core\src\main\java\org\apache\ignite\transactions\TransactionMetrics.java",
- typeof(ITransactionMetrics));
+ typeof(ITransactionMetrics),knownMissingMembers: MissingProperties);
}
}
}
\ No newline at end of file
[04/50] [abbrv] ignite git commit: IGNITE-8200 Web Console: Override
clonedCluster in cluster-edit-form if caches or models have changed. This
improves interop with "import from DB" feature,
which might update caches/models of cluster currently opene
Posted by ag...@apache.org.
IGNITE-8200 Web Console: Override clonedCluster in cluster-edit-form if caches or models have changed.
This improves interop with "import from DB" feature, which might update caches/models of cluster currently opened for editing.
The import dialog works as a separate state, so the form change detection mechanism ensures that any changes to the original
cluster are safe and won't interfere with changes made by user in cluster edit form.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/77316692
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/77316692
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/77316692
Branch: refs/heads/ignite-6083
Commit: 77316692f14d083138d7515affd2d3f225b709e0
Parents: 3cebf91
Author: Ilya Borisov <kl...@gmail.com>
Authored: Tue Apr 17 17:15:57 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Apr 17 17:15:57 2018 +0700
----------------------------------------------------------------------
.../components/cluster-edit-form/controller.js | 24 +++++-
.../cluster-edit-form/controller.spec.js | 81 ++++++++++++++++++++
2 files changed, 102 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/77316692/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
index 35b43e0..0207729 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
@@ -17,6 +17,7 @@
import cloneDeep from 'lodash/cloneDeep';
import get from 'lodash/get';
+import isEqual from 'lodash/isEqual';
import _ from 'lodash';
export default class ClusterEditFormController {
@@ -29,9 +30,11 @@ export default class ClusterEditFormController {
constructor(IgniteLegacyUtils, IgniteEventGroups, IgniteConfirm, IgniteVersion, $scope, Clusters, IgniteFormUtils) {
Object.assign(this, {IgniteLegacyUtils, IgniteEventGroups, IgniteConfirm, IgniteVersion, $scope, Clusters, IgniteFormUtils});
}
+
$onDestroy() {
this.subscription.unsubscribe();
}
+
$onInit() {
this.available = this.IgniteVersion.available.bind(this.IgniteVersion);
@@ -87,10 +90,9 @@ export default class ClusterEditFormController {
this.$scope.ui = this.IgniteFormUtils.formUI();
this.$scope.ui.loadedPanels = ['checkpoint', 'serviceConfiguration', 'odbcConfiguration'];
}
+
$onChanges(changes) {
- if (
- 'cluster' in changes && get(this.clonedCluster, '_id') !== get(this.cluster, '_id')
- ) {
+ if ('cluster' in changes && this.shouldOverwriteValue(this.cluster, this.clonedCluster)) {
this.clonedCluster = cloneDeep(changes.cluster.currentValue);
if (this.$scope.ui && this.$scope.ui.inputForm) {
this.$scope.ui.inputForm.$setPristine();
@@ -100,14 +102,30 @@ export default class ClusterEditFormController {
if ('caches' in changes)
this.cachesMenu = (changes.caches.currentValue || []).map((c) => ({label: c.name, value: c._id}));
}
+
+ /**
+ * The form should accept incoming cluster value if:
+ * 1. It has different _id ("new" to real id).
+ * 2. Different caches or models (imported from DB).
+ * @param {Object} a Incoming value.
+ * @param {Object} b Current value.
+ */
+ shouldOverwriteValue(a, b) {
+ return get(a, '_id') !== get(b, '_id') ||
+ !isEqual(get(a, 'caches'), get(b, 'caches')) ||
+ !isEqual(get(a, 'models'), get(b, 'models'));
+ }
+
getValuesToCompare() {
return [this.cluster, this.clonedCluster].map(this.Clusters.normalize);
}
+
save() {
if (this.$scope.ui.inputForm.$invalid)
return this.IgniteFormUtils.triggerValidation(this.$scope.ui.inputForm, this.$scope);
this.onSave({$event: cloneDeep(this.clonedCluster)});
}
+
reset = () => this.clonedCluster = cloneDeep(this.cluster);
confirmAndReset() {
return this.IgniteConfirm.confirm('Are you sure you want to undo all changes for current cluster?')
http://git-wip-us.apache.org/repos/asf/ignite/blob/77316692/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.spec.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.spec.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.spec.js
new file mode 100644
index 0000000..cac888f
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.spec.js
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import 'mocha';
+import {assert} from 'chai';
+import {spy} from 'sinon';
+import Controller from './controller';
+
+suite('cluster-edit-form controller', () => {
+ test('cluster binding changes', () => {
+ const $scope = {
+ ui: {
+ inputForm: {
+ $setPristine: spy(),
+ $setUntouched: spy()
+ }
+ }
+ };
+
+ const mocks = Controller.$inject.map((token) => {
+ switch (token) {
+ case '$scope': return $scope;
+ default: return null;
+ }
+ });
+
+ const changeBoundCluster = ($ctrl, cluster) => {
+ $ctrl.cluster = cluster;
+ $ctrl.$onChanges({
+ cluster: {
+ currentValue: cluster
+ }
+ });
+ };
+
+ const $ctrl = new Controller(...mocks);
+
+ const cluster1 = {_id: 1, caches: [1, 2, 3]};
+ const cluster2 = {_id: 1, caches: [1, 2, 3, 4, 5, 6], models: [1, 2, 3]};
+ const cluster3 = {_id: 1, caches: [1, 2, 3, 4, 5, 6], models: [1, 2, 3], name: 'Foo'};
+
+ changeBoundCluster($ctrl, cluster1);
+
+ assert.notEqual($ctrl.clonedCluster, cluster1, 'Cloned cluster is really cloned');
+ assert.deepEqual($ctrl.clonedCluster, cluster1, 'Cloned cluster is really a clone of incloming value');
+ assert.equal(1, $scope.ui.inputForm.$setPristine.callCount, 'Sets form pristine when cluster value changes');
+ assert.equal(1, $scope.ui.inputForm.$setUntouched.callCount, 'Sets form untouched when cluster value changes');
+
+ changeBoundCluster($ctrl, cluster2);
+
+ assert.deepEqual(
+ $ctrl.clonedCluster,
+ cluster2,
+ 'Overrides clonedCluster if incoming cluster has same id but different caches or models'
+ );
+ assert.equal(2, $scope.ui.inputForm.$setPristine.callCount, 'Sets form pristine when bound cluster caches/models change');
+ assert.equal(2, $scope.ui.inputForm.$setUntouched.callCount, 'Sets form untouched when bound cluster caches/models change');
+
+ changeBoundCluster($ctrl, cluster3);
+
+ assert.deepEqual(
+ $ctrl.clonedCluster,
+ cluster2,
+ 'Does not change cloned cluster value if fields other than id, chaches and models change'
+ );
+ });
+});
[48/50] [abbrv] ignite git commit: IGNITE-8154 - Add an ability to
provide ExceptionListener to JmsStreamer - Fixes #3828
Posted by ag...@apache.org.
IGNITE-8154 - Add an ability to provide ExceptionListener to JmsStreamer - Fixes #3828
Signed-off-by: Valentin Kulichenko <va...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ff7c4888
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ff7c4888
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ff7c4888
Branch: refs/heads/ignite-6083
Commit: ff7c4888f7d24c0d90fb77f3da6ecbbbe06039e4
Parents: 3f3239b
Author: Anton Kurbanov <an...@gmail.com>
Authored: Sat Apr 21 11:23:21 2018 -0700
Committer: Valentin Kulichenko <va...@gmail.com>
Committed: Sat Apr 21 11:23:21 2018 -0700
----------------------------------------------------------------------
.../apache/ignite/stream/jms11/JmsStreamer.java | 28 +++++++
.../stream/jms11/IgniteJmsStreamerTest.java | 81 ++++++++++++++++++--
2 files changed, 103 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ff7c4888/modules/jms11/src/main/java/org/apache/ignite/stream/jms11/JmsStreamer.java
----------------------------------------------------------------------
diff --git a/modules/jms11/src/main/java/org/apache/ignite/stream/jms11/JmsStreamer.java b/modules/jms11/src/main/java/org/apache/ignite/stream/jms11/JmsStreamer.java
index 1b203ad..55441d7 100644
--- a/modules/jms11/src/main/java/org/apache/ignite/stream/jms11/JmsStreamer.java
+++ b/modules/jms11/src/main/java/org/apache/ignite/stream/jms11/JmsStreamer.java
@@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.jms.Connection;
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
+import javax.jms.ExceptionListener;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageConsumer;
@@ -40,6 +41,7 @@ import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.stream.StreamAdapter;
/**
@@ -150,6 +152,9 @@ public class JmsStreamer<T extends Message, K, V> extends StreamAdapter<T, K, V>
/** Message listeners. */
private Set<IgniteJmsMessageListener> listeners = Collections.newSetFromMap(new ConcurrentHashMap<IgniteJmsMessageListener, Boolean>());
+ /** The Jms {@link ExceptionListener} to use. */
+ private ExceptionListener exceptionListener;
+
/** Scheduler for handling {@link #batchClosureMillis}. */
private ScheduledExecutorService scheduler;
@@ -215,6 +220,8 @@ public class JmsStreamer<T extends Message, K, V> extends StreamAdapter<T, K, V>
connection.setClientID(clientId.trim());
}
+ connection.setExceptionListener(new IgniteJmsExceptionListener());
+
// build the JMS objects
if (destinationType == Queue.class) {
initializeJmsObjectsForQueue();
@@ -424,6 +431,15 @@ public class JmsStreamer<T extends Message, K, V> extends StreamAdapter<T, K, V>
this.durableSubscriptionName = durableSubscriptionName;
}
+ /**
+ * Exception listener for queue/topic failures.
+ *
+ * @param exceptionListener ExceptionListener interface implementation.
+ */
+ public void setExceptionListener(ExceptionListener exceptionListener) {
+ this.exceptionListener = exceptionListener;
+ }
+
private void initializeJmsObjectsForTopic() throws JMSException {
Session session = connection.createSession(transacted, Session.AUTO_ACKNOWLEDGE);
Topic topic = (Topic)destination;
@@ -536,4 +552,16 @@ public class JmsStreamer<T extends Message, K, V> extends StreamAdapter<T, K, V>
}
}
+ /**
+ * Exception listener for JmsExceptions.
+ */
+ private class IgniteJmsExceptionListener implements ExceptionListener {
+ /** {@inheritDoc} */
+ @Override public void onException(JMSException e) {
+ U.error(log, "Caught JMS internal exception.", e);
+
+ if (exceptionListener != null)
+ exceptionListener.onException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ff7c4888/modules/jms11/src/test/java/org/apache/ignite/stream/jms11/IgniteJmsStreamerTest.java
----------------------------------------------------------------------
diff --git a/modules/jms11/src/test/java/org/apache/ignite/stream/jms11/IgniteJmsStreamerTest.java b/modules/jms11/src/test/java/org/apache/ignite/stream/jms11/IgniteJmsStreamerTest.java
index 2011684..caf8f9e 100644
--- a/modules/jms11/src/test/java/org/apache/ignite/stream/jms11/IgniteJmsStreamerTest.java
+++ b/modules/jms11/src/test/java/org/apache/ignite/stream/jms11/IgniteJmsStreamerTest.java
@@ -17,14 +17,19 @@
package org.apache.ignite.stream.jms11;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
+import javax.jms.ExceptionListener;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageProducer;
@@ -34,6 +39,7 @@ import javax.jms.Session;
import javax.jms.TextMessage;
import javax.jms.Topic;
import org.apache.activemq.ActiveMQConnectionFactory;
+import org.apache.activemq.broker.BrokerPlugin;
import org.apache.activemq.broker.BrokerRegistry;
import org.apache.activemq.broker.BrokerService;
import org.apache.activemq.broker.region.DestinationStatistics;
@@ -42,11 +48,14 @@ import org.apache.activemq.broker.region.policy.PolicyEntry;
import org.apache.activemq.broker.region.policy.PolicyMap;
import org.apache.activemq.command.ActiveMQQueue;
import org.apache.activemq.command.ActiveMQTopic;
+import org.apache.activemq.security.SimpleAuthenticationPlugin;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
import org.apache.ignite.events.CacheEvent;
import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.After;
import org.junit.Before;
@@ -71,17 +80,17 @@ public class IgniteJmsStreamerTest extends GridCommonAbstractTest {
/** */
private static final Map<String, String> TEST_DATA = new HashMap<>();
+ static {
+ for (int i = 1; i <= CACHE_ENTRY_COUNT; i++)
+ TEST_DATA.put(Integer.toString(i), "v" + i);
+ }
+
/** */
private BrokerService broker;
/** */
private ConnectionFactory connFactory;
- static {
- for (int i = 1; i <= CACHE_ENTRY_COUNT; i++)
- TEST_DATA.put(Integer.toString(i), "v" + i);
- }
-
/** Constructor. */
public IgniteJmsStreamerTest() {
super(true);
@@ -116,7 +125,6 @@ public class IgniteJmsStreamerTest extends GridCommonAbstractTest {
}
/**
- *
* @throws Exception Iff ailed.
*/
@After
@@ -500,6 +508,67 @@ public class IgniteJmsStreamerTest extends GridCommonAbstractTest {
}
/**
+ * Test for ExceptionListener functionality.
+ *
+ * @throws Exception If fails.
+ */
+ public void testExceptionListener() throws Exception {
+ // restart broker with auth plugin
+ if (broker.isStarted())
+ broker.stop();
+
+ broker.waitUntilStopped();
+
+ broker.setPlugins(new BrokerPlugin[]{new SimpleAuthenticationPlugin(new ArrayList())});
+
+ broker.start(true);
+
+ connFactory = new ActiveMQConnectionFactory(BrokerRegistry.getInstance().findFirst().getVmConnectorURI());
+
+ final List<Throwable> lsnrExceptions = new LinkedList<>();
+
+ final CountDownLatch latch = new CountDownLatch(1);
+
+ Destination dest = new ActiveMQQueue(QUEUE_NAME);
+
+ try (IgniteDataStreamer<String, String> dataStreamer = grid().dataStreamer(DEFAULT_CACHE_NAME)) {
+ JmsStreamer<ObjectMessage, String, String> jmsStreamer = newJmsStreamer(ObjectMessage.class, dataStreamer);
+
+ jmsStreamer.setExceptionListener(new ExceptionListener() {
+ @Override public void onException(JMSException e) {
+ System.out.println("ERROR");
+
+ lsnrExceptions.add(e);
+
+ latch.countDown();
+ }
+ });
+
+ jmsStreamer.setDestination(dest);
+
+ GridTestUtils.assertThrowsWithCause(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ jmsStreamer.start();
+
+ return null;
+ }
+ }, SecurityException.class);
+
+ assertTrue(latch.await(10, TimeUnit.SECONDS));
+
+ assertTrue(lsnrExceptions.size() > 0);
+
+ GridTestUtils.assertThrowsWithCause(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ jmsStreamer.stop();
+
+ return null;
+ }
+ }, IgniteException.class);
+ }
+ }
+
+ /**
*
*/
private void assertAllCacheEntriesLoaded() {
[15/50] [abbrv] ignite git commit: IGNITE-8301
testReconnectCacheDestroyedAndCreated should excpect recreated client cache -
Fixes #3856.
Posted by ag...@apache.org.
IGNITE-8301 testReconnectCacheDestroyedAndCreated should excpect recreated client cache - Fixes #3856.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/56be24b9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/56be24b9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/56be24b9
Branch: refs/heads/ignite-6083
Commit: 56be24b9dfc14023bacaab63f40e0504b317eda3
Parents: c3ff274
Author: Maxim Muzafarov <ma...@gmail.com>
Authored: Tue Apr 17 21:18:36 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Tue Apr 17 21:18:36 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/IgniteClientReconnectCacheTest.java | 6 ------
1 file changed, 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/56be24b9/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
index 3cb82e0..ec5eab2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
@@ -894,12 +894,6 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
}
});
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- return clientCache.get(1);
- }
- }, IllegalStateException.class, null);
-
checkCacheDiscoveryData(srv, client, DEFAULT_CACHE_NAME, true, false, false);
IgniteCache<Object, Object> clientCache0 = client.cache(DEFAULT_CACHE_NAME);
[49/50] [abbrv] ignite git commit: IGNITE-8353 Fixed serialVersionUID
for AbstractEvictionPolicy
Posted by ag...@apache.org.
IGNITE-8353 Fixed serialVersionUID for AbstractEvictionPolicy
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ecea655b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ecea655b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ecea655b
Branch: refs/heads/ignite-6083
Commit: ecea655b5bfae19767ddb3de2eae2245c4125821
Parents: ff7c488
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Sun Apr 22 20:07:28 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Sun Apr 22 20:07:28 2018 +0300
----------------------------------------------------------------------
.../org/apache/ignite/cache/eviction/AbstractEvictionPolicy.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ecea655b/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicy.java b/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicy.java
index d58077f..bd1c4ca 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicy.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicy.java
@@ -28,6 +28,9 @@ import org.apache.ignite.internal.util.typedef.internal.A;
* Common functionality implementation for eviction policies with max size/max memory and batch eviction support.
*/
public abstract class AbstractEvictionPolicy<K, V> implements EvictionPolicy<K, V>, Externalizable {
+ /** */
+ private static final long serialVersionUID = 4358725333474509598L;
+
/** Max memory size occupied by elements in container. */
private volatile long maxMemSize;
[19/50] [abbrv] ignite git commit: IGNITE-8143: Modified fetching
EXTERNAL_LIBS for docker container. - Fixes #3751.
Posted by ag...@apache.org.
IGNITE-8143: Modified fetching EXTERNAL_LIBS for docker container. - Fixes #3751.
Signed-off-by: shroman <rs...@yahoo.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/17ed189d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/17ed189d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/17ed189d
Branch: refs/heads/ignite-6083
Commit: 17ed189d70663289859c1da0d25ad69481cf9a6c
Parents: ced5d27
Author: shroman <rs...@yahoo.com>
Authored: Wed Apr 18 19:38:52 2018 +0900
Committer: shroman <rs...@yahoo.com>
Committed: Wed Apr 18 19:38:52 2018 +0900
----------------------------------------------------------------------
modules/docker/run.sh | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/17ed189d/modules/docker/run.sh
----------------------------------------------------------------------
diff --git a/modules/docker/run.sh b/modules/docker/run.sh
index dbf2871..72cfdd8 100644
--- a/modules/docker/run.sh
+++ b/modules/docker/run.sh
@@ -29,12 +29,8 @@ if [ ! -z "$EXTERNAL_LIBS" ]; then
IFS=, LIBS_LIST=("$EXTERNAL_LIBS")
for lib in ${LIBS_LIST[@]}; do
- echo $lib >> temp
+ wget $lib -P $IGNITE_HOME/libs
done
-
- wget -i temp -P $IGNITE_HOME/libs
-
- rm temp
fi
QUIET=""
[41/50] [abbrv] ignite git commit: IGNITE-8258 Race in page memory in
copyPageForCheckpoint, Ignite PDS 1 suite,
test failed suite IgnitePdsPageReplacementTest.testPageReplacement. - Fixes
#3821.
Posted by ag...@apache.org.
IGNITE-8258 Race in page memory in copyPageForCheckpoint, Ignite PDS 1 suite, test failed suite IgnitePdsPageReplacementTest.testPageReplacement. - Fixes #3821.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8caeb3dc
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8caeb3dc
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8caeb3dc
Branch: refs/heads/ignite-6083
Commit: 8caeb3dc19e623fcb082b31625489b3ae0c0cc5e
Parents: 10a4c48
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Fri Apr 20 15:28:31 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Fri Apr 20 15:28:31 2018 +0300
----------------------------------------------------------------------
.../cache/persistence/pagemem/PageMemoryImpl.java | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8caeb3dc/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 4463224..c684ce1 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -1144,7 +1144,7 @@ public class PageMemoryImpl implements PageMemoryEx {
assert success : "Page was pin when we resolve abs pointer, it can not be evicted";
- if (tmpRelPtr != INVALID_REL_PTR){
+ if (tmpRelPtr != INVALID_REL_PTR) {
PageHeader.tempBufferPointer(absPtr, INVALID_REL_PTR);
long tmpAbsPtr = checkpointPool.absolute(tmpRelPtr);
@@ -1158,9 +1158,6 @@ public class PageMemoryImpl implements PageMemoryEx {
checkpointPool.releaseFreePage(tmpRelPtr);
- // We pinned the page when allocated the temp buffer, release it now.
- PageHeader.releasePage(absPtr);
-
// Need release again because we pin page when resolve abs pointer,
// and page did not have tmp buffer page.
if (!pageSingleAcquire)
@@ -1171,9 +1168,6 @@ public class PageMemoryImpl implements PageMemoryEx {
copyInBuffer(absPtr, outBuf);
PageHeader.dirty(absPtr, false);
-
- // We pinned the page when resolve abs pointer.
- PageHeader.releasePage(absPtr);
}
assert PageIO.getType(outBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
@@ -1183,6 +1177,10 @@ public class PageMemoryImpl implements PageMemoryEx {
}
finally {
rwLock.writeUnlock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
+
+ // We pinned the page either when allocated the temp buffer, or when resolved abs pointer.
+ // Must release the page only after write unlock.
+ PageHeader.releasePage(absPtr);
}
}
[46/50] [abbrv] ignite git commit: IGNITE-7077 Fixed header licenses
Posted by ag...@apache.org.
IGNITE-7077 Fixed header licenses
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/fa841242
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/fa841242
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/fa841242
Branch: refs/heads/ignite-6083
Commit: fa841242a3ce7d5179e33663a724f3c3c07ef673
Parents: c82277e
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Fri Apr 20 19:17:32 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Apr 20 19:17:32 2018 +0300
----------------------------------------------------------------------
.../impl/optimization/SimpleExpressions.scala | 17 +++++++++++++++++
.../ignite/spark/IgniteOptimizationSpec.scala | 2 +-
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/fa841242/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
index a1c9458..10d021a 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.ignite.spark.impl.optimization
import java.text.SimpleDateFormat
http://git-wip-us.apache.org/repos/asf/ignite/blob/fa841242/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
index 00075f4..128f625 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
@@ -10,7 +10,7 @@
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expdfs or implied.
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
[16/50] [abbrv] ignite git commit: IGNITE-8140 Web Console: Fixed
code generation for large numbers in configuration params.
Posted by ag...@apache.org.
IGNITE-8140 Web Console: Fixed code generation for large numbers in configuration params.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/eda5fe73
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/eda5fe73
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/eda5fe73
Branch: refs/heads/ignite-6083
Commit: eda5fe736df920cad305540b500dcaf1a772f2d8
Parents: 56be24b
Author: Vasiliy Sisko <vs...@gridgain.com>
Authored: Wed Apr 18 10:44:44 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Apr 18 10:44:44 2018 +0700
----------------------------------------------------------------------
.../generator/ConfigurationGenerator.js | 106 +++++++++----------
1 file changed, 53 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/eda5fe73/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 45d9ad1..c5f82d3 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -720,8 +720,8 @@ export default class IgniteConfigurationGenerator {
.intProperty('connectionTimeout')
.intProperty('requestTimeout')
.stringProperty('signerOverride')
- .intProperty('connectionTTL')
- .intProperty('connectionMaxIdleMillis')
+ .longProperty('connectionTTL')
+ .longProperty('connectionMaxIdleMillis')
.emptyBeanProperty('dnsResolver')
.intProperty('responseMetadataCacheSize')
.emptyBeanProperty('secureRandom')
@@ -790,7 +790,7 @@ export default class IgniteConfigurationGenerator {
if (!available('2.3.0'))
return cfg;
- cfg.intProperty('longQueryWarningTimeout');
+ cfg.longProperty('longQueryWarningTimeout');
if (_.get(cluster, 'clientConnectorConfiguration.enabled') !== true)
return cfg;
@@ -837,7 +837,7 @@ export default class IgniteConfigurationGenerator {
colSpi.intProperty('activeJobsThreshold')
.intProperty('waitJobsThreshold')
- .intProperty('messageExpireTime')
+ .longProperty('messageExpireTime')
.intProperty('maximumStealingAttempts')
.boolProperty('stealingEnabled')
.emptyBeanProperty('externalCollisionListener')
@@ -892,9 +892,9 @@ export default class IgniteConfigurationGenerator {
.intProperty('sharedMemoryPort')
.intProperty('directBuffer')
.intProperty('directSendBuffer')
- .intProperty('idleConnectionTimeout')
- .intProperty('connectTimeout')
- .intProperty('maxConnectTimeout')
+ .longProperty('idleConnectionTimeout')
+ .longProperty('connectTimeout')
+ .longProperty('maxConnectTimeout')
.intProperty('reconnectCount')
.intProperty('socketSendBuffer')
.intProperty('socketReceiveBuffer')
@@ -903,19 +903,19 @@ export default class IgniteConfigurationGenerator {
.intProperty('tcpNoDelay')
.intProperty('ackSendThreshold')
.intProperty('unacknowledgedMessagesBufferSize')
- .intProperty('socketWriteTimeout')
+ .longProperty('socketWriteTimeout')
.intProperty('selectorsCount')
.emptyBeanProperty('addressResolver');
if (commSpi.nonEmpty())
cfg.beanProperty('communicationSpi', commSpi);
- cfg.intProperty('networkTimeout')
- .intProperty('networkSendRetryDelay')
+ cfg.longProperty('networkTimeout')
+ .longProperty('networkSendRetryDelay')
.intProperty('networkSendRetryCount');
if (available(['1.0.0', '2.3.0']))
- cfg.intProperty('discoveryStartupDelay');
+ cfg.longProperty('discoveryStartupDelay');
return cfg;
}
@@ -930,9 +930,9 @@ export default class IgniteConfigurationGenerator {
.stringProperty('host')
.intProperty('port')
.intProperty('portRange')
- .intProperty('idleTimeout')
- .intProperty('idleQueryCursorTimeout')
- .intProperty('idleQueryCursorCheckFrequency')
+ .longProperty('idleTimeout')
+ .longProperty('idleQueryCursorTimeout')
+ .longProperty('idleQueryCursorCheckFrequency')
.intProperty('receiveBufferSize')
.intProperty('sendBufferSize')
.intProperty('sendQueueLimit')
@@ -1020,11 +1020,11 @@ export default class IgniteConfigurationGenerator {
.intProperty('localPort')
.intProperty('localPortRange')
.emptyBeanProperty('addressResolver')
- .intProperty('socketTimeout')
- .intProperty('ackTimeout')
- .intProperty('maxAckTimeout')
- .intProperty('networkTimeout')
- .intProperty('joinTimeout')
+ .longProperty('socketTimeout')
+ .longProperty('ackTimeout')
+ .longProperty('maxAckTimeout')
+ .longProperty('networkTimeout')
+ .longProperty('joinTimeout')
.intProperty('threadPriority');
// Removed in ignite 2.0
@@ -1034,13 +1034,13 @@ export default class IgniteConfigurationGenerator {
.intProperty('maxMissedClientHeartbeats');
}
- discoSpi.intProperty('topHistorySize')
+ discoSpi.longProperty('topHistorySize')
.emptyBeanProperty('listener')
.emptyBeanProperty('dataExchange')
.emptyBeanProperty('metricsProvider')
.intProperty('reconnectCount')
- .intProperty('statisticsPrintFrequency')
- .intProperty('ipFinderCleanFrequency')
+ .longProperty('statisticsPrintFrequency')
+ .longProperty('ipFinderCleanFrequency')
.emptyBeanProperty('authenticator')
.intProperty('forceServerMode')
.intProperty('clientReconnectDisabled');
@@ -1090,8 +1090,8 @@ export default class IgniteConfigurationGenerator {
case 'Memory':
eventStorageBean = new Bean('org.apache.ignite.spi.eventstorage.memory.MemoryEventStorageSpi', 'eventStorage', eventStorage.Memory, clusterDflts.eventStorage.Memory);
- eventStorageBean.intProperty('expireAgeMs')
- .intProperty('expireCount')
+ eventStorageBean.longProperty('expireAgeMs')
+ .longProperty('expireCount')
.emptyBeanProperty('filter');
break;
@@ -1128,8 +1128,8 @@ export default class IgniteConfigurationGenerator {
// Since ignite 2.0
if (available('2.0.0')) {
- cfg.intProperty('failureDetectionTimeout')
- .intProperty('clientFailureDetectionTimeout');
+ cfg.longProperty('failureDetectionTimeout')
+ .longProperty('clientFailureDetectionTimeout');
}
_.forEach(cluster.failoverSpi, (spi) => {
@@ -1210,7 +1210,7 @@ export default class IgniteConfigurationGenerator {
if (plannerBean)
hadoopBean.beanProperty('mapReducePlanner', plannerBean);
- hadoopBean.intProperty('finishedJobInfoTtl')
+ hadoopBean.longProperty('finishedJobInfoTtl')
.intProperty('maxParallelTasks')
.intProperty('maxTaskQueueSize')
.arrayProperty('nativeLibraryNames', 'nativeLibraryNames', _.get(hadoop, 'nativeLibraryNames'));
@@ -1368,12 +1368,12 @@ export default class IgniteConfigurationGenerator {
memoryBean.intProperty('pageSize')
.intProperty('concurrencyLevel')
- .intProperty('systemCacheInitialSize')
- .intProperty('systemCacheMaxSize')
+ .longProperty('systemCacheInitialSize')
+ .longProperty('systemCacheMaxSize')
.stringProperty('defaultMemoryPolicyName');
if (memoryBean.valueOf('defaultMemoryPolicyName') === 'default')
- memoryBean.intProperty('defaultMemoryPolicySize');
+ memoryBean.longProperty('defaultMemoryPolicySize');
const policies = [];
@@ -1438,8 +1438,8 @@ export default class IgniteConfigurationGenerator {
storageBean.intProperty('pageSize')
.intProperty('concurrencyLevel')
- .intProperty('systemRegionInitialSize')
- .intProperty('systemRegionMaxSize');
+ .longProperty('systemRegionInitialSize')
+ .longProperty('systemRegionMaxSize');
const dfltDataRegionCfg = this.dataRegionConfiguration(_.get(dataStorageCfg, 'defaultDataRegionConfiguration'));
@@ -1461,7 +1461,7 @@ export default class IgniteConfigurationGenerator {
storageBean.varArgProperty('dataRegionConfigurations', 'dataRegionConfigurations', dataRegionCfgs, 'org.apache.ignite.configuration.DataRegionConfiguration');
storageBean.stringProperty('storagePath')
- .intProperty('checkpointFrequency')
+ .longProperty('checkpointFrequency')
.intProperty('checkpointThreads')
.enumProperty('checkpointWriteOrder')
.enumProperty('walMode')
@@ -1480,7 +1480,7 @@ export default class IgniteConfigurationGenerator {
.longProperty('lockWaitTime')
.intProperty('walThreadLocalBufferSize')
.intProperty('metricsSubIntervalCount')
- .intProperty('metricsRateTimeInterval')
+ .longProperty('metricsRateTimeInterval')
.longProperty('walAutoArchiveAfterInactivity')
.boolProperty('metricsEnabled')
.boolProperty('alwaysWriteFullPages')
@@ -1594,13 +1594,13 @@ export default class IgniteConfigurationGenerator {
// Generate metrics group.
static clusterMetrics(cluster, available, cfg = this.igniteConfigurationBean(cluster)) {
- cfg.intProperty('metricsExpireTime')
+ cfg.longProperty('metricsExpireTime')
.intProperty('metricsHistorySize')
- .intProperty('metricsLogFrequency');
+ .longProperty('metricsLogFrequency');
// Since ignite 2.0
if (available('2.0.0'))
- cfg.intProperty('metricsUpdateFrequency');
+ cfg.longProperty('metricsUpdateFrequency');
return cfg;
}
@@ -1630,7 +1630,7 @@ export default class IgniteConfigurationGenerator {
if (!available('2.1.0'))
return cfg;
- cfg.intProperty('longQueryWarningTimeout');
+ cfg.longProperty('longQueryWarningTimeout');
if (_.get(cluster, 'sqlConnectorConfiguration.enabled') !== true)
return cfg;
@@ -1663,7 +1663,7 @@ export default class IgniteConfigurationGenerator {
bean.stringProperty('persistentStorePath')
.boolProperty('metricsEnabled')
.boolProperty('alwaysWriteFullPages')
- .intProperty('checkpointingFrequency')
+ .longProperty('checkpointingFrequency')
.longProperty('checkpointingPageBufferSize')
.intProperty('checkpointingThreads')
.stringProperty('walStorePath')
@@ -1675,7 +1675,7 @@ export default class IgniteConfigurationGenerator {
.longProperty('walFsyncDelayNanos')
.intProperty('walRecordIteratorBufferSize')
.longProperty('lockWaitTime')
- .intProperty('rateTimeInterval')
+ .longProperty('rateTimeInterval')
.intProperty('tlbSize')
.intProperty('subIntervals');
@@ -1782,7 +1782,7 @@ export default class IgniteConfigurationGenerator {
.intProperty('igfsThreadPoolSize')
.intProperty('rebalanceThreadPoolSize')
.intProperty('utilityCacheThreadPoolSize', 'utilityCachePoolSize')
- .intProperty('utilityCacheKeepAliveTime')
+ .longProperty('utilityCacheKeepAliveTime')
.intProperty('asyncCallbackPoolSize')
.intProperty('stripedPoolSize');
@@ -1817,7 +1817,7 @@ export default class IgniteConfigurationGenerator {
bean.enumProperty('defaultTxConcurrency')
.enumProperty('defaultTxIsolation')
- .intProperty('defaultTxTimeout')
+ .longProperty('defaultTxTimeout')
.intProperty('pessimisticTxLogLinger')
.intProperty('pessimisticTxLogSize')
.boolProperty('txSerializableEnabled')
@@ -2070,7 +2070,7 @@ export default class IgniteConfigurationGenerator {
ccfg.enumProperty('memoryMode');
if (ccfg.valueOf('memoryMode') !== 'OFFHEAP_VALUES')
- ccfg.intProperty('offHeapMaxMemory');
+ ccfg.longProperty('offHeapMaxMemory');
}
// Since ignite 2.0
@@ -2105,7 +2105,7 @@ export default class IgniteConfigurationGenerator {
if (available(['1.0.0', '2.0.0']))
ccfg.intProperty('sqlOnheapRowCacheSize');
- ccfg.intProperty('longQueryWarningTimeout')
+ ccfg.longProperty('longQueryWarningTimeout')
.arrayProperty('indexedTypes', 'indexedTypes', indexedTypes, 'java.lang.Class')
.intProperty('queryDetailMetricsSize')
.arrayProperty('sqlFunctionClasses', 'sqlFunctionClasses', cache.sqlFunctionClasses, 'java.lang.Class');
@@ -2225,7 +2225,7 @@ export default class IgniteConfigurationGenerator {
ccfg.boolProperty('writeBehindEnabled')
.intProperty('writeBehindBatchSize')
.intProperty('writeBehindFlushSize')
- .intProperty('writeBehindFlushFrequency')
+ .longProperty('writeBehindFlushFrequency')
.intProperty('writeBehindFlushThreadCount');
// Since ignite 2.0
@@ -2239,7 +2239,7 @@ export default class IgniteConfigurationGenerator {
// Generate cache concurrency control group.
static cacheConcurrency(cache, available, ccfg = this.cacheConfigurationBean(cache)) {
ccfg.intProperty('maxConcurrentAsyncOperations')
- .intProperty('defaultLockTimeout');
+ .longProperty('defaultLockTimeout');
// Removed in ignite 2.0
if (available(['1.0.0', '2.0.0']))
@@ -2297,11 +2297,11 @@ export default class IgniteConfigurationGenerator {
ccfg.enumProperty('rebalanceMode')
.intProperty('rebalanceThreadPoolSize')
.intProperty('rebalanceBatchSize')
- .intProperty('rebalanceBatchesPrefetchCount')
+ .longProperty('rebalanceBatchesPrefetchCount')
.intProperty('rebalanceOrder')
- .intProperty('rebalanceDelay')
- .intProperty('rebalanceTimeout')
- .intProperty('rebalanceThrottle');
+ .longProperty('rebalanceDelay')
+ .longProperty('rebalanceTimeout')
+ .longProperty('rebalanceThrottle');
}
if (ccfg.includes('igfsAffinnityGroupSize')) {
@@ -2456,8 +2456,8 @@ export default class IgniteConfigurationGenerator {
static igfsFragmentizer(igfs, cfg = this.igfsConfigurationBean(igfs)) {
if (igfs.fragmentizerEnabled) {
cfg.intProperty('fragmentizerConcurrentFiles')
- .intProperty('fragmentizerThrottlingBlockLength')
- .intProperty('fragmentizerThrottlingDelay');
+ .longProperty('fragmentizerThrottlingBlockLength')
+ .longProperty('fragmentizerThrottlingDelay');
}
else
cfg.boolProperty('fragmentizerEnabled');
@@ -2490,7 +2490,7 @@ export default class IgniteConfigurationGenerator {
if (available(['1.0.0', '2.0.0']))
cfg.intProperty('maxSpaceSize');
- cfg.intProperty('maximumTaskRangeLength')
+ cfg.longProperty('maximumTaskRangeLength')
.intProperty('managementPort')
.intProperty('perNodeBatchSize')
.intProperty('perNodeParallelBatchCount')
[23/50] [abbrv] ignite git commit: IGNITE-6681 .NET: QueryMetrics
Posted by ag...@apache.org.
IGNITE-6681 .NET: QueryMetrics
This closes #3839
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ea8bfac6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ea8bfac6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ea8bfac6
Branch: refs/heads/ignite-6083
Commit: ea8bfac60c0e6f4e4b456f7ea9b3ab5af97a3c7e
Parents: 9d184c9
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Wed Apr 18 17:16:17 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Wed Apr 18 17:16:17 2018 +0300
----------------------------------------------------------------------
.../platform/cache/PlatformCache.java | 37 ++++
.../ExpiryCacheHolderTest.cs | 10 +
.../Apache.Ignite.Core.Tests.csproj | 1 +
.../ApiParity/CacheParityTest.cs | 2 -
.../Cache/CacheQueryMetricsTest.cs | 190 +++++++++++++++++++
.../Cache/CacheTestAsyncWrapper.cs | 10 +
.../Apache.Ignite.Core.csproj | 2 +
.../dotnet/Apache.Ignite.Core/Cache/ICache.cs | 11 ++
.../Apache.Ignite.Core/Cache/IQueryMetrics.cs | 65 +++++++
.../Apache.Ignite.Core/Impl/Cache/CacheImpl.cs | 17 ++
.../Apache.Ignite.Core/Impl/Cache/CacheOp.cs | 4 +-
.../Impl/Cache/QueryMetricsImpl.cs | 71 +++++++
12 files changed, 417 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ea8bfac6/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
index 19dac83..c78f2d6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
@@ -39,6 +39,7 @@ import org.apache.ignite.cache.CacheMetrics;
import org.apache.ignite.cache.CachePartialUpdateException;
import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.cache.query.Query;
+import org.apache.ignite.cache.query.QueryMetrics;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.SqlQuery;
@@ -326,6 +327,12 @@ public class PlatformCache extends PlatformAbstractTarget {
/** */
public static final int OP_GET_LOST_PARTITIONS = 84;
+ /** */
+ public static final int OP_QUERY_METRICS = 85;
+
+ /** */
+ public static final int OP_RESET_QUERY_METRICS = 86;
+
/** Underlying JCache in binary mode. */
private final IgniteCacheProxy cache;
@@ -988,6 +995,14 @@ public class PlatformCache extends PlatformAbstractTarget {
break;
+ case OP_QUERY_METRICS: {
+ QueryMetrics metrics = cache.queryMetrics();
+
+ writeQueryMetrics(writer, metrics);
+
+ break;
+ }
+
default:
super.processOutStream(type, writer);
}
@@ -1094,6 +1109,11 @@ public class PlatformCache extends PlatformAbstractTarget {
cache.removeAll();
return TRUE;
+
+ case OP_RESET_QUERY_METRICS:
+ cache.resetQueryMetrics();
+
+ return TRUE;
}
return super.processInLongOutLong(type, val);
}
@@ -1486,6 +1506,23 @@ public class PlatformCache extends PlatformAbstractTarget {
}
/**
+ * Writes query metrics.
+ *
+ * @param writer Writer.
+ * @param metrics Metrics.
+ */
+ public static void writeQueryMetrics(BinaryRawWriter writer, QueryMetrics metrics) {
+ assert writer != null;
+ assert metrics != null;
+
+ writer.writeLong(metrics.minimumTime());
+ writer.writeLong(metrics.maximumTime());
+ writer.writeDouble(metrics.averageTime());
+ writer.writeInt(metrics.executions());
+ writer.writeInt(metrics.fails());
+ }
+
+ /**
* Writes error with EntryProcessorException cause.
*/
private static class GetAllWriter implements PlatformFutureUtils.Writer {
http://git-wip-us.apache.org/repos/asf/ignite/blob/ea8bfac6/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/ExpiryCacheHolderTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/ExpiryCacheHolderTest.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/ExpiryCacheHolderTest.cs
index 93e3a53..34d4b4d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/ExpiryCacheHolderTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/ExpiryCacheHolderTest.cs
@@ -513,6 +513,16 @@ namespace Apache.Ignite.AspNet.Tests
{
throw new NotImplementedException();
}
+
+ public IQueryMetrics GetQueryMetrics()
+ {
+ throw new NotImplementedException();
+ }
+
+ public void ResetQueryMetrics()
+ {
+ throw new NotImplementedException();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ea8bfac6/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
index 8cd64cc..a04509a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
@@ -106,6 +106,7 @@
<Compile Include="Binary\Serializable\GenericCollectionsTest.cs" />
<Compile Include="Cache\Affinity\AffinityAttributeTest.cs" />
<Compile Include="Cache\CacheCreateTest.cs" />
+ <Compile Include="Cache\CacheQueryMetricsTest.cs" />
<Compile Include="Cache\DataRegionMetricsTest.cs" />
<Compile Include="Cache\DataStorageMetricsTest.cs" />
<Compile Include="Cache\PersistenceTest.cs" />
http://git-wip-us.apache.org/repos/asf/ignite/blob/ea8bfac6/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs
index 68b822c..6471ec5 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs
@@ -51,8 +51,6 @@ namespace Apache.Ignite.Core.Tests.ApiParity
/** Members that are missing on .NET side and should be added in future. */
private static readonly string[] MissingMembers =
{
- "queryMetrics", // IGNITE-6681
- "resetQueryMetrics", // IGNITE-6681
"queryDetailMetrics", // IGNITE-6680
"resetQueryDetailMetrics", // IGNITE-6680
"sizeLong", // IGNITE-6563
http://git-wip-us.apache.org/repos/asf/ignite/blob/ea8bfac6/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheQueryMetricsTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheQueryMetricsTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheQueryMetricsTest.cs
new file mode 100644
index 0000000..5b85328
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheQueryMetricsTest.cs
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// ReSharper disable UnusedAutoPropertyAccessor.Local
+namespace Apache.Ignite.Core.Tests.Cache
+{
+ extern alias ExamplesDll;
+ using System.Collections.Generic;
+ using System.Diagnostics.CodeAnalysis;
+ using System.Linq;
+ using Apache.Ignite.Core.Cache;
+ using Apache.Ignite.Core.Cache.Configuration;
+ using Apache.Ignite.Core.Cache.Query;
+ using Apache.Ignite.Core.Common;
+ using Apache.Ignite.Linq;
+ using NUnit.Framework;
+
+ /// <summary>
+ /// Tests query metrics propagation.
+ /// </summary>
+ public class CacheQueryMetricsTest
+ {
+ /// <summary>
+ /// Fixture set up.
+ /// </summary>
+ [TestFixtureSetUp]
+ public void FixtureSetUp()
+ {
+ Ignition.Start(TestUtils.GetTestConfiguration());
+ }
+
+ /// <summary>
+ /// Fixture tear down.
+ /// </summary>
+ [TestFixtureTearDown]
+ public void FixtureTearDown()
+ {
+ Ignition.StopAll(true);
+ }
+
+ /// <summary>
+ /// Tests query metrics.
+ /// </summary>
+ [Test]
+ public void TestQueryMetrics()
+ {
+ var cache = GetCache();
+
+ cache.ResetQueryMetrics();
+
+ ExecuteTwoQueries(cache);
+
+ CheckMetrics(cache.GetQueryMetrics());
+ }
+
+ /// <summary>
+ /// Tests failed queries count.
+ /// </summary>
+ [Test]
+ public void TestQueryFails()
+ {
+ var cache = GetCache();
+
+ Assert.Throws<IgniteException>(() => cache.Query(new SqlFieldsQuery("select * from NOT_A_TABLE")));
+
+ Assert.AreEqual(1, cache.GetQueryMetrics().Fails, "Check Fails count.");
+ }
+
+ /// <summary>
+ /// Tests query metrics reset.
+ /// </summary>
+ [Test]
+ public void TestQueryMetricsReset()
+ {
+ var cache = GetCache();
+
+ ExecuteTwoQueries(cache);
+
+ cache.ResetQueryMetrics();
+
+ IQueryMetrics metrics = cache.GetQueryMetrics();
+
+ Assert.AreEqual(0, metrics.Executions, "Check Executions count.");
+ Assert.AreEqual(0, metrics.Fails, "Check Fails count.");
+ Assert.AreEqual(0, metrics.MinimumTime, "Check MinimumTime.");
+ Assert.AreEqual(0, metrics.MaximumTime, "Check MaximumTime.");
+ Assert.AreEqual(0, metrics.AverageTime, 0.1, "Check AverageTime.");
+
+ ExecuteTwoQueries(cache);
+
+ CheckMetrics(cache.GetQueryMetrics());
+ }
+
+ /// <summary>
+ /// Get Cache instance.
+ /// </summary>
+ /// <returns>Cache instance.</returns>
+ private ICache<int, Person> GetCache()
+ {
+ var ignite = Ignition.GetIgnite();
+
+ var cache = ignite.GetOrCreateCache<int, Person>(GetCacheConfiguration());
+
+ if (cache.GetSize() == 0)
+ {
+ Person person = new Person()
+ {
+ Name = "Adam",
+ Age = 35000
+ };
+
+ cache.Put(1, person);
+ }
+
+ return cache;
+ }
+
+ /// <summary>
+ /// Check cache configuration.
+ /// </summary>
+ /// <returns>Cache configuration.</returns>
+ private CacheConfiguration GetCacheConfiguration()
+ {
+ return new CacheConfiguration("cacheName")
+ {
+ EnableStatistics = true,
+
+ QueryEntities = new List<QueryEntity>()
+ {
+ new QueryEntity(typeof(int), typeof(Person))
+ }
+ };
+ }
+
+ /// <summary>
+ /// Person.
+ /// </summary>
+ private class Person
+ {
+ [QuerySqlField]
+ public string Name { get; set; }
+
+ [QuerySqlField]
+ public int Age { get; set; }
+ }
+
+ /// <summary>
+ /// Execute two queries.
+ /// </summary>
+ /// <param name="cache">Cache instance.</param>
+ [SuppressMessage("ReSharper", "ReturnValueOfPureMethodIsNotUsed")]
+ private static void ExecuteTwoQueries(ICache<int, Person> cache)
+ {
+ IQueryable<ICacheEntry<int, Person>> queryable = cache.AsCacheQueryable();
+
+ queryable.Count(p => p.Value.Age > 0);
+
+ queryable.Count(p => p.Value.Age > 1000);
+ }
+
+ /// <summary>
+ /// Check metrics after two queries are executed.
+ /// </summary>
+ /// <param name="metrics">Query metrics.</param>
+ private static void CheckMetrics(IQueryMetrics metrics)
+ {
+ Assert.AreEqual(2, metrics.Executions, "Check Executions count.");
+ Assert.AreEqual(0, metrics.Fails, "Check Fails count.");
+
+ Assert.GreaterOrEqual(metrics.MinimumTime, 0, "Check MinimumTime.");
+ Assert.GreaterOrEqual(metrics.MaximumTime, metrics.MinimumTime, "Check MaximumTime.");
+
+ Assert.AreEqual((double)(metrics.MinimumTime + metrics.MaximumTime) / 2, metrics.AverageTime, 1, "Check AverageTime.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ea8bfac6/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheTestAsyncWrapper.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheTestAsyncWrapper.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheTestAsyncWrapper.cs
index c4e50d1..0fe16a5 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheTestAsyncWrapper.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheTestAsyncWrapper.cs
@@ -573,6 +573,16 @@ namespace Apache.Ignite.Core.Tests.Cache
{
return GetEnumerator();
}
+
+ public IQueryMetrics GetQueryMetrics()
+ {
+ return _cache.GetQueryMetrics();
+ }
+
+ public void ResetQueryMetrics()
+ {
+ _cache.ResetQueryMetrics();
+ }
}
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/ea8bfac6/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
index 93c45c3..f217a0d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
@@ -54,6 +54,7 @@
<Compile Include="Binary\TimestampAttribute.cs" />
<Compile Include="Cache\Configuration\CacheKeyConfiguration.cs" />
<Compile Include="Cache\Configuration\DataPageEvictionMode.cs" />
+ <Compile Include="Cache\IQueryMetrics.cs" />
<Compile Include="Client\Cache\CacheClientConfiguration.cs" />
<Compile Include="Client\IgniteClientConfigurationSection.cs" />
<Compile Include="Client\ISslStreamFactory.cs" />
@@ -72,6 +73,7 @@
<Compile Include="Client\IgniteClientException.cs" />
<Compile Include="Client\IIgniteClient.cs" />
<Compile Include="Common\ExceptionFactory.cs" />
+ <Compile Include="Impl\Cache\QueryMetricsImpl.cs" />
<Compile Include="Impl\Common\TaskRunner.cs" />
<Compile Include="Ssl\ISslContextFactory.cs" />
<Compile Include="Configuration\Package-Info.cs" />
http://git-wip-us.apache.org/repos/asf/ignite/blob/ea8bfac6/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICache.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICache.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICache.cs
index 8a9fe59..8f52e03 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICache.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICache.cs
@@ -907,5 +907,16 @@ namespace Apache.Ignite.Core.Cache
/// and <see cref="IIgnite.ResetLostPartitions(IEnumerable{string})"/>.
/// </summary>
ICollection<int> GetLostPartitions();
+
+ /// <summary>
+ /// Gets query metrics.
+ /// </summary>
+ /// <returns>Query metrics.</returns>
+ IQueryMetrics GetQueryMetrics();
+
+ /// <summary>
+ /// Reset query metrics.
+ /// </summary>
+ void ResetQueryMetrics();
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ea8bfac6/modules/platforms/dotnet/Apache.Ignite.Core/Cache/IQueryMetrics.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/IQueryMetrics.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/IQueryMetrics.cs
new file mode 100644
index 0000000..232bf99
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/IQueryMetrics.cs
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Core.Cache
+{
+ /// <summary>
+ /// Cache query metrics used to obtain statistics on query.
+ /// </summary>
+ public interface IQueryMetrics
+ {
+ /// <summary>
+ /// Gets minimum execution time of query.
+ /// </summary>
+ /// <returns>
+ /// Minimum execution time of query.
+ /// </returns>
+ long MinimumTime { get; }
+
+ /// <summary>
+ /// Gets maximum execution time of query.
+ /// </summary>
+ /// <returns>
+ /// Maximum execution time of query.
+ /// </returns>
+ long MaximumTime { get; }
+
+ /// <summary>
+ /// Gets average execution time of query.
+ /// </summary>
+ /// <returns>
+ /// Average execution time of query.
+ /// </returns>
+ double AverageTime { get; }
+
+ /// <summary>
+ /// Gets total number execution of query.
+ /// </summary>
+ /// <returns>
+ /// Number of executions.
+ /// </returns>
+ int Executions { get; }
+
+ /// <summary>
+ /// Gets total number of times a query execution failed.
+ /// </summary>
+ /// <returns>
+ /// Total number of times a query execution failed.
+ /// </returns>
+ int Fails { get; }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/ea8bfac6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
index 6661161..6e46f4d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
@@ -1408,5 +1408,22 @@ namespace Apache.Ignite.Core.Impl.Cache
if (_txManager != null)
_txManager.StartTx();
}
+
+ /** <inheritdoc /> */
+ public IQueryMetrics GetQueryMetrics()
+ {
+ return DoInOp((int)CacheOp.QueryMetrics, stream =>
+ {
+ IBinaryRawReader reader = Marshaller.StartUnmarshal(stream, false);
+
+ return new QueryMetricsImpl(reader);
+ });
+ }
+
+ /** <inheritdoc /> */
+ public void ResetQueryMetrics()
+ {
+ DoOutInOp((int)CacheOp.ResetQueryMetrics);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ea8bfac6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheOp.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheOp.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheOp.cs
index b3cbd95..00a7135 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheOp.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheOp.cs
@@ -104,6 +104,8 @@ namespace Apache.Ignite.Core.Impl.Cache
PutIfAbsentAsync = 81,
Extension = 82,
GlobalMetrics = 83,
- GetLostPartitions = 84
+ GetLostPartitions = 84,
+ QueryMetrics = 85,
+ ResetQueryMetrics = 86
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/ea8bfac6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/QueryMetricsImpl.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/QueryMetricsImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/QueryMetricsImpl.cs
new file mode 100644
index 0000000..d71d40e0
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/QueryMetricsImpl.cs
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Core.Impl.Cache
+{
+ using Apache.Ignite.Core.Binary;
+ using Apache.Ignite.Core.Cache;
+
+ /// <summary>
+ /// Cache query metrics used to obtain statistics on query.
+ /// </summary>
+ internal class QueryMetricsImpl : IQueryMetrics
+ {
+ /** */
+ private readonly long _minimumTime;
+
+ /** */
+ private readonly long _maximumTime;
+
+ /** */
+ private readonly double _averageTime;
+
+ /** */
+ private readonly int _executions;
+
+ /** */
+ private readonly int _fails;
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="QueryMetricsImpl"/> class.
+ /// </summary>
+ /// <param name="reader">The reader.</param>
+ public QueryMetricsImpl(IBinaryRawReader reader)
+ {
+ _minimumTime = reader.ReadLong();
+ _maximumTime = reader.ReadLong();
+ _averageTime = reader.ReadDouble();
+ _executions = reader.ReadInt();
+ _fails = reader.ReadInt();
+ }
+
+ /** <inheritDoc /> */
+ public long MinimumTime { get { return _minimumTime; } }
+
+ /** <inheritDoc /> */
+ public long MaximumTime { get { return _maximumTime; } }
+
+ /** <inheritDoc /> */
+ public double AverageTime { get { return _averageTime; } }
+
+ /** <inheritDoc /> */
+ public int Executions { get { return _executions; } }
+
+ /** <inheritDoc /> */
+ public int Fails { get { return _fails; } }
+ }
+}
\ No newline at end of file
[29/50] [abbrv] ignite git commit: IGNITE-8302 Licenses fix for
Reduce test IO consumption in case Direct IO
Posted by ag...@apache.org.
IGNITE-8302 Licenses fix for Reduce test IO consumption in case Direct IO
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e7716f45
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e7716f45
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e7716f45
Branch: refs/heads/ignite-6083
Commit: e7716f454cea0af8acba82b274bd4715badc6fc6
Parents: d853ebb
Author: dpavlov <dp...@apache.org>
Authored: Wed Apr 18 21:16:35 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Wed Apr 18 21:16:35 2018 +0300
----------------------------------------------------------------------
...eNativeIoPdsRecoveryAfterFileCorruptionTest.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e7716f45/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java
----------------------------------------------------------------------
diff --git a/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java b/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java
index 4d65bf3..41b6c79 100644
--- a/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java
+++ b/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.ignite.internal.processors.cache.persistence;
/**
[02/50] [abbrv] ignite git commit: IGNITE-8285 Web console: Removed
debug output.
Posted by ag...@apache.org.
IGNITE-8285 Web console: Removed debug output.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8c80dce3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8c80dce3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8c80dce3
Branch: refs/heads/ignite-6083
Commit: 8c80dce3b8601ffc123cb6dec0255b66e4576272
Parents: e76fcb4
Author: Ilya Borisov <kl...@gmail.com>
Authored: Tue Apr 17 15:46:10 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Apr 17 15:46:10 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/app/components/page-configure/index.js | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8c80dce3/modules/web-console/frontend/app/components/page-configure/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/index.js b/modules/web-console/frontend/app/components/page-configure/index.js
index 3209ede..34b8cfe 100644
--- a/modules/web-console/frontend/app/components/page-configure/index.js
+++ b/modules/web-console/frontend/app/components/page-configure/index.js
@@ -164,7 +164,6 @@ export default angular
state: actionsWindow.filter((a) => !actions.includes(a)).reduce(ConfigureState._combinedReducer, {})
};
})
- .debug('UNDOED')
.do((a) => ConfigureState.dispatchAction(a))
.subscribe();
ConfigEffects.connect();
[14/50] [abbrv] ignite git commit: IGNITE-8274 sqlline.sh script uses
JAVA_HOME now
Posted by ag...@apache.org.
IGNITE-8274 sqlline.sh script uses JAVA_HOME now
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c3ff274d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c3ff274d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c3ff274d
Branch: refs/heads/ignite-6083
Commit: c3ff274d4a9a8f228fabfc4bfbe85823e0cc0ec8
Parents: c000fbc
Author: oleg-ostanin <oo...@gridagin.com>
Authored: Tue Apr 17 20:58:53 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Apr 17 20:58:53 2018 +0300
----------------------------------------------------------------------
modules/sqlline/bin/sqlline.sh | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c3ff274d/modules/sqlline/bin/sqlline.sh
----------------------------------------------------------------------
diff --git a/modules/sqlline/bin/sqlline.sh b/modules/sqlline/bin/sqlline.sh
index 5745aea..552440d 100644
--- a/modules/sqlline/bin/sqlline.sh
+++ b/modules/sqlline/bin/sqlline.sh
@@ -36,6 +36,11 @@ SCRIPTS_HOME="${IGNITE_HOME_TMP}/bin"
source "${SCRIPTS_HOME}"/include/functions.sh
#
+# Discover path to Java executable and check it's version.
+#
+checkJava
+
+#
# Discover IGNITE_HOME environment variable.
#
setIgniteHome
@@ -51,4 +56,4 @@ CP="${IGNITE_LIBS}"
CP="${CP}${SEP}${IGNITE_HOME_TMP}/bin/include/sqlline/*"
-java -cp ${CP} sqlline.SqlLine -d org.apache.ignite.IgniteJdbcThinDriver $@
\ No newline at end of file
+"$JAVA" -cp ${CP} sqlline.SqlLine -d org.apache.ignite.IgniteJdbcThinDriver $@
\ No newline at end of file
[09/50] [abbrv] ignite git commit: IGNITE-8257
GridFutureAdapterSelfTest#testChaining flaky-fails on TC (rarely)
Posted by ag...@apache.org.
IGNITE-8257 GridFutureAdapterSelfTest#testChaining flaky-fails on TC (rarely)
Signed-off-by: Anton Vinogradov <av...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1b3a292a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1b3a292a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1b3a292a
Branch: refs/heads/ignite-6083
Commit: 1b3a292a013527b53b13ad222da9ea0eabfff891
Parents: 63dc75b
Author: Vitaliy Biryukov <Bi...@gmail.com>
Authored: Tue Apr 17 16:47:10 2018 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Tue Apr 17 16:47:10 2018 +0300
----------------------------------------------------------------------
.../ignite/internal/util/future/GridFutureAdapterSelfTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b3a292a/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java
index 4bc9f01..89f3a03 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java
@@ -280,7 +280,7 @@ public class GridFutureAdapterSelfTest extends GridCommonAbstractTest {
fut.onDone("result");
- assertEquals("result", chain.get(1));
+ assertEquals("result", exec == null ? chain.get(1) : chain.get());
// Test exception re-thrown.
[26/50] [abbrv] ignite git commit: IGNITE-8116 Fixed historical
rebalance from WAL
Posted by ag...@apache.org.
IGNITE-8116 Fixed historical rebalance from WAL
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5c718543
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5c718543
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5c718543
Branch: refs/heads/ignite-6083
Commit: 5c718543695fdc3006236652e878e25ab9946de5
Parents: b058bf6
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Wed Apr 18 19:41:44 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Apr 18 19:41:44 2018 +0300
----------------------------------------------------------------------
.../cache/IgniteCacheOffheapManagerImpl.java | 28 ++--
.../dht/preloader/GridDhtPartitionDemander.java | 118 ++++++++-----
.../dht/preloader/GridDhtPartitionSupplier.java | 54 ++++--
.../dht/preloader/GridDhtPreloader.java | 4 +-
.../IgniteDhtPartitionsToReloadMap.java | 2 +-
.../persistence/GridCacheOffheapManager.java | 82 ++++++++--
modules/core/src/test/config/log4j-test.xml | 6 -
...PdsAtomicCacheHistoricalRebalancingTest.java | 40 +++++
.../IgnitePdsCacheRebalancingAbstractTest.java | 32 ++--
...nitePdsTxCacheHistoricalRebalancingTest.java | 39 +++++
.../db/wal/IgniteWalRebalanceTest.java | 164 +++++++++++++++++++
...lientQueryReplicatedNodeRestartSelfTest.java | 1 -
.../IgnitePdsWithIndexingCoreTestSuite.java | 7 +
13 files changed, 467 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index f8cc86f..5c78eb5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -861,7 +861,8 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
}
@Override protected void onClose() throws IgniteCheckedException {
- assert loc != null && loc.state() == OWNING && loc.reservations() > 0;
+ assert loc != null && loc.state() == OWNING && loc.reservations() > 0
+ : "Partition should be in OWNING state and has at least 1 reservation: " + loc;
loc.release();
}
@@ -874,36 +875,37 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
throws IgniteCheckedException {
final TreeMap<Integer, GridCloseableIterator<CacheDataRow>> iterators = new TreeMap<>();
- Set<Integer> missing = null;
+
+ Set<Integer> missing = new HashSet<>();
for (Integer p : parts.fullSet()) {
GridCloseableIterator<CacheDataRow> partIter = reservedIterator(p, topVer);
if (partIter == null) {
- if (missing == null)
- missing = new HashSet<>();
-
missing.add(p);
+
+ continue;
}
- else
- iterators.put(p, partIter);
+
+ iterators.put(p, partIter);
}
- IgniteRebalanceIterator iter = new IgniteRebalanceIteratorImpl(iterators, historicalIterator(parts.historicalMap()));
+ IgniteHistoricalIterator historicalIterator = historicalIterator(parts.historicalMap(), missing);
- if (missing != null) {
- for (Integer p : missing)
- iter.setPartitionMissing(p);
- }
+ IgniteRebalanceIterator iter = new IgniteRebalanceIteratorImpl(iterators, historicalIterator);
+
+ for (Integer p : missing)
+ iter.setPartitionMissing(p);
return iter;
}
/**
* @param partCntrs Partition counters map.
+ * @param missing Set of partitions need to populate if partition is missing or failed to reserve.
* @return Historical iterator.
*/
- @Nullable protected IgniteHistoricalIterator historicalIterator(CachePartitionPartialCountersMap partCntrs)
+ @Nullable protected IgniteHistoricalIterator historicalIterator(CachePartitionPartialCountersMap partCntrs, Set<Integer> missing)
throws IgniteCheckedException {
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index dc4bfe9..c94f511 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -459,7 +459,9 @@ public class GridDhtPartitionDemander {
+ ", topology=" + fut.topologyVersion() + ", rebalanceId=" + fut.rebalanceId + "]");
}
- int stripes = ctx.gridConfig().getRebalanceThreadPoolSize();
+ int totalStripes = ctx.gridConfig().getRebalanceThreadPoolSize();
+
+ int stripes = totalStripes;
final List<IgniteDhtDemandedPartitionsMap> stripePartitions = new ArrayList<>(stripes);
for (int i = 0; i < stripes; i++)
@@ -467,7 +469,7 @@ public class GridDhtPartitionDemander {
// Reserve one stripe for historical partitions.
if (parts.hasHistorical()) {
- stripePartitions.add(stripes - 1, new IgniteDhtDemandedPartitionsMap(parts.historicalMap(), null));
+ stripePartitions.set(stripes - 1, new IgniteDhtDemandedPartitionsMap(parts.historicalMap(), null));
if (stripes > 1)
stripes--;
@@ -478,7 +480,7 @@ public class GridDhtPartitionDemander {
for (int i = 0; it.hasNext(); i++)
stripePartitions.get(i % stripes).addFull(it.next());
- for (int stripe = 0; stripe < stripes; stripe++) {
+ for (int stripe = 0; stripe < totalStripes; stripe++) {
if (!stripePartitions.get(stripe).isEmpty()) {
// Create copy of demand message with new striped partitions map.
final GridDhtPartitionDemandMessage demandMsg = d.withNewPartitionsMap(stripePartitions.get(stripe));
@@ -489,23 +491,27 @@ public class GridDhtPartitionDemander {
final int topicId = stripe;
- Runnable initDemandRequestTask = () -> {
+ IgniteInternalFuture<?> clearAllFuture = clearFullPartitions(fut, demandMsg.partitions().fullSet());
+
+ // Start rebalancing after clearing full partitions is finished.
+ clearAllFuture.listen(f -> ctx.kernalContext().closure().runLocalSafe(() -> {
+ if (fut.isDone())
+ return;
+
try {
- if (!fut.isDone()) {
- ctx.io().sendOrderedMessage(node, rebalanceTopics.get(topicId),
- demandMsg.convertIfNeeded(node.version()), grp.ioPolicy(), demandMsg.timeout());
-
- // Cleanup required in case partitions demanded in parallel with cancellation.
- synchronized (fut) {
- if (fut.isDone())
- fut.cleanupRemoteContexts(node.id());
- }
+ ctx.io().sendOrderedMessage(node, rebalanceTopics.get(topicId),
+ demandMsg.convertIfNeeded(node.version()), grp.ioPolicy(), demandMsg.timeout());
- if (log.isDebugEnabled())
- log.debug("Requested rebalancing [from node=" + node.id() + ", listener index=" +
- topicId + ", partitions count=" + stripePartitions.get(topicId).size() +
- " (" + stripePartitions.get(topicId).partitionsList() + ")]");
+ // Cleanup required in case partitions demanded in parallel with cancellation.
+ synchronized (fut) {
+ if (fut.isDone())
+ fut.cleanupRemoteContexts(node.id());
}
+
+ if (log.isDebugEnabled())
+ log.debug("Requested rebalancing [from node=" + node.id() + ", listener index=" +
+ topicId + " " + demandMsg.rebalanceId() + ", partitions count=" + stripePartitions.get(topicId).size() +
+ " (" + stripePartitions.get(topicId).partitionsList() + ")]");
}
catch (IgniteCheckedException e1) {
ClusterTopologyCheckedException cause = e1.getCause(ClusterTopologyCheckedException.class);
@@ -522,31 +528,26 @@ public class GridDhtPartitionDemander {
fut.cancel();
}
- };
-
- awaitClearingAndStartRebalance(fut, demandMsg, initDemandRequestTask);
+ }, true));
}
}
}
}
/**
- * Awaits partitions clearing for full partitions and sends initial demand request
- * after all partitions are cleared and safe to consume data.
+ * Creates future which will be completed when all {@code fullPartitions} are cleared.
*
* @param fut Rebalance future.
- * @param demandMessage Initial demand message which contains set of full partitions to await.
- * @param initDemandRequestTask Task which sends initial demand request.
+ * @param fullPartitions Set of full partitions need to be cleared.
+ * @return Future which will be completed when given partitions are cleared.
*/
- private void awaitClearingAndStartRebalance(RebalanceFuture fut,
- GridDhtPartitionDemandMessage demandMessage,
- Runnable initDemandRequestTask) {
- Set<Integer> fullPartitions = demandMessage.partitions().fullSet();
+ private IgniteInternalFuture<?> clearFullPartitions(RebalanceFuture fut, Set<Integer> fullPartitions) {
+ final GridFutureAdapter clearAllFuture = new GridFutureAdapter();
if (fullPartitions.isEmpty()) {
- ctx.kernalContext().closure().runLocalSafe(initDemandRequestTask, true);
+ clearAllFuture.onDone();
- return;
+ return clearAllFuture;
}
for (GridCacheContext cctx : grp.caches()) {
@@ -560,16 +561,19 @@ public class GridDhtPartitionDemander {
final AtomicInteger clearingPartitions = new AtomicInteger(fullPartitions.size());
for (int partId : fullPartitions) {
- if (fut.isDone())
- return;
+ if (fut.isDone()) {
+ clearAllFuture.onDone();
+
+ return clearAllFuture;
+ }
GridDhtLocalPartition part = grp.topology().localPartition(partId);
if (part != null && part.state() == MOVING) {
part.onClearFinished(f -> {
- // Cancel rebalance if partition clearing was failed.
- if (f.error() != null) {
- if (!fut.isDone()) {
+ if (!fut.isDone()) {
+ // Cancel rebalance if partition clearing was failed.
+ if (f.error() != null) {
for (GridCacheContext cctx : grp.caches()) {
if (cctx.statisticsEnabled()) {
final CacheMetricsImpl metrics = cctx.cache().metrics0();
@@ -581,30 +585,54 @@ public class GridDhtPartitionDemander {
log.error("Unable to await partition clearing " + part, f.error());
fut.cancel();
+
+ clearAllFuture.onDone(f.error());
}
- }
- else {
- if (!fut.isDone()) {
- int existed = clearingPartitions.decrementAndGet();
+ else {
+ int remaining = clearingPartitions.decrementAndGet();
for (GridCacheContext cctx : grp.caches()) {
if (cctx.statisticsEnabled()) {
final CacheMetricsImpl metrics = cctx.cache().metrics0();
- metrics.rebalanceClearingPartitions(existed);
+ metrics.rebalanceClearingPartitions(remaining);
}
}
- // If all partitions are cleared send initial demand message.
- if (existed == 0)
- ctx.kernalContext().closure().runLocalSafe(initDemandRequestTask, true);
+ if (log.isDebugEnabled())
+ log.debug("Remaining clearing partitions [grp=" + grp.cacheOrGroupName()
+ + ", remaining=" + remaining + "]");
+
+ if (remaining == 0)
+ clearAllFuture.onDone();
}
}
+ else {
+ clearAllFuture.onDone();
+ }
});
}
- else
- clearingPartitions.decrementAndGet();
+ else {
+ int remaining = clearingPartitions.decrementAndGet();
+
+ for (GridCacheContext cctx : grp.caches()) {
+ if (cctx.statisticsEnabled()) {
+ final CacheMetricsImpl metrics = cctx.cache().metrics0();
+
+ metrics.rebalanceClearingPartitions(remaining);
+ }
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Remaining clearing partitions [grp=" + grp.cacheOrGroupName()
+ + ", remaining=" + remaining + "]");
+
+ if (remaining == 0)
+ clearAllFuture.onDone();
+ }
}
+
+ return clearAllFuture;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 6d2f526..a3ee305 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -173,7 +173,8 @@ class GridDhtPartitionSupplier {
if (curTop.compareTo(demTop) > 0) {
if (log.isDebugEnabled())
- log.debug("Demand request outdated [currentTopVer=" + curTop
+ log.debug("Demand request outdated [grp=" + grp.cacheOrGroupName()
+ + ", currentTopVer=" + curTop
+ ", demandTopVer=" + demTop
+ ", from=" + nodeId
+ ", topicId=" + topicId + "]");
@@ -189,10 +190,19 @@ class GridDhtPartitionSupplier {
if (sctx != null && sctx.rebalanceId == -d.rebalanceId()) {
clearContext(scMap.remove(contextId), log);
+
+ if (log.isDebugEnabled())
+ log.debug("Supply context cleaned [grp=" + grp.cacheOrGroupName()
+ + ", from=" + nodeId
+ + ", demandMsg=" + d
+ + ", supplyContext=" + sctx + "]");
}
else {
if (log.isDebugEnabled())
- log.debug("Stale context cleanup message " + d + ", supplyContext=" + sctx);
+ log.debug("Stale supply context cleanup message [grp=" + grp.cacheOrGroupName()
+ + ", from=" + nodeId
+ + ", demandMsg=" + d
+ + ", supplyContext=" + sctx + "]");
}
return;
@@ -200,13 +210,16 @@ class GridDhtPartitionSupplier {
}
if (log.isDebugEnabled())
- log.debug("Demand request accepted [current=" + curTop + ", demanded=" + demTop +
- ", from=" + nodeId + ", topicId=" + topicId + "]");
+ log.debug("Demand request accepted [grp=" + grp.cacheOrGroupName()
+ + ", from=" + nodeId
+ + ", currentVer=" + curTop
+ + ", demandedVer=" + demTop
+ + ", topicId=" + topicId + "]");
ClusterNode node = grp.shared().discovery().node(nodeId);
if (node == null)
- return; // Context will be cleaned at topology change.
+ return;
try {
SupplyContext sctx;
@@ -217,13 +230,27 @@ class GridDhtPartitionSupplier {
if (sctx != null && d.rebalanceId() < sctx.rebalanceId) {
// Stale message, return context back and return.
scMap.put(contextId, sctx);
+
+ if (log.isDebugEnabled())
+ log.debug("Stale demand message [grp=" + grp.cacheOrGroupName()
+ + ", actualContext=" + sctx
+ + ", from=" + nodeId
+ + ", demandMsg=" + d + "]");
+
return;
}
}
// Demand request should not contain empty partitions if no supply context is associated with it.
- if (sctx == null && (d.partitions() == null || d.partitions().isEmpty()))
+ if (sctx == null && (d.partitions() == null || d.partitions().isEmpty())) {
+ if (log.isDebugEnabled())
+ log.debug("Empty demand message [grp=" + grp.cacheOrGroupName()
+ + ", from=" + nodeId
+ + ", topicId=" + topicId
+ + ", demandMsg=" + d + "]");
+
return;
+ }
assert !(sctx != null && !d.partitions().isEmpty());
@@ -271,7 +298,8 @@ class GridDhtPartitionSupplier {
GridDhtLocalPartition loc = top.localPartition(part, d.topologyVersion(), false);
- assert loc != null && loc.state() == GridDhtPartitionState.OWNING;
+ assert loc != null && loc.state() == GridDhtPartitionState.OWNING
+ : "Partition should be in OWNING state: " + loc;
s.addEstimatedKeysCount(grp.offheap().totalPartitionEntriesCount(part));
}
@@ -323,7 +351,8 @@ class GridDhtPartitionSupplier {
GridDhtLocalPartition loc = top.localPartition(part, d.topologyVersion(), false);
- assert (loc != null && loc.state() == OWNING && loc.reservations() > 0) || iter.isPartitionMissing(part) : loc;
+ assert (loc != null && loc.state() == OWNING && loc.reservations() > 0) || iter.isPartitionMissing(part)
+ : "Partition should be in OWNING state and has at least 1 reservation " + loc;
if (iter.isPartitionMissing(part) && remainingParts.contains(part)) {
s.missed(part);
@@ -361,9 +390,6 @@ class GridDhtPartitionSupplier {
remainingParts.remove(part);
}
-
- // Need to manually prepare cache message.
- // TODO GG-11141.
}
Iterator<Integer> remainingIter = remainingParts.iterator();
@@ -374,7 +400,8 @@ class GridDhtPartitionSupplier {
if (iter.isPartitionDone(p)) {
GridDhtLocalPartition loc = top.localPartition(p, d.topologyVersion(), false);
- assert loc != null;
+ assert loc != null
+ : "Supply partition is gone: grp=" + grp.cacheOrGroupName() + ", p=" + p;
s.last(p, loc.updateCounter());
@@ -387,7 +414,8 @@ class GridDhtPartitionSupplier {
}
}
- assert remainingParts.isEmpty();
+ assert remainingParts.isEmpty()
+ : "Partitions after rebalance should be either done or missing: " + remainingParts;
if (sctx != null)
clearContext(sctx, log);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index ddcb81e..700f0cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -187,7 +187,7 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter {
AffinityAssignment aff = grp.affinity().cachedAffinity(topVer);
- CachePartitionFullCountersMap cntrMap = top.fullUpdateCounters();
+ CachePartitionFullCountersMap countersMap = grp.topology().fullUpdateCounters();
for (int p = 0; p < partCnt; p++) {
if (ctx.exchange().hasPendingExchange()) {
@@ -251,7 +251,7 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter {
);
}
- msg.partitions().addHistorical(p, cntrMap.initialUpdateCounter(p), cntrMap.updateCounter(p), partCnt);
+ msg.partitions().addHistorical(p, part.initialUpdateCounter(), countersMap.updateCounter(p), partCnt);
}
else {
Collection<ClusterNode> picked = pickOwners(p, topVer);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
index 7066e0d..8515004 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
@@ -90,7 +90,7 @@ public class IgniteDhtPartitionsToReloadMap implements Serializable {
/**
* @return {@code True} if empty.
*/
- public boolean isEmpty() {
+ public synchronized boolean isEmpty() {
return map == null || map.isEmpty();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 68ec83d..5feaa25 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -755,8 +755,8 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
}
/** {@inheritDoc} */
- @Override @Nullable protected IgniteHistoricalIterator historicalIterator(
- CachePartitionPartialCountersMap partCntrs) throws IgniteCheckedException {
+ @Override @Nullable protected WALHistoricalIterator historicalIterator(
+ CachePartitionPartialCountersMap partCntrs, Set<Integer> missing) throws IgniteCheckedException {
if (partCntrs == null || partCntrs.isEmpty())
return null;
@@ -773,13 +773,18 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
if (startPtr == null)
throw new IgniteCheckedException("Could not find start pointer for partition [part=" + p + ", partCntrSince=" + initCntr + "]");
- if (minPtr == null || startPtr.compareTo(minPtr) == -1)
+ if (minPtr == null || startPtr.compareTo(minPtr) < 0)
minPtr = startPtr;
}
WALIterator it = grp.shared().wal().replay(minPtr);
- return new WALIteratorAdapter(grp, partCntrs, it);
+ WALHistoricalIterator iterator = new WALHistoricalIterator(grp, partCntrs, it);
+
+ // Add historical partitions which are unabled to reserve to missing set.
+ missing.addAll(iterator.missingParts);
+
+ return iterator;
}
/**
@@ -807,7 +812,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
/**
*
*/
- private static class WALIteratorAdapter implements IgniteHistoricalIterator {
+ private static class WALHistoricalIterator implements IgniteHistoricalIterator {
/** */
private static final long serialVersionUID = 0L;
@@ -817,6 +822,9 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
/** Partition counters map. */
private final CachePartitionPartialCountersMap partMap;
+ /** Partitions marked as missing (unable to reserve or partition is not in OWNING state). */
+ private final Set<Integer> missingParts = new HashSet<>();
+
/** Partitions marked as done. */
private final Set<Integer> doneParts = new HashSet<>();
@@ -830,19 +838,24 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
private Iterator<DataEntry> entryIt;
/** */
- private CacheDataRow next;
+ private DataEntry next;
+
+ /** Flag indicates that partition belongs to current {@link #next} is finished and no longer needs to rebalance. */
+ private boolean reachedPartitionEnd;
/**
* @param grp Cache context.
* @param walIt WAL iterator.
*/
- private WALIteratorAdapter(CacheGroupContext grp, CachePartitionPartialCountersMap partMap, WALIterator walIt) {
+ private WALHistoricalIterator(CacheGroupContext grp, CachePartitionPartialCountersMap partMap, WALIterator walIt) {
this.grp = grp;
this.partMap = partMap;
this.walIt = walIt;
cacheIds = grp.cacheIds();
+ reservePartitions();
+
advance();
}
@@ -859,6 +872,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
/** {@inheritDoc} */
@Override public void close() throws IgniteCheckedException {
walIt.close();
+ releasePartitions();
}
/** {@inheritDoc} */
@@ -896,7 +910,13 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
if (next == null)
throw new NoSuchElementException();
- CacheDataRow val = next;
+ CacheDataRow val = new DataEntryRow(next);
+
+ if (reachedPartitionEnd) {
+ doneParts.add(next.partitionId());
+
+ reachedPartitionEnd = false;
+ }
advance();
@@ -909,6 +929,46 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
}
/**
+ * Reserve historical partitions.
+ * If partition is unable to reserve, id of that partition is placed to {@link #missingParts} set.
+ */
+ private void reservePartitions() {
+ for (int i = 0; i < partMap.size(); i++) {
+ int p = partMap.partitionAt(i);
+ GridDhtLocalPartition part = grp.topology().localPartition(p);
+
+ if (part == null || !part.reserve()) {
+ missingParts.add(p);
+ continue;
+ }
+
+ if (part.state() != OWNING) {
+ part.release();
+ missingParts.add(p);
+ }
+ }
+ }
+
+ /**
+ * Release historical partitions.
+ */
+ private void releasePartitions() {
+ for (int i = 0; i < partMap.size(); i++) {
+ int p = partMap.partitionAt(i);
+
+ if (missingParts.contains(p))
+ continue;
+
+ GridDhtLocalPartition part = grp.topology().localPartition(p);
+
+ assert part != null && part.state() == OWNING && part.reservations() > 0
+ : "Partition should in OWNING state and has at least 1 reservation";
+
+ part.release();
+ }
+ }
+
+ /**
*
*/
private void advance() {
@@ -922,7 +982,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
if (cacheIds.contains(entry.cacheId())) {
int idx = partMap.partitionIndex(entry.partitionId());
- if (idx < 0)
+ if (idx < 0 || missingParts.contains(idx))
continue;
long from = partMap.initialUpdateCounterAt(idx);
@@ -930,9 +990,9 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
if (entry.partitionCounter() >= from && entry.partitionCounter() <= to) {
if (entry.partitionCounter() == to)
- doneParts.add(entry.partitionId());
+ reachedPartitionEnd = true;
- next = new DataEntryRow(entry);
+ next = entry;
return;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/core/src/test/config/log4j-test.xml
----------------------------------------------------------------------
diff --git a/modules/core/src/test/config/log4j-test.xml b/modules/core/src/test/config/log4j-test.xml
index 9138c02..b0b08e7 100755
--- a/modules/core/src/test/config/log4j-test.xml
+++ b/modules/core/src/test/config/log4j-test.xml
@@ -79,12 +79,6 @@
Uncomment to enable Ignite query execution debugging.
-->
<!--
- <category name="org.apache.ignite.cache.query">
- <level value="DEBUG"/>
- </category>
- -->
-
- <!--
<category name="org.apache.ignite.internal.processors.query">
<level value="DEBUG"/>
</category>
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java
new file mode 100644
index 0000000..a090381
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.IgniteSystemProperties;
+
+/**
+ *
+ */
+public class IgnitePdsAtomicCacheHistoricalRebalancingTest extends IgnitePdsAtomicCacheRebalancingTest {
+ /** {@inheritDoc */
+ @Override protected void beforeTest() throws Exception {
+ // Use rebalance from WAL if possible.
+ System.setProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc */
+ @Override protected void afterTest() throws Exception {
+ System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
+
+ super.afterTest();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
index 10f9b03..0dd9c78 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
@@ -78,12 +78,15 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
cfg.setConsistentId(gridName);
+ cfg.setRebalanceThreadPoolSize(2);
+
CacheConfiguration ccfg1 = cacheConfiguration(cacheName)
.setPartitionLossPolicy(PartitionLossPolicy.READ_WRITE_SAFE)
- .setBackups(2)
+ .setBackups(1)
.setRebalanceMode(CacheRebalanceMode.ASYNC)
.setIndexedTypes(Integer.class, Integer.class)
.setAffinity(new RendezvousAffinityFunction(false, 32))
+ .setRebalanceBatchesPrefetchCount(2)
.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
CacheConfiguration ccfg2 = cacheConfiguration("indexed");
@@ -172,8 +175,6 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
stopAllGrids();
cleanPersistenceDir();
-
- System.clearProperty(IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE);
}
/**
@@ -184,7 +185,7 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
public void testRebalancingOnRestart() throws Exception {
Ignite ignite0 = startGrid(0);
- ignite0.active(true);
+ ignite0.cluster().active(true);
startGrid(1);
@@ -233,8 +234,6 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
* @throws Exception If fails.
*/
public void testRebalancingOnRestartAfterCheckpoint() throws Exception {
- fail("IGNITE-5302");
-
IgniteEx ignite0 = startGrid(0);
IgniteEx ignite1 = startGrid(1);
@@ -242,7 +241,7 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
IgniteEx ignite2 = startGrid(2);
IgniteEx ignite3 = startGrid(3);
- ignite0.active(true);
+ ignite0.cluster().active(true);
ignite0.cache(cacheName).rebalance().get();
ignite1.cache(cacheName).rebalance().get();
@@ -264,6 +263,8 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
ignite2.close();
ignite3.close();
+ resetBaselineTopology();
+
ignite0.resetLostPartitions(Collections.singletonList(cache1.getName()));
assert cache1.lostPartitions().isEmpty();
@@ -306,7 +307,7 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
IgniteEx ignite3 = (IgniteEx)G.start(getConfiguration("test3"));
IgniteEx ignite4 = (IgniteEx)G.start(getConfiguration("test4"));
- ignite1.active(true);
+ ignite1.cluster().active(true);
awaitPartitionMapExchange();
@@ -325,7 +326,7 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
ignite3 = (IgniteEx)G.start(getConfiguration("test3"));
ignite4 = (IgniteEx)G.start(getConfiguration("test4"));
- ignite1.active(true);
+ ignite1.cluster().active(true);
awaitPartitionMapExchange();
@@ -348,13 +349,13 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
* @throws Exception If fails.
*/
public void testPartitionLossAndRecover() throws Exception {
- fail("IGNITE-5302");
-
Ignite ignite1 = startGrid(0);
Ignite ignite2 = startGrid(1);
Ignite ignite3 = startGrid(2);
Ignite ignite4 = startGrid(3);
+ ignite1.cluster().active(true);
+
awaitPartitionMapExchange();
IgniteCache<String, String> cache1 = ignite1.cache(cacheName);
@@ -537,7 +538,7 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
final Ignite ig = grid(1);
- ig.active(true);
+ ig.cluster().active(true);
awaitPartitionMapExchange();
@@ -574,8 +575,6 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
* @throws Exception If failed
*/
public void testPartitionCounterConsistencyOnUnstableTopology() throws Exception {
- System.setProperty(IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, "1");
-
final Ignite ig = startGrids(4);
ig.cluster().active(true);
@@ -595,9 +594,8 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
try {
stopGrid(3);
- // Clear checkpoint history to avoid rebalance from WAL.
- forceCheckpoint();
forceCheckpoint();
+
U.sleep(500); // Wait for data load.
IgniteEx ig0 = startGrid(3);
@@ -609,8 +607,6 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
awaitPartitionMapExchange();
- // Clear checkpoint history to avoid rebalance from WAL.
- forceCheckpoint();
forceCheckpoint();
startGrid(2);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheHistoricalRebalancingTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheHistoricalRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheHistoricalRebalancingTest.java
new file mode 100644
index 0000000..179c8e0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheHistoricalRebalancingTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.IgniteSystemProperties;
+
+/**
+ *
+ */
+public class IgnitePdsTxCacheHistoricalRebalancingTest extends IgnitePdsTxCacheRebalancingTest {
+ /** {@inheritDoc */
+ @Override protected void beforeTest() throws Exception {
+ // Use rebalance from WAL if possible.
+ System.setProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc */
+ @Override protected void afterTest() throws Exception {
+ System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
+
+ super.afterTest();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
new file mode 100644
index 0000000..6387dac
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
+
+/**
+ * Historic WAL rebalance base test.
+ */
+public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
+ /** Cache name. */
+ private static final String CACHE_NAME = "cache";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ System.setProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0"); //to make all rebalance wal-based
+
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(CACHE_NAME);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+ ccfg.setRebalanceMode(CacheRebalanceMode.ASYNC);
+
+ ccfg.setCacheMode(CacheMode.REPLICATED);
+
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ cfg.setCacheConfiguration(ccfg);
+
+ DataStorageConfiguration dbCfg = new DataStorageConfiguration()
+ .setWalHistorySize(Integer.MAX_VALUE)
+ .setWalMode(WALMode.LOG_ONLY)
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true));
+
+ cfg.setDataStorageConfiguration(dbCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ System.clearProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void test() throws Exception {
+ IgniteEx ig0 = startGrid(0);
+ IgniteEx ig1 = startGrid(1);
+ final int entryCnt = 10_000;
+
+ ig0.cluster().active(true);
+
+ IgniteCache<Object, Object> cache = ig0.cache(CACHE_NAME);
+
+ for (int k = 0; k < entryCnt; k++)
+ cache.put(k, new IndexedObject(k));
+
+ forceCheckpoint();
+
+ stopGrid(1, false);
+
+ for (int k = 0; k < entryCnt; k++)
+ cache.put(k, new IndexedObject(k + 1));
+
+ forceCheckpoint();
+
+ ig1 = startGrid(1);
+
+ IgniteCache<Object, Object> cache1 = ig1.cache(CACHE_NAME);
+
+ cache1.rebalance().get(2, TimeUnit.MINUTES);
+
+ for (int k = 0; k < entryCnt; k++)
+ assertEquals(new IndexedObject(k + 1), cache.get(k));
+ }
+
+ /**
+ *
+ */
+ private static class IndexedObject {
+ /** */
+ @QuerySqlField(index = true)
+ private int iVal;
+
+ /** */
+ private byte[] payload = new byte[1024];
+
+ /**
+ * @param iVal Integer value.
+ */
+ private IndexedObject(int iVal) {
+ this.iVal = iVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof IndexedObject))
+ return false;
+
+ IndexedObject that = (IndexedObject)o;
+
+ return iVal == that.iVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return iVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IndexedObject.class, this);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
index de7ee5f..996d6a7 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
@@ -209,7 +209,6 @@ public class IgniteCacheClientQueryReplicatedNodeRestartSelfTest extends GridCom
* @throws Exception If failed.
*/
public void testRestarts() throws Exception {
- fail("https://issues.apache.org/jira/browse/IGNITE-7946");
int duration = 90 * 1000;
int qryThreadNum = 5;
int restartThreadsNum = 2; // 2 of 4 data nodes
http://git-wip-us.apache.org/repos/asf/ignite/blob/5c718543/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index 447b622..943d43f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -17,10 +17,12 @@
package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsAtomicCacheHistoricalRebalancingTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsAtomicCacheRebalancingTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsBinaryMetadataOnClusterRestartTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsBinarySortObjectFieldsTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsMarshallerMappingRestoreOnNodeStartTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheHistoricalRebalancingTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheRebalancingTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreCacheGroupsTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMultiNodePutGetRestartTest;
@@ -30,6 +32,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePds
import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsDiskErrorsRecoveringTest;
import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsNoActualWalHistoryTest;
import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsThreadInterruptionTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRebalanceTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryPPCTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryWithCompactionTest;
@@ -60,6 +63,10 @@ public class IgnitePdsWithIndexingCoreTestSuite extends TestSuite {
suite.addTestSuite(IgnitePdsAtomicCacheRebalancingTest.class);
suite.addTestSuite(IgnitePdsTxCacheRebalancingTest.class);
+ suite.addTestSuite(IgnitePdsAtomicCacheHistoricalRebalancingTest.class);
+ suite.addTestSuite(IgnitePdsTxCacheHistoricalRebalancingTest.class);
+ suite.addTestSuite(IgniteWalRebalanceTest.class);
+
suite.addTestSuite(IgniteWalRecoveryPPCTest.class);
suite.addTestSuite(IgnitePdsDiskErrorsRecoveringTest.class);
suite.addTestSuite(IgnitePdsCacheDestroyDuringCheckpointTest.class);
[34/50] [abbrv] ignite git commit: IGNITE-8082 Added
ZookeeperDiscoverySpi MBean - Fixes #3862.
Posted by ag...@apache.org.
IGNITE-8082 Added ZookeeperDiscoverySpi MBean - Fixes #3862.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/09879b82
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/09879b82
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/09879b82
Branch: refs/heads/ignite-6083
Commit: 09879b828b5606d1881b9ec596b6cb9e52282527
Parents: 8fc1824
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Thu Apr 19 15:44:26 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Apr 19 15:44:26 2018 +0300
----------------------------------------------------------------------
.../ignite/spi/discovery/DiscoverySpiMBean.java | 84 ++++++++++++++++++
.../spi/discovery/tcp/TcpDiscoverySpi.java | 10 +++
.../spi/discovery/tcp/TcpDiscoverySpiMBean.java | 13 +--
.../spi/discovery/zk/ZookeeperDiscoverySpi.java | 93 +++++++++++++++++---
.../zk/ZookeeperDiscoverySpiMBean.java | 67 ++++++++++++++
.../discovery/zk/internal/ZookeeperClient.java | 7 ++
.../zk/internal/ZookeeperDiscoveryImpl.java | 36 +++++++-
.../internal/ZookeeperDiscoveryStatistics.java | 48 ++++++++++
.../zk/internal/ZookeeperDiscoverySpiTest.java | 40 ++++++++-
9 files changed, 376 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/09879b82/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMBean.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMBean.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMBean.java
new file mode 100644
index 0000000..d86a497
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMBean.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery;
+
+import java.util.UUID;
+import org.apache.ignite.mxbean.MXBeanDescription;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Generic MBean interface to monitor DiscoverySpi subsystem.
+ */
+public interface DiscoverySpiMBean {
+ /**
+ * Gets current SPI state.
+ *
+ * @return Current SPI state.
+ */
+ @MXBeanDescription("SPI state.")
+ public String getSpiState();
+
+ /**
+ * Gets joined nodes count.
+ *
+ * @return Nodes joined count.
+ */
+ @MXBeanDescription("Nodes joined count.")
+ public long getNodesJoined();
+
+ /**
+ * Gets failed nodes count.
+ *
+ * @return Failed nodes count.
+ */
+ @MXBeanDescription("Nodes failed count.")
+ public long getNodesFailed();
+
+ /**
+ * Gets left nodes count.
+ *
+ * @return Left nodes count.
+ */
+ @MXBeanDescription("Nodes left count.")
+ public long getNodesLeft();
+
+ /**
+ * Gets current coordinator.
+ *
+ * @return Gets current coordinator.
+ * @deprecated Use {@link #getCoordinatorNodeFormatted()} instead.
+ */
+ @Deprecated
+ @MXBeanDescription("Coordinator node ID.")
+ @Nullable public UUID getCoordinator();
+
+ /**
+ * Gets current coordinator node formatted as a string.
+ *
+ * @return Current coordinator string representation.
+ */
+ @MXBeanDescription("Coordinator node formatted as a string.")
+ @Nullable public String getCoordinatorNodeFormatted();
+
+ /**
+ * Gets local node formatted as a string.
+ *
+ * @return Local node string representation.
+ */
+ @MXBeanDescription("Local node formatted as a string.")
+ public String getLocalNodeFormatted();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/09879b82/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index f0a5186..7c73b28 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -2350,6 +2350,16 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements IgniteDiscovery
}
/** {@inheritDoc} */
+ @Nullable @Override public String getCoordinatorNodeFormatted() {
+ return String.valueOf(impl.getNode(impl.getCoordinator()));
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getLocalNodeFormatted() {
+ return String.valueOf(getLocalNode());
+ }
+
+ /** {@inheritDoc} */
@Override public void dumpDebugInfo() {
impl.dumpDebugInfo(log);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/09879b82/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
index 8d51d77..cb0fd36 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
@@ -23,20 +23,21 @@ import org.apache.ignite.mxbean.MXBeanDescription;
import org.apache.ignite.mxbean.MXBeanParametersDescriptions;
import org.apache.ignite.mxbean.MXBeanParametersNames;
import org.apache.ignite.spi.IgniteSpiManagementMBean;
+import org.apache.ignite.spi.discovery.DiscoverySpiMBean;
import org.jetbrains.annotations.Nullable;
/**
* Management bean for {@link TcpDiscoverySpi}.
*/
@MXBeanDescription("MBean provide access to TCP-based discovery SPI.")
-public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean {
+public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean, DiscoverySpiMBean {
/**
* Gets current SPI state.
*
* @return Current SPI state.
*/
@MXBeanDescription("SPI state.")
- public String getSpiState();
+ @Override public String getSpiState();
/**
* Gets {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} (string representation).
@@ -116,7 +117,7 @@ public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean {
* @return Nodes joined count.
*/
@MXBeanDescription("Nodes joined count.")
- public long getNodesJoined();
+ @Override public long getNodesJoined();
/**
* Gets left nodes count.
@@ -124,7 +125,7 @@ public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean {
* @return Left nodes count.
*/
@MXBeanDescription("Nodes left count.")
- public long getNodesLeft();
+ @Override public long getNodesLeft();
/**
* Gets failed nodes count.
@@ -132,7 +133,7 @@ public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean {
* @return Failed nodes count.
*/
@MXBeanDescription("Nodes failed count.")
- public long getNodesFailed();
+ @Override public long getNodesFailed();
/**
* Gets pending messages registered count.
@@ -212,7 +213,7 @@ public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean {
* @return Gets current coordinator.
*/
@MXBeanDescription("Coordinator node ID.")
- @Nullable public UUID getCoordinator();
+ @Override @Nullable public UUID getCoordinator();
/**
* Gets message acknowledgement timeout.
http://git-wip-us.apache.org/repos/asf/ignite/blob/09879b82/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
index 860c71c..52f77fc 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
@@ -41,13 +41,12 @@ import org.apache.ignite.lang.IgniteProductVersion;
import org.apache.ignite.resources.LoggerResource;
import org.apache.ignite.spi.IgniteSpiAdapter;
import org.apache.ignite.spi.IgniteSpiConfiguration;
-import org.apache.ignite.spi.IgniteSpiContext;
import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiMBeanAdapter;
import org.apache.ignite.spi.IgniteSpiMultipleInstancesSupport;
import org.apache.ignite.spi.communication.CommunicationSpi;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
import org.apache.ignite.spi.discovery.DiscoveryMetricsProvider;
-import org.apache.ignite.spi.discovery.DiscoverySpi;
import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange;
import org.apache.ignite.spi.discovery.DiscoverySpiHistorySupport;
@@ -57,6 +56,7 @@ import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
import org.apache.ignite.spi.discovery.zk.internal.ZookeeperClusterNode;
import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryStatistics;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_CONSISTENT_ID_BY_HOST_WITHOUT_PORT;
@@ -69,7 +69,7 @@ import static org.apache.ignite.IgniteSystemProperties.getBoolean;
@DiscoverySpiOrderSupport(true)
@DiscoverySpiHistorySupport(true)
@DiscoverySpiMutableCustomMessageSupport(false)
-public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi, IgniteDiscoverySpi {
+public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements IgniteDiscoverySpi {
/** */
public static final String DFLT_ROOT_PATH = "/apacheIgnite";
@@ -137,6 +137,9 @@ public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements Discovery
/** */
private IgniteDiscoverySpiInternalListener internalLsnr;
+ /** */
+ private final ZookeeperDiscoveryStatistics stats = new ZookeeperDiscoveryStatistics();
+
/**
* @return Base path in ZK for znodes created by SPI.
*/
@@ -384,7 +387,7 @@ public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements Discovery
/** {@inheritDoc} */
@Override public void setAuthenticator(DiscoverySpiNodeAuthenticator auth) {
- this.nodeAuth = auth;
+ nodeAuth = auth;
}
/**
@@ -422,12 +425,9 @@ public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements Discovery
}
/** {@inheritDoc} */
- @Override protected void onContextInitialized0(IgniteSpiContext spiCtx) throws IgniteSpiException {
- super.onContextInitialized0(spiCtx);
- }
-
- /** {@inheritDoc} */
@Override public void spiStart(@Nullable String igniteInstanceName) throws IgniteSpiException {
+ startStopwatch();
+
if (sesTimeout == 0)
sesTimeout = ignite.configuration().getFailureDetectionTimeout().intValue();
@@ -465,7 +465,10 @@ public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements Discovery
locNode,
lsnr,
exchange,
- internalLsnr);
+ internalLsnr,
+ stats);
+
+ registerMBean(igniteInstanceName, new ZookeeperDiscoverySpiMBeanImpl(this), ZookeeperDiscoverySpiMBean.class);
try {
impl.startJoinAndWait();
@@ -492,6 +495,8 @@ public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements Discovery
/** {@inheritDoc} */
@Override public void spiStop() throws IgniteSpiException {
+ unregisterMBean();
+
if (impl != null)
impl.stop();
}
@@ -554,4 +559,72 @@ public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements Discovery
@Override public String toString() {
return S.toString(ZookeeperDiscoverySpi.class, this);
}
+
+ /** */
+ private class ZookeeperDiscoverySpiMBeanImpl extends IgniteSpiMBeanAdapter implements ZookeeperDiscoverySpiMBean {
+ /** {@inheritDoc} */
+ public ZookeeperDiscoverySpiMBeanImpl(IgniteSpiAdapter spiAdapter) {
+ super(spiAdapter);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getSpiState() {
+ return impl.getSpiState();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getNodesJoined() {
+ return stats.joinedNodesCnt();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getNodesLeft() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getNodesFailed() {
+ return stats.failedNodesCnt();
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public UUID getCoordinator() {
+ return impl.getCoordinator();
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public String getCoordinatorNodeFormatted() {
+ return String.valueOf(impl.node(impl.getCoordinator()));
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getLocalNodeFormatted() {
+ return String.valueOf(getLocalNode());
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getZkConnectionString() {
+ return zkConnectionString;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getZkSessionTimeout() {
+ return sesTimeout;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getZkSessionId() {
+ return impl.getZkSessionId();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getZkRootPath() {
+ return zkRootPath;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getNodeOrder() {
+ return getLocalNode().order();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/09879b82/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiMBean.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiMBean.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiMBean.java
new file mode 100644
index 0000000..1eed0b4
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiMBean.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery.zk;
+
+import org.apache.ignite.mxbean.MXBeanDescription;
+import org.apache.ignite.spi.IgniteSpiManagementMBean;
+import org.apache.ignite.spi.discovery.DiscoverySpiMBean;
+
+/**
+ * Management bean for {@link ZookeeperDiscoverySpi}.
+ */
+@MXBeanDescription("MBean provide access to Zookeeper-based discovery SPI.")
+public interface ZookeeperDiscoverySpiMBean extends IgniteSpiManagementMBean, DiscoverySpiMBean {
+ /**
+ * Gets node join order.
+ *
+ * @return Node Join Order.
+ */
+ @MXBeanDescription("Node Join Order.")
+ public long getNodeOrder();
+
+ /**
+ * Gets connection string used to connect to ZooKeeper cluster.
+ *
+ * @return Zk Connection String.
+ */
+ @MXBeanDescription("Zk Connection String.")
+ public String getZkConnectionString();
+
+ /**
+ * Gets session timeout used by Zk client of local Ignite node.
+ *
+ * @return Zk Session Timeout.
+ */
+ @MXBeanDescription("Zk Session Timeout (milliseconds).")
+ public long getZkSessionTimeout();
+
+ /**
+ * Gets session id of Zk client established with ZooKeeper cluster.
+ *
+ * @return Zk Session Id.
+ */
+ @MXBeanDescription("Zk Session Id.")
+ public String getZkSessionId();
+
+ /**
+ * Gets root path in ZooKeeper cluster Zk client uses to put data to.
+ *
+ * @return Zk Root Path.
+ */
+ @MXBeanDescription("Zk Root Path.")
+ public String getZkRootPath();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/09879b82/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
index 21703c6..cc525d3 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
@@ -162,6 +162,13 @@ public class ZookeeperClient implements Watcher {
}
}
+ /** */
+ String state() {
+ synchronized (stateMux) {
+ return state.toString();
+ }
+ }
+
/** {@inheritDoc} */
@Override public void process(WatchedEvent evt) {
if (closing)
http://git-wip-us.apache.org/repos/asf/ignite/blob/09879b82/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 7708358..3010d16 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -59,8 +59,6 @@ import org.apache.ignite.internal.IgnitionEx;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.events.DiscoveryCustomEvent;
import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.security.SecurityContext;
import org.apache.ignite.internal.util.GridLongList;
import org.apache.ignite.internal.util.GridSpinBusyLock;
@@ -71,7 +69,6 @@ import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.LT;
import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgniteRunnable;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.marshaller.MarshallerUtils;
@@ -191,6 +188,9 @@ public class ZookeeperDiscoveryImpl {
/** */
private long prevSavedEvtsTopVer;
+ /** */
+ private final ZookeeperDiscoveryStatistics stats;
+
/**
* @param spi Discovery SPI.
* @param igniteInstanceName Instance name.
@@ -200,6 +200,7 @@ public class ZookeeperDiscoveryImpl {
* @param lsnr Discovery events listener.
* @param exchange Discovery data exchange.
* @param internalLsnr Internal listener (used for testing only).
+ * @param stats Zookeeper DiscoverySpi statistics collector.
*/
public ZookeeperDiscoveryImpl(
ZookeeperDiscoverySpi spi,
@@ -209,7 +210,8 @@ public class ZookeeperDiscoveryImpl {
ZookeeperClusterNode locNode,
DiscoverySpiListener lsnr,
DiscoverySpiDataExchange exchange,
- IgniteDiscoverySpiInternalListener internalLsnr) {
+ IgniteDiscoverySpiInternalListener internalLsnr,
+ ZookeeperDiscoveryStatistics stats) {
assert locNode.id() != null && locNode.isLocal() : locNode;
MarshallerUtils.setNodeName(marsh, igniteInstanceName);
@@ -235,6 +237,8 @@ public class ZookeeperDiscoveryImpl {
if (internalLsnr != null)
this.internalLsnr = internalLsnr;
+
+ this.stats = stats;
}
/**
@@ -2184,6 +2188,8 @@ public class ZookeeperDiscoveryImpl {
joinCtx.addJoinedNode(nodeEvtData, commonData);
rtState.evtsData.onNodeJoin(joinedNode);
+
+ stats.onNodeJoined();
}
/**
@@ -3485,6 +3491,8 @@ public class ZookeeperDiscoveryImpl {
topSnapshot,
Collections.<Long, Collection<ClusterNode>>emptyMap(),
null);
+
+ stats.onNodeFailed();
}
/**
@@ -4461,4 +4469,24 @@ public class ZookeeperDiscoveryImpl {
/** */
STOPPED
}
+
+ /** */
+ public UUID getCoordinator() {
+ Map.Entry<Long, ZookeeperClusterNode> e = rtState.top.nodesByOrder.firstEntry();
+
+ return e != null ? e.getValue().id() : null;
+ }
+
+ /** */
+ public String getSpiState() {
+ return rtState.zkClient.state();
+ }
+
+ /** */
+ public String getZkSessionId() {
+ if (rtState.zkClient != null && rtState.zkClient.zk() != null)
+ return Long.toHexString(rtState.zkClient.zk().getSessionId());
+ else
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/09879b82/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java
new file mode 100644
index 0000000..ef4c37e
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ *
+ */
+public class ZookeeperDiscoveryStatistics {
+ /** */
+ private int joinedNodesCnt;
+
+ /** */
+ private int failedNodesCnt;
+
+ /** */
+ public int joinedNodesCnt() {
+ return joinedNodesCnt;
+ }
+
+ /** */
+ public int failedNodesCnt() {
+ return failedNodesCnt;
+ }
+
+ /** */
+ public void onNodeJoined() {
+ joinedNodesCnt++;
+ }
+
+ /** */
+ public void onNodeFailed() {
+ failedNodesCnt++;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/09879b82/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index fb12c3a..408af30 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.spi.discovery.zk.internal;
import java.io.Serializable;
+import java.lang.management.ManagementFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
@@ -40,6 +41,9 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import javax.management.JMX;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
import org.apache.curator.test.TestingCluster;
import org.apache.curator.test.TestingZooKeeperServer;
import org.apache.ignite.Ignite;
@@ -60,7 +64,7 @@ import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.DiscoverySpiTestListener;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.IgnitionEx;
@@ -71,7 +75,6 @@ import org.apache.ignite.internal.managers.discovery.DiscoCache;
import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
import org.apache.ignite.internal.managers.discovery.DiscoveryLocalJoinData;
import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
-import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
@@ -107,6 +110,7 @@ import org.apache.ignite.spi.discovery.DiscoverySpi;
import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiMBean;
import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiTestSuite2;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -560,6 +564,38 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
}
/**
+ * @throws Exception if failed.
+ */
+ public void testMbean() throws Exception {
+ startGrids(3);
+
+ MBeanServer srv = ManagementFactory.getPlatformMBeanServer();
+
+ UUID crdNodeId = grid(0).localNode().id();
+
+ try {
+ for (int i = 0; i < 3; i++) {
+ IgniteEx grid = grid(i);
+
+ ObjectName spiName = U.makeMBeanName(grid.context().igniteInstanceName(), "SPIs",
+ ZookeeperDiscoverySpi.class.getSimpleName());
+
+ ZookeeperDiscoverySpiMBean bean = JMX.newMBeanProxy(srv, spiName, ZookeeperDiscoverySpiMBean.class);
+
+ assertNotNull(bean);
+
+ assertEquals(String.valueOf(grid.cluster().node(crdNodeId)), bean.getCoordinatorNodeFormatted());
+ assertEquals(String.valueOf(grid.cluster().localNode()), bean.getLocalNodeFormatted());
+ assertEquals(zkCluster.getConnectString(), bean.getZkConnectionString());
+ assertEquals((long)grid.configuration().getFailureDetectionTimeout(), bean.getZkSessionTimeout());
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
* @throws Exception If failed.
*/
public void testClientNodesStatus() throws Exception {
[33/50] [abbrv] ignite git commit: IGNITE-8021 Fixed tests - Fixes
#3864.
Posted by ag...@apache.org.
IGNITE-8021 Fixed tests - Fixes #3864.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8fc18248
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8fc18248
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8fc18248
Branch: refs/heads/ignite-6083
Commit: 8fc182486902600fe28b5b90c80ccc09c3e18c91
Parents: 02a0865
Author: Ivan Daschinskiy <iv...@gmail.com>
Authored: Thu Apr 19 15:25:23 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Apr 19 15:25:23 2018 +0300
----------------------------------------------------------------------
...eConfigurationDataAfterDestroyCacheTest.java | 326 -------------------
.../IgnitePdsDestroyCacheAbstractTest.java | 207 ++++++++++++
.../persistence/IgnitePdsDestroyCacheTest.java | 85 +++++
...tePdsDestroyCacheWithoutCheckpointsTest.java | 86 +++++
.../ignite/testsuites/IgnitePdsTestSuite.java | 7 +-
5 files changed, 383 insertions(+), 328 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8fc18248/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java
deleted file mode 100644
index d2767d4..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy;
-import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Test correct clean up cache configuration data after destroying cache.
- */
-public class IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** */
- private static final int CACHES = 3;
-
- /** */
- private static final int NODES = 3;
-
- /** */
- private static final int NUM_OF_KEYS = 100;
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
-
- return cfg.setDiscoverySpi(new TcpDiscoverySpi()
- .setIpFinder(IP_FINDER))
- .setDataStorageConfiguration(new DataStorageConfiguration()
- .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
- .setMaxSize(200 * 1024 * 1024)
- .setPersistenceEnabled(true)));
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- super.beforeTest();
-
- cleanPersistenceDir();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- cleanPersistenceDir();
-
- super.afterTest();
- }
-
- /**
- * {@inheritDoc}
- * @returns always {@code true} in order to be able to kill nodes when checkpointer thread hangs.
- */
- @Override protected boolean isMultiJvm() {
- return true;
- }
-
- /**
- * Test destroy non grouped caches.
- *
- * @throws Exception If failed.
- */
- public void testDestroyCaches() throws Exception {
- Ignite ignite = startGrids(NODES);
-
- ignite.cluster().active(true);
-
- startCachesDynamically(ignite);
-
- checkDestroyCaches(ignite);
- }
-
- /**
- * Test destroy grouped caches.
- *
- * @throws Exception If failed.
- */
- public void testDestroyGroupCaches() throws Exception {
- Ignite ignite = startGrids(NODES);
-
- ignite.cluster().active(true);
-
- startGroupCachesDynamically(ignite);
-
- checkDestroyCaches(ignite);
- }
-
- /**
- * Test destroy caches with disabled checkpoints.
- *
- * @throws Exception If failed.
- */
- public void testDestroyCachesAbruptlyWithoutCheckpoints() throws Exception {
- Ignite ignite = startGrids(NODES);
-
- ignite.cluster().active(true);
-
- startCachesDynamically(ignite);
-
- enableCheckpoints(false);
-
- checkDestroyCachesAbruptly(ignite);
- }
-
- /**
- * Test destroy group caches with disabled checkpoints.
- *
- * @throws Exception If failed.
- */
- public void testDestroyGroupCachesAbruptlyWithoutCheckpoints() throws Exception {
- Ignite ignite = startGrids(NODES);
-
- ignite.cluster().active(true);
-
- startGroupCachesDynamically(ignite);
-
- enableCheckpoints(false);
-
- checkDestroyCachesAbruptly(ignite);
- }
-
- /**
- * Test destroy caches abruptly with checkpoints.
- *
- * @throws Exception If failed.
- */
- public void testDestroyCachesAbruptly() throws Exception {
- Ignite ignite = startGrids(NODES);
-
- ignite.cluster().active(true);
-
- startCachesDynamically(ignite);
-
- checkDestroyCachesAbruptly(ignite);
- }
-
-
- /**
- * Test destroy group caches abruptly with checkpoints.
- *
- * @throws Exception If failed.
- */
- public void testDestroyGroupCachesAbruptly() throws Exception {
- Ignite ignite = startGrids(NODES);
-
- ignite.cluster().active(true);
-
- startGroupCachesDynamically(ignite);
-
- checkDestroyCachesAbruptly(ignite);
- }
-
- /**
- * @param ignite Ignite.
- */
- private void loadCaches(Ignite ignite) {
- for (int i = 0; i < CACHES; i++) {
- try (IgniteDataStreamer<Object, Object> s = ignite.dataStreamer(cacheName(i))) {
- s.allowOverwrite(true);
-
- for (int j = 0; j < NUM_OF_KEYS; j++)
- s.addData(j, "cache: " + i + " data: " + j);
-
- s.flush();
- }
- }
- }
-
- /**
- * @param ignite Ignite.
- */
- private void checkDestroyCaches(Ignite ignite) throws Exception {
- loadCaches(ignite);
-
- log.warning("destroying caches....");
-
- ignite.cache(cacheName(0)).destroy();
- ignite.cache(cacheName(1)).destroy();
-
- assertEquals(CACHES - 2, ignite.cacheNames().size());
-
- log.warning("Stopping grid");
-
- stopAllGrids();
-
- log.warning("Grid stopped");
-
- log.warning("Starting grid");
-
- ignite = startGrids(NODES);
-
- log.warning("Grid started");
-
- assertEquals("Check that caches don't survived", CACHES - 2, ignite.cacheNames().size());
-
- for(Ignite ig: G.allGrids()) {
- IgniteCache cache = ig.cache(cacheName(2));
-
- for (int j = 0; j < NUM_OF_KEYS; j++)
- assertNotNull("Check that cache2 contains key: " + j + " node: " + ignite.name(), cache.get(j));
- }
- }
-
-
- /**
- * @param ignite Ignite instance.
- */
- private void checkDestroyCachesAbruptly(Ignite ignite) throws Exception {
- loadCaches(ignite);
-
- log.warning("Destroying caches");
-
- ((GatewayProtectedCacheProxy)ignite.cache(cacheName(0))).destroyAsync();
- ((GatewayProtectedCacheProxy)ignite.cache(cacheName(1))).destroyAsync();
-
- log.warning("Stopping grid");
-
- stopAllGrids();
-
- log.warning("Grid stopped");
-
- log.warning("Starting grid");
-
- startGrids(NODES);
-
- log.warning("Grid started");
-
- for(Ignite ig: G.allGrids()) {
- assertTrue(ig.cacheNames().contains(cacheName(2)));
-
- IgniteCache cache = ig.cache(cacheName(2));
-
- for (int j = 0; j < NUM_OF_KEYS; j++)
- assertNotNull("Check that survived cache cache2 contains key: " + j + " node: " + ig.name(), cache.get(j));
- }
- }
-
- /**
- * @param ignite Ignite.
- */
- private void startCachesDynamically(Ignite ignite) {
- List<CacheConfiguration> ccfg = new ArrayList<>(CACHES);
-
- for (int i = 0; i < CACHES; i++)
- ccfg.add(new CacheConfiguration<>(cacheName(i))
- .setBackups(1)
- .setAffinity(new RendezvousAffinityFunction(false, 32)));
-
- ignite.createCaches(ccfg);
- }
-
- /**
- * @param ignite Ignite instance.
- */
- private void startGroupCachesDynamically(Ignite ignite) {
- List<CacheConfiguration> ccfg = new ArrayList<>(CACHES);
-
- for (int i = 0; i < CACHES; i++)
- ccfg.add(new CacheConfiguration<>(cacheName(i))
- .setGroupName(i % 2 == 0 ? "grp-even" : "grp-odd")
- .setBackups(1)
- .setAffinity(new RendezvousAffinityFunction(false, 32)));
-
- ignite.createCaches(ccfg);
- }
-
-
- /**
- * Generate cache name from idx.
- *
- * @param idx Index.
- */
- private String cacheName(int idx) {
- return "cache" + idx;
- }
-
- /**
- * Enable/disable checkpoints on multi JVM nodes only.
- *
- * @param enabled Enabled flag.
- * @throws IgniteCheckedException If failed.
- */
- private void enableCheckpoints(boolean enabled) throws IgniteCheckedException {
- for (Ignite ignite : G.allGrids()) {
- assert !ignite.cluster().localNode().isClient();
-
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- dbMgr.enableCheckpoints(enabled).get();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8fc18248/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheAbstractTest.java
new file mode 100644
index 0000000..434007e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheAbstractTest.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Base class for {@link IgnitePdsDestroyCacheTest} and {@link IgnitePdsDestroyCacheWithoutCheckpointsTest}
+ */
+public abstract class IgnitePdsDestroyCacheAbstractTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ protected static final int CACHES = 3;
+
+ /** */
+ protected static final int NODES = 3;
+
+ /** */
+ private static final int NUM_OF_KEYS = 100;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ return cfg.setDiscoverySpi(new TcpDiscoverySpi()
+ .setIpFinder(IP_FINDER))
+ .setDataStorageConfiguration(new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setMaxSize(200 * 1024 * 1024)
+ .setPersistenceEnabled(true)));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+
+ super.afterTest();
+ }
+
+ /**
+ * @param ignite Ignite.
+ */
+ private void loadCaches(Ignite ignite) {
+ for (int i = 0; i < CACHES; i++) {
+ try (IgniteDataStreamer<Object, Object> s = ignite.dataStreamer(cacheName(i))) {
+ s.allowOverwrite(true);
+
+ for (int j = 0; j < NUM_OF_KEYS; j++)
+ s.addData(j, "cache: " + i + " data: " + j);
+
+ s.flush();
+ }
+ }
+ }
+
+ /**
+ * @param ignite Ignite.
+ */
+ protected void checkDestroyCaches(Ignite ignite) throws Exception {
+ loadCaches(ignite);
+
+ log.warning("destroying caches....");
+
+ ignite.cache(cacheName(0)).destroy();
+ ignite.cache(cacheName(1)).destroy();
+
+ assertEquals(CACHES - 2, ignite.cacheNames().size());
+
+ log.warning("Stopping grid");
+
+ stopAllGrids(false);
+
+ log.warning("Grid stopped");
+
+ log.warning("Starting grid");
+
+ ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ log.warning("Grid started");
+
+ assertEquals("Check that caches don't survived", CACHES - 2, ignite.cacheNames().size());
+
+ for(Ignite ig: G.allGrids()) {
+ IgniteCache cache = ig.cache(cacheName(2));
+
+ for (int j = 0; j < NUM_OF_KEYS; j++)
+ assertNotNull("Check that cache2 contains key: " + j + " node: " + ignite.name(), cache.get(j));
+ }
+ }
+
+
+ /**
+ * @param ignite Ignite instance.
+ */
+ protected void checkDestroyCachesAbruptly(Ignite ignite) throws Exception {
+ loadCaches(ignite);
+
+ log.warning("Destroying caches");
+
+ ((GatewayProtectedCacheProxy)ignite.cache(cacheName(0))).destroyAsync();
+ ((GatewayProtectedCacheProxy)ignite.cache(cacheName(1))).destroyAsync();
+
+ log.warning("Stopping grid");
+
+ stopAllGrids();
+
+ log.warning("Grid stopped");
+
+ log.warning("Starting grid");
+
+ ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ log.warning("Grid started");
+
+ for(Ignite ig: G.allGrids()) {
+ assertTrue(ig.cacheNames().contains(cacheName(2)));
+
+ IgniteCache cache = ig.cache(cacheName(2));
+
+ for (int j = 0; j < NUM_OF_KEYS; j++)
+ assertNotNull("Check that survived cache cache2 contains key: " + j + " node: " + ig.name(), cache.get(j));
+ }
+ }
+
+ /**
+ * @param ignite Ignite.
+ */
+ protected void startCachesDynamically(Ignite ignite) {
+ List<CacheConfiguration> ccfg = new ArrayList<>(CACHES);
+
+ for (int i = 0; i < CACHES; i++)
+ ccfg.add(new CacheConfiguration<>(cacheName(i))
+ .setBackups(1)
+ .setAffinity(new RendezvousAffinityFunction(false, 32)));
+
+ ignite.createCaches(ccfg);
+ }
+
+ /**
+ * @param ignite Ignite instance.
+ */
+ protected void startGroupCachesDynamically(Ignite ignite) {
+ List<CacheConfiguration> ccfg = new ArrayList<>(CACHES);
+
+ for (int i = 0; i < CACHES; i++)
+ ccfg.add(new CacheConfiguration<>(cacheName(i))
+ .setGroupName(i % 2 == 0 ? "grp-even" : "grp-odd")
+ .setBackups(1)
+ .setAffinity(new RendezvousAffinityFunction(false, 32)));
+
+ ignite.createCaches(ccfg);
+ }
+
+ /**
+ * Generate cache name from idx.
+ *
+ * @param idx Index.
+ */
+ protected String cacheName(int idx) {
+ return "cache" + idx;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8fc18248/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheTest.java
new file mode 100644
index 0000000..99d6f01
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.Ignite;
+
+/**
+ * Test correct clean up cache configuration data after destroying cache.
+ */
+public class IgnitePdsDestroyCacheTest extends IgnitePdsDestroyCacheAbstractTest {
+ /**
+ * Test destroy non grouped caches.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDestroyCaches() throws Exception {
+ Ignite ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ startCachesDynamically(ignite);
+
+ checkDestroyCaches(ignite);
+ }
+
+ /**
+ * Test destroy grouped caches.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDestroyGroupCaches() throws Exception {
+ Ignite ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ startGroupCachesDynamically(ignite);
+
+ checkDestroyCaches(ignite);
+ }
+
+ /**
+ * Test destroy caches abruptly with checkpoints.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDestroyCachesAbruptly() throws Exception {
+ Ignite ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ startCachesDynamically(ignite);
+
+ checkDestroyCachesAbruptly(ignite);
+ }
+
+ /**
+ * Test destroy group caches abruptly with checkpoints.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDestroyGroupCachesAbruptly() throws Exception {
+ Ignite ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ startGroupCachesDynamically(ignite);
+
+ checkDestroyCachesAbruptly(ignite);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8fc18248/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheWithoutCheckpointsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheWithoutCheckpointsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheWithoutCheckpointsTest.java
new file mode 100644
index 0000000..1bb6f5d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheWithoutCheckpointsTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.G;
+
+/**
+ * Check that cluster survives after destroy caches abruptly with disabled checkpoints.
+ */
+public class IgnitePdsDestroyCacheWithoutCheckpointsTest extends IgnitePdsDestroyCacheAbstractTest {
+ /**
+ * {@inheritDoc}
+ * @returns {@code true} in order to be able to kill nodes when checkpointer thread hangs.
+ */
+ @Override protected boolean isMultiJvm() {
+ return true;
+ }
+
+ /**
+ * Test destroy caches with disabled checkpoints.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDestroyCachesAbruptlyWithoutCheckpoints() throws Exception {
+ Ignite ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ startCachesDynamically(ignite);
+
+ disableCheckpoints();
+
+ checkDestroyCachesAbruptly(ignite);
+ }
+
+ /**
+ * Test destroy group caches with disabled checkpoints.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDestroyGroupCachesAbruptlyWithoutCheckpoints() throws Exception {
+ Ignite ignite = startGrids(NODES);
+
+ ignite.cluster().active(true);
+
+ startGroupCachesDynamically(ignite);
+
+ disableCheckpoints();
+
+ checkDestroyCachesAbruptly(ignite);
+ }
+
+ /**
+ * Disable checkpoints on nodes.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ private void disableCheckpoints() throws IgniteCheckedException {
+ for (Ignite ignite : G.allGrids()) {
+ assert !ignite.cluster().localNode().isClient();
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.enableCheckpoints(false).get();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8fc18248/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index af0b7ad..d11ceb3 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -19,7 +19,8 @@ package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistence;
-import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheWithoutCheckpointsTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest;
@@ -114,7 +115,9 @@ public class IgnitePdsTestSuite extends TestSuite {
suite.addTestSuite(IgnitePdsCacheRestoreTest.class);
suite.addTestSuite(IgnitePdsDataRegionMetricsTest.class);
- suite.addTestSuite(IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.class);
+
+ suite.addTestSuite(IgnitePdsDestroyCacheTest.class);
+ suite.addTestSuite(IgnitePdsDestroyCacheWithoutCheckpointsTest.class);
suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
[50/50] [abbrv] ignite git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/ignite into ignite-6083
Posted by ag...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/ignite into ignite-6083
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bea574b9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bea574b9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bea574b9
Branch: refs/heads/ignite-6083
Commit: bea574b93c9f80fd6e31a0348b2bc8cca445ae37
Parents: ecefdd3 ecea655
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Mon Apr 23 10:11:28 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Apr 23 10:11:28 2018 +0300
----------------------------------------------------------------------
.../clustering/DatasetWithObviousStructure.java | 105 ---
.../ml/clustering/FuzzyCMeansExample.java | 134 ----
.../ml/clustering/FuzzyCMeansLocalExample.java | 95 ---
.../clustering/KMeansClusterizationExample.java | 226 +++++++
.../KMeansDistributedClustererExample.java | 97 ---
.../clustering/KMeansLocalClustererExample.java | 106 ---
.../OptimizeMakeChangeTerminateCriteria.java | 2 +-
.../genetic/movie/MovieTerminateCriteria.java | 2 +-
.../ml/knn/KNNClassificationExample.java | 4 +-
.../examples/ml/knn/KNNRegressionExample.java | 310 +++++++++
.../examples/ml/nn/MLPTrainerExample.java | 2 +-
...nWithLSQRTrainerAndNormalizationExample.java | 180 -----
...dLinearRegressionWithLSQRTrainerExample.java | 169 -----
...tedLinearRegressionWithQRTrainerExample.java | 137 ----
...edLinearRegressionWithSGDTrainerExample.java | 177 -----
.../LinearRegressionLSQRTrainerExample.java | 169 +++++
...sionLSQRTrainerWithNormalizationExample.java | 180 +++++
.../LinearRegressionSGDTrainerExample.java | 176 +++++
.../client/suite/IgniteClientTestSuite.java | 6 +-
.../JettyRestProcessorAbstractSelfTest.java | 62 +-
...ettyRestProcessorAuthenticationSelfTest.java | 45 +-
...rocessorAuthenticationWithCredsSelfTest.java | 32 +
...rocessorAuthenticationWithTokenSelfTest.java | 80 +++
.../jdbc/thin/JdbcThinConnectionSSLTest.java | 20 +-
.../org/apache/ignite/DataRegionMetrics.java | 35 +
.../org/apache/ignite/DataStorageMetrics.java | 70 ++
.../apache/ignite/IgniteSystemProperties.java | 7 +
.../org/apache/ignite/cache/QueryEntity.java | 179 ++++-
.../apache/ignite/cache/QueryEntityPatch.java | 118 ++++
.../cache/eviction/AbstractEvictionPolicy.java | 3 +
.../apache/ignite/internal/IgniteKernal.java | 57 +-
.../internal/TransactionMetricsMxBeanImpl.java | 104 +++
.../ignite/internal/binary/BinaryFieldImpl.java | 9 +
.../internal/binary/BinaryObjectImpl.java | 1 +
.../internal/client/thin/ReliableChannel.java | 2 +
.../pagemem/store/IgnitePageStoreManager.java | 9 +
.../affinity/GridAffinityAssignment.java | 4 +-
.../affinity/GridAffinityAssignmentCache.java | 2 +-
.../processors/cache/CacheGroupContext.java | 54 +-
.../cache/CacheGroupMetricsMXBeanImpl.java | 86 ++-
.../cache/CacheJoinNodeDiscoveryData.java | 15 +-
.../processors/cache/ClusterCachesInfo.java | 435 +++++++++---
.../cache/DynamicCacheDescriptor.java | 28 +
.../processors/cache/ExchangeActions.java | 2 +-
.../cache/GatewayProtectedCacheProxy.java | 676 ++++++++-----------
.../processors/cache/GridCacheAdapter.java | 3 +-
.../processors/cache/GridCacheGateway.java | 7 +
.../processors/cache/GridCacheMapEntry.java | 4 +-
.../GridCachePartitionExchangeManager.java | 9 +-
.../processors/cache/GridCacheProcessor.java | 105 ++-
.../cache/GridCacheSharedContext.java | 6 +-
.../processors/cache/GridCacheTtlManager.java | 10 +-
.../cache/IgniteCacheOffheapManagerImpl.java | 28 +-
.../processors/cache/IgniteCacheProxyImpl.java | 31 +-
.../cache/LocalJoinCachesContext.java | 15 -
.../processors/cache/WalStateManager.java | 160 ++++-
.../dht/GridClientPartitionTopology.java | 5 +
.../dht/GridDhtPartitionTopology.java | 10 +-
.../dht/GridDhtPartitionTopologyImpl.java | 247 +++----
.../dht/GridDhtPartitionsStateValidator.java | 61 +-
.../dht/atomic/GridDhtAtomicCache.java | 3 +-
.../dht/preloader/GridDhtPartitionDemander.java | 142 ++--
.../dht/preloader/GridDhtPartitionSupplier.java | 54 +-
.../GridDhtPartitionsExchangeFuture.java | 8 +-
.../GridDhtPartitionsSingleMessage.java | 18 +-
.../dht/preloader/GridDhtPreloader.java | 8 +-
.../IgniteDhtPartitionsToReloadMap.java | 2 +-
.../preloader/latch/ExchangeLatchManager.java | 27 +-
...arOptimisticSerializableTxPrepareFuture.java | 2 +-
.../near/GridNearOptimisticTxPrepareFuture.java | 2 +-
.../GridNearPessimisticTxPrepareFuture.java | 2 +-
.../near/GridNearTxFastFinishFuture.java | 10 +-
.../cache/distributed/near/GridNearTxLocal.java | 57 +-
.../local/atomic/GridLocalAtomicCache.java | 2 +-
.../persistence/DataRegionMetricsImpl.java | 98 ++-
.../DataRegionMetricsMXBeanImpl.java | 25 +
.../persistence/DataRegionMetricsSnapshot.java | 45 ++
.../persistence/DataStorageMetricsImpl.java | 195 +++++-
.../persistence/DataStorageMetricsSnapshot.java | 90 +++
.../GridCacheDatabaseSharedManager.java | 168 +++--
.../persistence/GridCacheOffheapManager.java | 82 ++-
.../IgniteCacheDatabaseSharedManager.java | 41 +-
.../persistence/file/FilePageStoreManager.java | 82 ++-
.../persistence/pagemem/PageMemoryImpl.java | 18 +-
.../wal/FileWriteAheadLogManager.java | 27 +-
.../wal/FsyncModeFileWriteAheadLogManager.java | 25 +-
.../transactions/IgniteTxLocalAdapter.java | 5 +-
.../cache/transactions/IgniteTxManager.java | 5 +-
.../transactions/TransactionMetricsAdapter.java | 206 +++++-
.../cluster/DiscoveryDataClusterState.java | 7 +
.../cluster/GridClusterStateProcessor.java | 8 +-
.../platform/cache/PlatformCache.java | 37 +
.../internal/processors/query/QueryField.java | 10 +
.../internal/processors/query/QuerySchema.java | 84 ++-
.../processors/query/QuerySchemaPatch.java | 96 +++
.../processors/rest/GridRestCommand.java | 3 +
.../processors/rest/GridRestProcessor.java | 26 +-
.../auth/AuthenticationCommandHandler.java | 70 ++
.../rest/handlers/auth/package-info.java | 22 +
.../service/GridServiceProcessor.java | 20 +-
.../ignite/internal/worker/WorkersRegistry.java | 4 +-
.../ignite/mxbean/CacheGroupMetricsMXBean.java | 24 +
.../ignite/mxbean/DataRegionMetricsMXBean.java | 20 +
.../ignite/mxbean/DataStorageMetricsMXBean.java | 40 ++
.../ignite/mxbean/TransactionMetricsMxBean.java | 86 +++
.../ignite/spi/discovery/DiscoverySpiMBean.java | 84 +++
.../ignite/spi/discovery/tcp/ClientImpl.java | 6 +-
.../spi/discovery/tcp/TcpDiscoverySpi.java | 10 +
.../spi/discovery/tcp/TcpDiscoverySpiMBean.java | 13 +-
.../ignite/transactions/TransactionMetrics.java | 52 ++
modules/core/src/test/config/log4j-test.xml | 6 -
.../apache/ignite/client/FunctionalTest.java | 37 +-
.../IgniteClientReconnectCacheTest.java | 6 -
.../IgniteClientReconnectDelayedSpiTest.java | 103 +++
.../ignite/internal/IgniteClientRejoinTest.java | 13 +-
.../TransactionMetricsMxBeanImplTest.java | 302 +++++++++
.../binary/BinaryFieldExtractionSelfTest.java | 35 +
...erActivateDeactivateTestWithPersistence.java | 18 +-
.../distributed/CacheBaselineTopologyTest.java | 137 +++-
...CacheLoadingConcurrentGridStartSelfTest.java | 3 +
.../GridCachePartitionsStateValidationTest.java | 4 +
...idCachePartitionsStateValidatorSelfTest.java | 47 +-
.../dht/TxRecoveryStoreEnabledTest.java | 3 +
...PdsAtomicCacheHistoricalRebalancingTest.java | 40 ++
.../IgnitePdsCacheRebalancingAbstractTest.java | 32 +-
.../IgnitePdsDestroyCacheAbstractTest.java | 207 ++++++
.../persistence/IgnitePdsDestroyCacheTest.java | 85 +++
...tePdsDestroyCacheWithoutCheckpointsTest.java | 86 +++
...gnitePdsRecoveryAfterFileCorruptionTest.java | 29 +-
...nitePdsTxCacheHistoricalRebalancingTest.java | 39 ++
...lWalModeChangeDuringRebalancingSelfTest.java | 571 ++++++++++++++++
.../db/wal/IgniteWalRebalanceTest.java | 164 +++++
.../pagemem/NoOpPageStoreManager.java | 5 +
.../IgniteCacheQueryCacheDestroySelfTest.java | 4 +
...timisticDeadlockDetectionCrossCacheTest.java | 147 ++--
.../transactions/TxRollbackOnTimeoutTest.java | 2 +
.../database/SwapPathConstructionSelfTest.java | 4 +-
.../ServiceDeploymentOnActivationTest.java | 244 +++++++
.../ServiceDeploymentOutsideBaselineTest.java | 280 ++++++++
.../util/future/GridFutureAdapterSelfTest.java | 2 +-
.../junits/common/GridCommonAbstractTest.java | 2 +-
.../IgniteCacheMetricsSelfTestSuite.java | 3 +
.../IgniteClientReconnectTestSuite.java | 2 +
.../testsuites/IgniteKernalSelfTestSuite.java | 4 +
.../ignite/testsuites/IgnitePdsTestSuite.java | 5 +
.../ignite/testsuites/IgnitePdsTestSuite2.java | 10 +-
.../testsuites/IgniteStandByClusterSuite.java | 2 +-
.../file/AlignedBuffersDirectFileIO.java | 6 +-
.../persistence/file/IgniteNativeIoLib.java | 10 +
...iveIoPdsRecoveryAfterFileCorruptionTest.java | 28 +
.../testsuites/IgnitePdsNativeIoTestSuite2.java | 4 +
modules/docker/run.sh | 6 +-
.../org/apache/ignite/client/SecurityTest.java | 22 +-
.../ClientReconnectAfterClusterRestartTest.java | 33 +-
.../cache/IgniteDynamicSqlRestoreTest.java | 529 +++++++++++++++
...lientQueryReplicatedNodeRestartSelfTest.java | 1 -
.../IgniteCacheQueryNodeRestartSelfTest.java | 2 +
...ynamicColumnsAbstractConcurrentSelfTest.java | 3 +-
...ngBaselineCacheQueryNodeRestartSelfTest.java | 4 +-
.../query/IgniteSqlNotNullConstraintTest.java | 71 +-
.../IgniteCacheQuerySelfTestSuite.java | 2 +
.../IgnitePdsWithIndexingCoreTestSuite.java | 7 +
.../apache/ignite/stream/jms11/JmsStreamer.java | 28 +
.../stream/jms11/IgniteJmsStreamerTest.java | 81 ++-
.../ignite/ml/FuzzyCMeansModelFormat.java | 76 ---
.../org/apache/ignite/ml/KMeansModelFormat.java | 77 ---
.../main/java/org/apache/ignite/ml/Trainer.java | 36 -
.../ml/clustering/BaseFuzzyCMeansClusterer.java | 90 ---
.../ml/clustering/BaseKMeansClusterer.java | 96 ---
.../apache/ignite/ml/clustering/Clusterer.java | 33 -
.../ml/clustering/ClusterizationModel.java | 29 -
.../FuzzyCMeansDistributedClusterer.java | 512 --------------
.../clustering/FuzzyCMeansLocalClusterer.java | 254 -------
.../ignite/ml/clustering/FuzzyCMeansModel.java | 88 ---
.../clustering/KMeansDistributedClusterer.java | 306 ---------
.../ml/clustering/KMeansLocalClusterer.java | 177 -----
.../ignite/ml/clustering/KMeansModel.java | 113 ----
.../ignite/ml/clustering/WeightedClusterer.java | 38 --
.../ignite/ml/clustering/kmeans/Clusterer.java | 33 +
.../clustering/kmeans/ClusterizationModel.java | 29 +
.../ml/clustering/kmeans/KMeansModel.java | 112 +++
.../ml/clustering/kmeans/KMeansModelFormat.java | 79 +++
.../ml/clustering/kmeans/KMeansTrainer.java | 320 +++++++++
.../ml/clustering/kmeans/package-info.java | 22 +
.../dataset/impl/cache/CacheBasedDataset.java | 14 +-
.../dataset/impl/cache/util/ComputeUtils.java | 9 +-
.../ml/dataset/impl/local/LocalDataset.java | 16 +-
.../dataset/impl/local/LocalDatasetBuilder.java | 8 +-
.../apache/ignite/ml/estimators/Estimators.java | 50 --
.../ignite/ml/estimators/package-info.java | 22 -
.../ignite/ml/genetic/utils/GAGridUtils.java | 27 -
.../java/org/apache/ignite/ml/knn/KNNUtils.java | 10 +-
.../classification/KNNClassificationModel.java | 41 +-
.../ml/knn/partitions/KNNPartitionContext.java | 28 -
.../ignite/ml/knn/partitions/package-info.java | 22 -
.../ml/knn/regression/KNNRegressionModel.java | 7 +-
.../ml/math/functions/IgniteBiFunction.java | 8 +-
.../LinSysPartitionDataBuilderOnHeap.java | 86 ---
.../math/isolve/LinSysPartitionDataOnHeap.java | 65 --
.../ml/math/isolve/lsqr/AbstractLSQR.java | 3 +-
.../ignite/ml/math/isolve/lsqr/LSQROnHeap.java | 27 +-
.../org/apache/ignite/ml/nn/MLPTrainer.java | 1 -
.../apache/ignite/ml/nn/UpdatesStrategy.java | 95 +++
.../ignite/ml/optimization/GradientDescent.java | 202 ------
.../ml/optimization/GradientFunction.java | 31 -
.../LeastSquaresGradientFunction.java | 33 -
.../util/SparseDistributedMatrixMapReducer.java | 84 ---
.../ml/optimization/util/package-info.java | 22 -
.../linear/LinearRegressionLSQRTrainer.java | 10 +-
.../linear/LinearRegressionQRTrainer.java | 72 --
.../linear/LinearRegressionSGDTrainer.java | 7 +-
.../partition/LabelPartitionContext.java | 28 -
.../LabelPartitionDataBuilderOnHeap.java | 1 -
.../preprocessing/LabellingMachine.java | 41 --
.../ml/structures/preprocessing/Normalizer.java | 80 ---
.../svm/SVMLinearBinaryClassificationModel.java | 3 +
.../SVMLinearBinaryClassificationTrainer.java | 9 +-
.../SVMLinearMultiClassClassificationModel.java | 3 +
...VMLinearMultiClassClassificationTrainer.java | 8 +-
.../ignite/ml/svm/SVMPartitionContext.java | 28 -
.../org/apache/ignite/ml/trainers/Trainer.java | 33 -
.../trainers/group/BaseLocalProcessorJob.java | 146 ----
.../ignite/ml/trainers/group/ConstModel.java | 46 --
.../ignite/ml/trainers/group/GroupTrainer.java | 208 ------
.../group/GroupTrainerBaseProcessorTask.java | 144 ----
.../ml/trainers/group/GroupTrainerCacheKey.java | 125 ----
.../group/GroupTrainerEntriesProcessorTask.java | 64 --
.../ml/trainers/group/GroupTrainerInput.java | 37 -
.../group/GroupTrainerKeysProcessorTask.java | 62 --
.../ml/trainers/group/GroupTrainingContext.java | 98 ---
.../group/LocalEntriesProcessorJob.java | 85 ---
.../trainers/group/LocalKeysProcessorJob.java | 78 ---
.../ignite/ml/trainers/group/Metaoptimizer.java | 93 ---
.../group/MetaoptimizerDistributedStep.java | 97 ---
.../group/MetaoptimizerGroupTrainer.java | 132 ----
.../ml/trainers/group/ResultAndUpdates.java | 178 -----
.../ml/trainers/group/UpdateStrategies.java | 47 --
.../ml/trainers/group/UpdatesStrategy.java | 95 ---
.../ignite/ml/trainers/group/chain/Chains.java | 56 --
.../trainers/group/chain/ComputationsChain.java | 246 -------
.../chain/DistributedEntryProcessingStep.java | 34 -
.../chain/DistributedKeyProcessingStep.java | 33 -
.../trainers/group/chain/DistributedStep.java | 70 --
.../trainers/group/chain/EntryAndContext.java | 70 --
.../trainers/group/chain/HasTrainingUUID.java | 32 -
.../ml/trainers/group/chain/KeyAndContext.java | 67 --
.../ml/trainers/group/chain/package-info.java | 22 -
.../ignite/ml/trainers/group/package-info.java | 22 -
.../org/apache/ignite/ml/IgniteMLTestSuite.java | 4 -
.../org/apache/ignite/ml/LocalModelsTest.java | 26 +-
.../ml/clustering/ClusteringTestSuite.java | 7 +-
.../FuzzyCMeansDistributedClustererTest.java | 180 -----
.../FuzzyCMeansLocalClustererTest.java | 202 ------
...KMeansDistributedClustererTestMultiNode.java | 138 ----
...MeansDistributedClustererTestSingleNode.java | 198 ------
.../ml/clustering/KMeansLocalClustererTest.java | 46 --
.../ignite/ml/clustering/KMeansModelTest.java | 63 ++
.../ignite/ml/clustering/KMeansTrainerTest.java | 73 ++
.../apache/ignite/ml/clustering/KMeansUtil.java | 33 -
.../impl/cache/CacheBasedDatasetTest.java | 9 +-
.../org/apache/ignite/ml/knn/BaseKNNTest.java | 89 ---
.../ignite/ml/knn/KNNClassificationTest.java | 54 +-
.../apache/ignite/ml/knn/KNNRegressionTest.java | 52 +-
.../ignite/ml/knn/LabeledDatasetHelper.java | 56 ++
.../ignite/ml/knn/LabeledDatasetTest.java | 54 +-
.../ml/math/isolve/lsqr/LSQROnHeapTest.java | 14 +-
.../ignite/ml/nn/MLPTrainerIntegrationTest.java | 1 -
.../org/apache/ignite/ml/nn/MLPTrainerTest.java | 1 -
.../MLPTrainerMnistIntegrationTest.java | 2 +-
.../ml/nn/performance/MLPTrainerMnistTest.java | 2 +-
.../ml/optimization/GradientDescentTest.java | 64 --
.../ml/optimization/OptimizationTestSuite.java | 33 -
.../SparseDistributedMatrixMapReducerTest.java | 135 ----
.../ml/regressions/RegressionsTestSuite.java | 3 -
.../linear/ArtificialRegressionDatasets.java | 404 -----------
...istributedLinearRegressionQRTrainerTest.java | 36 -
...istributedLinearRegressionQRTrainerTest.java | 36 -
.../GenericLinearRegressionTrainerTest.java | 206 ------
...wareAbstractLinearRegressionTrainerTest.java | 127 ----
.../linear/LinearRegressionSGDTrainerTest.java | 2 +-
.../LocalLinearRegressionQRTrainerTest.java | 36 -
.../group/DistributedWorkersChainTest.java | 189 ------
.../ml/trainers/group/GroupTrainerTest.java | 90 ---
.../trainers/group/SimpleGroupTrainerInput.java | 63 --
.../ml/trainers/group/TestGroupTrainer.java | 144 ----
.../group/TestGroupTrainerLocalContext.java | 85 ---
.../trainers/group/TestGroupTrainingCache.java | 70 --
.../group/TestGroupTrainingSecondCache.java | 56 --
.../ml/trainers/group/TestLocalContext.java | 51 --
.../ml/trainers/group/TestTrainingLoopStep.java | 65 --
.../trainers/group/TrainersGroupTestSuite.java | 32 -
.../ExpiryCacheHolderTest.cs | 10 +
.../Apache.Ignite.Core.Tests.csproj | 1 +
.../ApiParity/CacheParityTest.cs | 2 -
.../ApiParity/DataRegionMetricsParityTest.cs | 8 +-
.../ApiParity/DataStorageMetricsParityTest.cs | 14 +-
.../ApiParity/TransactionMetricsParityTest.cs | 14 +-
.../Cache/CacheQueryMetricsTest.cs | 190 ++++++
.../Cache/CacheTestAsyncWrapper.cs | 10 +
.../Query/CacheQueriesCodeConfigurationTest.cs | 9 +-
.../Client/ClientConnectionTest.cs | 14 +-
.../Apache.Ignite.Core.Tests/ReconnectTest.cs | 6 +-
.../Apache.Ignite.Core.csproj | 2 +
.../dotnet/Apache.Ignite.Core/Cache/ICache.cs | 11 +
.../Apache.Ignite.Core/Cache/IQueryMetrics.cs | 65 ++
.../Client/IgniteClientConfiguration.cs | 4 +-
.../IgniteClientConfigurationSection.xsd | 2 +-
.../Apache.Ignite.Core/Impl/Cache/CacheImpl.cs | 17 +
.../Apache.Ignite.Core/Impl/Cache/CacheOp.cs | 4 +-
.../Impl/Cache/QueryMetricsImpl.cs | 71 ++
.../Impl/Client/ClientSocket.cs | 14 +-
.../http/jetty/GridJettyRestHandler.java | 55 +-
.../ignite/spark/IgniteDataFrameSettings.scala | 9 +
.../org/apache/ignite/spark/IgniteRDD.scala | 7 +-
.../spark/impl/IgniteRelationProvider.scala | 20 +-
.../impl/IgniteSQLAccumulatorRelation.scala | 98 +++
.../spark/impl/IgniteSQLDataFrameRDD.scala | 16 +-
.../ignite/spark/impl/IgniteSQLRelation.scala | 67 +-
.../optimization/AggregateExpressions.scala | 114 ++++
.../optimization/ConditionExpressions.scala | 160 +++++
.../impl/optimization/DateExpressions.scala | 127 ++++
.../impl/optimization/IgniteQueryContext.scala | 52 ++
.../impl/optimization/MathExpressions.scala | 263 ++++++++
.../impl/optimization/SimpleExpressions.scala | 197 ++++++
.../impl/optimization/StringExpressions.scala | 154 +++++
.../optimization/SupportedExpressions.scala | 42 ++
.../impl/optimization/SystemExpressions.scala | 122 ++++
.../accumulator/JoinSQLAccumulator.scala | 222 ++++++
.../accumulator/QueryAccumulator.scala | 70 ++
.../accumulator/SelectAccumulator.scala | 70 ++
.../accumulator/SingleTableSQLAccumulator.scala | 124 ++++
.../accumulator/UnionSQLAccumulator.scala | 63 ++
.../spark/impl/optimization/package.scala | 230 +++++++
.../org/apache/ignite/spark/impl/package.scala | 48 +-
.../spark/sql/ignite/IgniteOptimization.scala | 436 ++++++++++++
.../spark/sql/ignite/IgniteSparkSession.scala | 10 +-
.../ignite/spark/AbstractDataFrameSpec.scala | 68 +-
.../apache/ignite/spark/IgniteCatalogSpec.scala | 7 +-
.../spark/IgniteDataFrameSchemaSpec.scala | 5 +-
.../ignite/spark/IgniteDataFrameSuite.scala | 9 +-
.../IgniteOptimizationAggregationFuncSpec.scala | 189 ++++++
.../spark/IgniteOptimizationDateFuncSpec.scala | 230 +++++++
.../IgniteOptimizationDisableEnableSpec.scala | 127 ++++
.../spark/IgniteOptimizationJoinSpec.scala | 543 +++++++++++++++
.../spark/IgniteOptimizationMathFuncSpec.scala | 358 ++++++++++
.../ignite/spark/IgniteOptimizationSpec.scala | 305 +++++++++
.../IgniteOptimizationStringFuncSpec.scala | 313 +++++++++
.../IgniteOptimizationSystemFuncSpec.scala | 147 ++++
modules/sqlline/bin/sqlline.sh | 7 +-
.../components/cluster-edit-form/controller.js | 24 +-
.../cluster-edit-form/controller.spec.js | 81 +++
.../app/components/page-configure/index.js | 1 -
.../app/components/page-signin/style.scss | 10 +
.../app/components/page-signin/template.pug | 39 +-
.../generator/ConfigurationGenerator.js | 106 +--
.../configuration/generator/Docker.service.js | 94 ++-
.../generator/Docker.service.spec.js | 133 ++++
.../frontend/app/modules/loading/loading.scss | 2 +-
.../app/modules/loading/loading.service.js | 6 +-
.../app/primitives/form-field/index.scss | 12 +
.../app/primitives/ui-grid-header/index.scss | 33 +-
.../frontend/app/primitives/ui-grid/index.scss | 11 +-
modules/web-console/frontend/package-lock.json | 614 ++++++++---------
modules/web-console/frontend/package.json | 1 +
.../frontend/public/stylesheets/style.scss | 10 +
modules/web-console/frontend/views/index.pug | 2 +-
.../ignite/console/agent/AgentLauncher.java | 11 +-
...uzzyCMeansDistributedClustererBenchmark.java | 130 ----
...gniteFuzzyCMeansLocalClustererBenchmark.java | 93 ---
...niteKMeansDistributedClustererBenchmark.java | 75 --
.../IgniteKMeansLocalClustererBenchmark.java | 50 --
.../yardstick/ml/clustering/package-info.java | 22 -
...iteOLSMultipleLinearRegressionBenchmark.java | 69 --
.../yardstick/ml/regression/package-info.java | 22 -
.../spi/discovery/zk/ZookeeperDiscoverySpi.java | 93 ++-
.../zk/ZookeeperDiscoverySpiMBean.java | 67 ++
.../discovery/zk/internal/ZookeeperClient.java | 7 +
.../zk/internal/ZookeeperDiscoveryImpl.java | 36 +-
.../internal/ZookeeperDiscoveryStatistics.java | 48 ++
.../zk/internal/ZookeeperDiscoverySpiTest.java | 40 +-
380 files changed, 15831 insertions(+), 11839 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bea574b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bea574b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
----------------------------------------------------------------------
[35/50] [abbrv] ignite git commit: IGNITE-7791 Fix for Ignite Client
Nodes: failed test
IgniteClientReconnectCacheTest.testReconnectCacheDestroyedAndCreat. - Fixes
#3779.
Posted by ag...@apache.org.
IGNITE-7791 Fix for Ignite Client Nodes: failed test IgniteClientReconnectCacheTest.testReconnectCacheDestroyedAndCreat. - Fixes #3779.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9796ec41
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9796ec41
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9796ec41
Branch: refs/heads/ignite-6083
Commit: 9796ec41c4925f56f7a8d7f8097d2257f592b559
Parents: 09879b8
Author: Maxim Muzafarov <ma...@gmail.com>
Authored: Thu Apr 19 17:26:50 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Thu Apr 19 17:26:50 2018 +0300
----------------------------------------------------------------------
.../processors/cache/ClusterCachesInfo.java | 7 +-
.../cache/LocalJoinCachesContext.java | 15 ---
.../IgniteClientReconnectDelayedSpiTest.java | 103 +++++++++++++++++++
.../IgniteClientReconnectTestSuite.java | 2 +
4 files changed, 106 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9796ec41/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 975617e..70ff110 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -1905,7 +1905,6 @@ class ClusterCachesInfo {
Set<Integer> stoppedCacheGrps = new HashSet<>();
Set<String> survivedCaches = new HashSet<>();
- Set<Integer> survivedCacheGrps = new HashSet<>();
if (!active) {
joinOnTransition = transition;
@@ -1948,11 +1947,8 @@ class ClusterCachesInfo {
if (stopped)
stoppedCacheGrps.add(locDesc.groupId());
- else {
+ else
assert locDesc.groupId() == desc.groupId();
-
- survivedCacheGrps.add(locDesc.groupId());
- }
}
for (Map.Entry<String, DynamicCacheDescriptor> e : cachesOnDisconnect.caches.entrySet()) {
@@ -1977,7 +1973,6 @@ class ClusterCachesInfo {
}
if (locJoinCachesCtx != null) {
- locJoinCachesCtx.removeSurvivedCacheGroups(survivedCacheGrps);
locJoinCachesCtx.removeSurvivedCaches(survivedCaches);
if (locJoinCachesCtx.isEmpty())
http://git-wip-us.apache.org/repos/asf/ignite/blob/9796ec41/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java
index db829b2..f41df60 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java
@@ -86,11 +86,6 @@ public class LocalJoinCachesContext {
* @param cacheNames Survived caches to clean.
*/
public void removeSurvivedCaches(Set<String> cacheNames) {
- if (cacheDescs != null) {
- for (String cacheName : cacheNames)
- cacheDescs.remove(cacheName);
- }
-
Iterator<T2<DynamicCacheDescriptor, NearCacheConfiguration>> it = locJoinStartCaches.iterator();
for (; it.hasNext();) {
@@ -104,16 +99,6 @@ public class LocalJoinCachesContext {
}
/**
- * @param cacheGrps Survived caches groups to clean.
- */
- public void removeSurvivedCacheGroups(Set<Integer> cacheGrps) {
- if (cacheGrpDescs != null) {
- for (Integer grpId : cacheGrps)
- cacheGrpDescs.remove(grpId);
- }
- }
-
- /**
* @return {@code True} if the context is empty.
*/
public boolean isEmpty() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/9796ec41/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDelayedSpiTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDelayedSpiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDelayedSpiTest.java
new file mode 100644
index 0000000..a4a0912
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDelayedSpiTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsAbstractMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Test cases for emulation of delayed messages sending with {@link TestRecordingCommunicationSpi} for blocking and
+ * resending messages at the moment we need it.
+ */
+public class IgniteClientReconnectDelayedSpiTest extends IgniteClientReconnectAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ TestRecordingCommunicationSpi spi = new TestRecordingCommunicationSpi();
+
+ cfg.setCommunicationSpi(spi);
+ cfg.setCacheConfiguration(new CacheConfiguration("preconfigured-cache"));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int serverCount() {
+ return 3;
+ }
+
+ /**
+ * Test checks correctness of stale {@link CacheAffinityChangeMessage} processing by client node as delayed
+ * {@link GridDhtPartitionsSingleMessage} with exchId = null sends after client node reconnect happens.
+ *
+ * @throws Exception If failed.
+ */
+ public void testReconnectCacheDestroyedDelayedAffinityChange() throws Exception {
+ Ignite ignite = ignite(1);
+
+ TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(ignite);
+
+ spi.blockMessages(GridDhtPartitionsSingleMessage.class, ignite.name());
+ spi.blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+ @Override public boolean apply(ClusterNode node, Message msg) {
+ return (msg instanceof GridDhtPartitionsSingleMessage) &&
+ ((GridDhtPartitionsAbstractMessage)msg).exchangeId() == null;
+ }
+ });
+
+ final Ignite client = startGrid(getConfiguration().setClientMode(true));
+
+ client.getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME));
+
+ final Ignite srv = clientRouter(client);
+
+ reconnectClientNode(client, srv, new Runnable() {
+ @Override public void run() {
+ srv.destroyCache(DEFAULT_CACHE_NAME);
+
+ srv.getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME));
+ }
+ });
+
+ // Resend delayed GridDhtPartitionsSingleMessage.
+ spi.waitForBlocked();
+ spi.stopBlock();
+
+ assertNotNull(client.cache(DEFAULT_CACHE_NAME));
+
+ final GridDiscoveryManager srvDisco = ((IgniteEx)srv).context().discovery();
+
+ assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return F.eq(true, srvDisco.cacheClientNode(client.cluster().localNode(), DEFAULT_CACHE_NAME));
+ }
+ }, 5000));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9796ec41/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
index d0e907c..d5ebd15 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
@@ -26,6 +26,7 @@ import org.apache.ignite.internal.IgniteClientReconnectCacheTest;
import org.apache.ignite.internal.IgniteClientReconnectCollectionsTest;
import org.apache.ignite.internal.IgniteClientReconnectComputeTest;
import org.apache.ignite.internal.IgniteClientReconnectContinuousProcessorTest;
+import org.apache.ignite.internal.IgniteClientReconnectDelayedSpiTest;
import org.apache.ignite.internal.IgniteClientReconnectDiscoveryStateTest;
import org.apache.ignite.internal.IgniteClientReconnectFailoverTest;
import org.apache.ignite.internal.IgniteClientReconnectServicesTest;
@@ -49,6 +50,7 @@ public class IgniteClientReconnectTestSuite extends TestSuite {
suite.addTestSuite(IgniteClientReconnectApiExceptionTest.class);
suite.addTestSuite(IgniteClientReconnectDiscoveryStateTest.class);
suite.addTestSuite(IgniteClientReconnectCacheTest.class);
+ suite.addTestSuite(IgniteClientReconnectDelayedSpiTest.class);
suite.addTestSuite(IgniteClientReconnectBinaryContexTest.class);
suite.addTestSuite(IgniteClientReconnectContinuousProcessorTest.class);
suite.addTestSuite(IgniteClientReconnectComputeTest.class);
[07/50] [abbrv] ignite git commit: IGNITE-8166 PME hangs when error
occurs during checkpoint
Posted by ag...@apache.org.
IGNITE-8166 PME hangs when error occurs during checkpoint
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e254e658
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e254e658
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e254e658
Branch: refs/heads/ignite-6083
Commit: e254e6588b7983ed8c13832a5512e662fb8c4215
Parents: b810bd4
Author: Aleksey Plekhanov <pl...@gmail.com>
Authored: Tue Apr 17 15:56:36 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Apr 17 15:56:36 2018 +0300
----------------------------------------------------------------------
.../cache/persistence/GridCacheDatabaseSharedManager.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e254e658/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index caf27b7..a371aa8 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -3667,7 +3667,14 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
private GridFutureAdapter cpBeginFut = new GridFutureAdapter<>();
/** */
- private GridFutureAdapter cpFinishFut = new GridFutureAdapter<>();
+ private GridFutureAdapter cpFinishFut = new GridFutureAdapter<Void>() {
+ @Override protected boolean onDone(@Nullable Void res, @Nullable Throwable err, boolean cancel) {
+ if (err != null && !cpBeginFut.isDone())
+ cpBeginFut.onDone(err);
+
+ return super.onDone(res, err, cancel);
+ }
+ };
/** */
private volatile boolean nextSnapshot;
[45/50] [abbrv] ignite git commit: IGNITE-8134 Subscribe to system
cache events on nodes outside BLT
Posted by ag...@apache.org.
IGNITE-8134 Subscribe to system cache events on nodes outside BLT
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c82277eb
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c82277eb
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c82277eb
Branch: refs/heads/ignite-6083
Commit: c82277eb4e48f95dfec8cb0206c019820a765432
Parents: fbe24f8
Author: Denis Mekhanikov <dm...@gmail.com>
Authored: Fri Apr 20 18:41:06 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Fri Apr 20 18:41:06 2018 +0300
----------------------------------------------------------------------
.../cluster/DiscoveryDataClusterState.java | 7 +
.../service/GridServiceProcessor.java | 14 +-
.../ServiceDeploymentOutsideBaselineTest.java | 280 +++++++++++++++++++
.../testsuites/IgniteKernalSelfTestSuite.java | 2 +
4 files changed, 300 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c82277eb/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
index b022754..8405410 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
@@ -201,6 +201,13 @@ public class DiscoveryDataClusterState implements Serializable {
}
/**
+ * @return {@code True} if baseline topology is set in the cluster. {@code False} otherwise.
+ */
+ public boolean hasBaselineTopology() {
+ return baselineTopology != null;
+ }
+
+ /**
* @return Nodes participating in state change exchange.
*/
public Set<UUID> transitionNodes() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/c82277eb/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 9cf27d2..b8022a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -72,6 +72,7 @@ import org.apache.ignite.internal.processors.cache.binary.MetadataUpdateProposed
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
import org.apache.ignite.internal.processors.cache.query.CacheQuery;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
@@ -244,10 +245,17 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
if (ctx.deploy().enabled())
ctx.cache().context().deploy().ignoreOwnership(true);
- if (!ctx.clientNode() && serviceCache.context().affinityNode()) {
- // Register query listener and run it for local entries. It is also invoked on rebalancing.
+ if (!ctx.clientNode()) {
+ DiscoveryDataClusterState clusterState = ctx.state().clusterState();
+
+ boolean isLocLsnr = !clusterState.hasBaselineTopology() ||
+ CU.baselineNode(ctx.cluster().get().localNode(), clusterState);
+
+ // Register query listener and run it for local entries, if data is available locally.
+ // It is also invoked on rebalancing.
+ // Otherwise remote listener is registered.
serviceCache.context().continuousQueries().executeInternalQuery(
- new ServiceEntriesListener(), null, true, true, false
+ new ServiceEntriesListener(), null, isLocLsnr, true, false
);
}
else { // Listener for client nodes is registered in onContinuousProcessorStarted method.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c82277eb/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceDeploymentOutsideBaselineTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceDeploymentOutsideBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceDeploymentOutsideBaselineTest.java
new file mode 100644
index 0000000..1a5b630
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceDeploymentOutsideBaselineTest.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCluster;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/** */
+public class ServiceDeploymentOutsideBaselineTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final String SERVICE_NAME = "test-service";
+
+ /** */
+ private boolean persistence;
+
+ /** */
+ private ServiceConfiguration srvcCfg;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ TcpDiscoverySpi discoverySpi = new TcpDiscoverySpi();
+ discoverySpi.setIpFinder(IP_FINDER);
+ cfg.setDiscoverySpi(discoverySpi);
+
+ if (persistence) {
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setMaxSize(10 * 1024 * 1024)
+ ).setWalMode(WALMode.LOG_ONLY)
+ );
+ }
+
+ if (srvcCfg != null)
+ cfg.setServiceConfiguration(srvcCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ persistence = false;
+ srvcCfg = null;
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeployOutsideBaseline() throws Exception {
+ checkDeploymentFromOutsideNode(true, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeployOutsideBaselineNoPersistence() throws Exception {
+ checkDeploymentFromOutsideNode(false, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeployOutsideBaselineStatic() throws Exception {
+ checkDeploymentFromOutsideNode(true, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeployOutsideBaselineStaticNoPersistence() throws Exception {
+ checkDeploymentFromOutsideNode(false, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeployFromNodeAddedToBlt() throws Exception {
+ checkDeployWithNodeAddedToBlt(true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeployToNodeAddedToBlt() throws Exception {
+ checkDeployWithNodeAddedToBlt(false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeployFromNodeRemovedFromBlt() throws Exception {
+ checkDeployFromNodeRemovedFromBlt(true, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeployFromNodeRemovedFromBltStatic() throws Exception {
+ checkDeployFromNodeRemovedFromBlt(true, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDeployToNodeRemovedFromBlt() throws Exception {
+ checkDeployFromNodeRemovedFromBlt(false, false);
+ }
+
+ /**
+ * @param persistence If {@code true}, then persistence will be enabled.
+ * @param staticDeploy If {@code true}, then static deployment will be used instead of a dynamic one.
+ * @throws Exception If failed.
+ */
+ private void checkDeploymentFromOutsideNode(boolean persistence, boolean staticDeploy) throws Exception {
+ this.persistence = persistence;
+
+ Ignite insideNode = startGrid(0);
+
+ if (persistence)
+ insideNode.cluster().active(true);
+ else {
+ IgniteCluster cluster = insideNode.cluster();
+
+ cluster.setBaselineTopology(cluster.topologyVersion());
+ }
+
+ CountDownLatch exeLatch = new CountDownLatch(1);
+
+ DummyService.exeLatch(SERVICE_NAME, exeLatch);
+
+ deployServiceFromNewNode(staticDeploy);
+
+ assertTrue(exeLatch.await(10, TimeUnit.SECONDS));
+ }
+
+ /**
+ * @param from If {@code true}, then added node will be an initiator of deployment.
+ * Otherwise deployment <b>to</b> this node will be tested.
+ * @throws Exception If failed.
+ */
+ private void checkDeployWithNodeAddedToBlt(boolean from) throws Exception {
+ persistence = true;
+
+ Ignite insideNode = startGrid(0);
+
+ IgniteCluster cluster = insideNode.cluster();
+
+ cluster.active(true);
+
+ Ignite outsideNode = startGrid(1);
+
+ cluster.setBaselineTopology(cluster.topologyVersion());
+
+ CountDownLatch exeLatch = new CountDownLatch(from ? 1 : 2);
+
+ DummyService.exeLatch(SERVICE_NAME, exeLatch);
+
+ if (from) {
+ IgniteFuture<Void> depFut = outsideNode.services().deployClusterSingletonAsync(SERVICE_NAME, new DummyService());
+
+ depFut.get(10, TimeUnit.SECONDS);
+ }
+ else {
+ IgniteFuture<Void> depFut = outsideNode.services().deployNodeSingletonAsync(SERVICE_NAME, new DummyService());
+
+ depFut.get(10, TimeUnit.SECONDS);
+ }
+
+ assertTrue(exeLatch.await(10, TimeUnit.SECONDS));
+ }
+
+ /**
+ * @param from If {@code true}, then added node will be an initiator of deployment.
+ * Otherwise deployment <b>to</b> this node will be tested.
+ * @param staticDeploy If {@code true}, then static deployment will be used instead of a dynamic one.
+ * @throws Exception If failed.
+ */
+ private void checkDeployFromNodeRemovedFromBlt(boolean from, boolean staticDeploy) throws Exception {
+ persistence = true;
+
+ Ignite insideNode = startGrid(0);
+ startGrid(1);
+
+ IgniteCluster cluster = insideNode.cluster();
+
+ cluster.active(true);
+
+ stopGrid(1);
+
+ cluster.setBaselineTopology(cluster.topologyVersion());
+
+ CountDownLatch exeLatch = new CountDownLatch(from ? 1 : 2);
+
+ DummyService.exeLatch(SERVICE_NAME, exeLatch);
+
+ if (from)
+ deployServiceFromNewNode(staticDeploy);
+ else {
+ startGrid(1);
+
+ IgniteFuture<Void> depFut = insideNode.services().deployNodeSingletonAsync(SERVICE_NAME, new DummyService());
+
+ depFut.get(10, TimeUnit.SECONDS);
+ }
+
+ assertTrue(exeLatch.await(10, TimeUnit.SECONDS));
+ }
+
+ /**
+ * @param staticDeploy If {@code true}, then static deployment will be used instead of a dynamic one.
+ * @throws Exception If node failed to start.
+ */
+ private void deployServiceFromNewNode(boolean staticDeploy) throws Exception {
+ if (staticDeploy) {
+ srvcCfg = getClusterSingletonServiceConfiguration();
+
+ startGrid(1);
+ }
+ else {
+ Ignite node = startGrid(1);
+
+ IgniteFuture<Void> depFut = node.services().deployClusterSingletonAsync(SERVICE_NAME, new DummyService());
+
+ depFut.get(10, TimeUnit.SECONDS);
+ }
+ }
+
+ /**
+ * @return Test service configuration.
+ */
+ private ServiceConfiguration getClusterSingletonServiceConfiguration() {
+ ServiceConfiguration srvcCfg = new ServiceConfiguration();
+ srvcCfg.setName(SERVICE_NAME);
+ srvcCfg.setService(new DummyService());
+ srvcCfg.setTotalCount(1);
+
+ return srvcCfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c82277eb/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index 12e4802..8e3411a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -69,6 +69,7 @@ import org.apache.ignite.internal.processors.service.IgniteServiceDynamicCachesS
import org.apache.ignite.internal.processors.service.IgniteServiceProxyTimeoutInitializedTest;
import org.apache.ignite.internal.processors.service.IgniteServiceReassignmentTest;
import org.apache.ignite.internal.processors.service.ServiceDeploymentOnActivationTest;
+import org.apache.ignite.internal.processors.service.ServiceDeploymentOutsideBaselineTest;
import org.apache.ignite.internal.processors.service.ServicePredicateAccessCacheTest;
import org.apache.ignite.internal.util.GridStartupWithUndefinedIgniteHomeSelfTest;
import org.apache.ignite.services.ServiceThreadPoolSelfTest;
@@ -148,6 +149,7 @@ public class IgniteKernalSelfTestSuite extends TestSuite {
suite.addTestSuite(GridServiceProcessorBatchDeploySelfTest.class);
suite.addTestSuite(GridServiceDeploymentCompoundFutureSelfTest.class);
suite.addTestSuite(ServiceDeploymentOnActivationTest.class);
+ suite.addTestSuite(ServiceDeploymentOutsideBaselineTest.class);
suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
suite.addTestSuite(IgniteServiceDeploymentClassLoadingJdkMarshallerTest.class);
[44/50] [abbrv] ignite git commit: ignite-8205 Clear list of local
services in GridServiceProcessor#onKernalStop
Posted by ag...@apache.org.
ignite-8205 Clear list of local services in GridServiceProcessor#onKernalStop
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/fbe24f8e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/fbe24f8e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/fbe24f8e
Branch: refs/heads/ignite-6083
Commit: fbe24f8e3b0d9016a69670ca2bc50766865adf38
Parents: 358d039
Author: Denis Mekhanikov <dm...@gmail.com>
Authored: Fri Apr 20 17:11:36 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Fri Apr 20 17:11:36 2018 +0300
----------------------------------------------------------------------
.../service/GridServiceProcessor.java | 8 +-
.../ServiceDeploymentOnActivationTest.java | 244 +++++++++++++++++++
.../testsuites/IgniteKernalSelfTestSuite.java | 2 +
3 files changed, 252 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/fbe24f8e/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 63f5027..9cf27d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -29,6 +29,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
@@ -107,7 +108,6 @@ import org.apache.ignite.thread.IgniteThreadFactory;
import org.apache.ignite.thread.OomExceptionHandler;
import org.apache.ignite.transactions.Transaction;
import org.jetbrains.annotations.Nullable;
-import java.util.concurrent.ConcurrentHashMap;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_SERVICES_COMPATIBILITY_MODE;
import static org.apache.ignite.IgniteSystemProperties.getString;
@@ -245,11 +245,12 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
ctx.cache().context().deploy().ignoreOwnership(true);
if (!ctx.clientNode() && serviceCache.context().affinityNode()) {
+ // Register query listener and run it for local entries. It is also invoked on rebalancing.
serviceCache.context().continuousQueries().executeInternalQuery(
new ServiceEntriesListener(), null, true, true, false
);
}
- else {
+ else { // Listener for client nodes is registered in onContinuousProcessorStarted method.
assert !ctx.isDaemon();
ctx.closure().runLocalSafe(new Runnable() {
@@ -324,6 +325,8 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
synchronized (locSvcs) {
for (Collection<ServiceContextImpl> ctxs0 : locSvcs.values())
ctxs.addAll(ctxs0);
+
+ locSvcs.clear();
}
for (ServiceContextImpl ctx : ctxs) {
@@ -1627,6 +1630,7 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
ClusterNode oldest = U.oldest(ctx.discovery().nodes(topVer), null);
+ // Process deployment on coordinator only.
if (oldest.isLocal())
onDeployment(dep, topVer);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/fbe24f8e/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceDeploymentOnActivationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceDeploymentOnActivationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceDeploymentOnActivationTest.java
new file mode 100644
index 0000000..52d706b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceDeploymentOnActivationTest.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/** */
+public class ServiceDeploymentOnActivationTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final String SERVICE_NAME = "test-service";
+
+ /** */
+ private static final IgnitePredicate<ClusterNode> CLIENT_FILTER = new IgnitePredicate<ClusterNode>() {
+ @Override public boolean apply(ClusterNode node) {
+ return node.isClient();
+ }
+ };
+
+ /** */
+ private boolean client;
+
+ /** */
+ private boolean persistence;
+
+ /** */
+ private ServiceConfiguration srvcCfg;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ TcpDiscoverySpi discoverySpi = new TcpDiscoverySpi();
+ discoverySpi.setIpFinder(IP_FINDER);
+ cfg.setDiscoverySpi(discoverySpi);
+
+ cfg.setClientMode(client);
+
+ if (srvcCfg != null)
+ cfg.setServiceConfiguration(srvcCfg);
+
+ if (persistence) {
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setMaxSize(10 * 1024 * 1024)
+ ).setWalMode(WALMode.LOG_ONLY)
+ );
+ }
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ client = false;
+ persistence = false;
+ srvcCfg = null;
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testServersWithPersistence() throws Exception {
+ persistence = true;
+
+ checkRedeployment(2, 0, F.alwaysTrue(), 2, false, true);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testClientsWithPersistence() throws Exception {
+ persistence = true;
+
+ checkRedeployment(2, 2, CLIENT_FILTER, 2, false, true);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testServersWithoutPersistence() throws Exception {
+ persistence = false;
+
+ checkRedeployment(2, 0, F.alwaysTrue(), 2, false, false);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testClientsWithoutPersistence() throws Exception {
+ persistence = false;
+
+ checkRedeployment(2, 2, CLIENT_FILTER, 2, false, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testServersStaticConfigWithPersistence() throws Exception {
+ persistence = true;
+
+ checkRedeployment(2, 0, F.alwaysTrue(), 2, true, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testClientsStaticConfigWithPersistence() throws Exception {
+ persistence = true;
+
+ checkRedeployment(2, 2, CLIENT_FILTER, 2, true, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testServersStaticConfigWithoutPersistence() throws Exception {
+ persistence = false;
+
+ checkRedeployment(2, 0, F.alwaysTrue(), 2, true, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void _testClientsStaticConfigWithoutPersistence() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8279");
+
+ persistence = false;
+
+ checkRedeployment(2, 2, CLIENT_FILTER, 2, true, true);
+ }
+
+ /**
+ * @param srvsNum Number of server nodes to start.
+ * @param clientsNum Number of client nodes to start.
+ * @param nodeFilter Node filter.
+ * @param deps Expected number of deployed services.
+ * @param isStatic Static or dynamic service deployment is used.
+ * @param expRedep {@code true} if services should be redeployed on activation. {@code false} otherwise.
+ * @throws Exception If failed.
+ */
+ private void checkRedeployment(int srvsNum, int clientsNum, IgnitePredicate<ClusterNode> nodeFilter, int deps,
+ boolean isStatic, boolean expRedep) throws Exception {
+
+ if (isStatic)
+ srvcCfg = getServiceConfiguration(nodeFilter);
+
+ CountDownLatch exeLatch = new CountDownLatch(deps);
+ CountDownLatch cancelLatch = new CountDownLatch(deps);
+
+ DummyService.exeLatch(SERVICE_NAME, exeLatch);
+ DummyService.cancelLatch(SERVICE_NAME, cancelLatch);
+
+ for (int i = 0; i < srvsNum; i++)
+ startGrid(i);
+
+ client = true;
+
+ for (int i = 0; i < clientsNum; i++)
+ startGrid(srvsNum + i);
+
+ Ignite ignite = grid(0);
+
+ ignite.cluster().active(true);
+
+ if (!isStatic) {
+ ServiceConfiguration srvcCfg = getServiceConfiguration(nodeFilter);
+
+ ignite.services().deploy(srvcCfg);
+ }
+
+ assertTrue(exeLatch.await(10, TimeUnit.SECONDS));
+
+ ignite.cluster().active(false);
+
+ assertTrue(cancelLatch.await(10, TimeUnit.SECONDS));
+
+ exeLatch = new CountDownLatch(expRedep ? deps : 1);
+
+ DummyService.exeLatch(SERVICE_NAME, exeLatch);
+
+ ignite.cluster().active(true);
+
+ if (expRedep)
+ assertTrue(exeLatch.await(10, TimeUnit.SECONDS));
+ else
+ assertFalse(exeLatch.await(1, TimeUnit.SECONDS));
+ }
+
+ /**
+ * @param nodeFilter Node filter.
+ * @return Service configuration.
+ */
+ private ServiceConfiguration getServiceConfiguration(IgnitePredicate<ClusterNode> nodeFilter) {
+ ServiceConfiguration srvcCfg = new ServiceConfiguration();
+ srvcCfg.setName(SERVICE_NAME);
+ srvcCfg.setMaxPerNodeCount(1);
+ srvcCfg.setNodeFilter(nodeFilter);
+ srvcCfg.setService(new DummyService());
+ return srvcCfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/fbe24f8e/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index f26205b..12e4802 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -68,6 +68,7 @@ import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClas
import org.apache.ignite.internal.processors.service.IgniteServiceDynamicCachesSelfTest;
import org.apache.ignite.internal.processors.service.IgniteServiceProxyTimeoutInitializedTest;
import org.apache.ignite.internal.processors.service.IgniteServiceReassignmentTest;
+import org.apache.ignite.internal.processors.service.ServiceDeploymentOnActivationTest;
import org.apache.ignite.internal.processors.service.ServicePredicateAccessCacheTest;
import org.apache.ignite.internal.util.GridStartupWithUndefinedIgniteHomeSelfTest;
import org.apache.ignite.services.ServiceThreadPoolSelfTest;
@@ -146,6 +147,7 @@ public class IgniteKernalSelfTestSuite extends TestSuite {
suite.addTestSuite(ServiceThreadPoolSelfTest.class);
suite.addTestSuite(GridServiceProcessorBatchDeploySelfTest.class);
suite.addTestSuite(GridServiceDeploymentCompoundFutureSelfTest.class);
+ suite.addTestSuite(ServiceDeploymentOnActivationTest.class);
suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
suite.addTestSuite(IgniteServiceDeploymentClassLoadingJdkMarshallerTest.class);
[13/50] [abbrv] ignite git commit: IGNITE-7770 Test
testRandomMixedTxConfigurations partialy fixed
Posted by ag...@apache.org.
IGNITE-7770 Test testRandomMixedTxConfigurations partialy fixed
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c000fbc5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c000fbc5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c000fbc5
Branch: refs/heads/ignite-6083
Commit: c000fbc5ba0014a24f3b3a81d1607d66159abdc7
Parents: 96cb795
Author: Andrey Kuznetsov <st...@gmail.com>
Authored: Tue Apr 17 19:58:43 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Apr 17 20:02:44 2018 +0300
----------------------------------------------------------------------
.../processors/cache/GridCacheAdapter.java | 3 +-
...arOptimisticSerializableTxPrepareFuture.java | 2 +-
.../near/GridNearOptimisticTxPrepareFuture.java | 2 +-
.../GridNearPessimisticTxPrepareFuture.java | 2 +-
.../near/GridNearTxFastFinishFuture.java | 10 ++--
.../cache/distributed/near/GridNearTxLocal.java | 57 +++++++++++++-------
.../cache/transactions/IgniteTxManager.java | 5 +-
.../transactions/TxRollbackOnTimeoutTest.java | 2 +
8 files changed, 55 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c000fbc5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index c2d0f42..bd613a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -104,6 +104,7 @@ import org.apache.ignite.internal.processors.platform.cache.PlatformCacheEntryFi
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
+import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -4283,7 +4284,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
catch (IgniteCheckedException e1) {
try {
- tx0.rollbackNearTxLocalAsync();
+ tx0.rollbackNearTxLocalAsync(e1 instanceof IgniteTxTimeoutCheckedException);
}
catch (Throwable e2) {
if (e2 != e1)
http://git-wip-us.apache.org/repos/asf/ignite/blob/c000fbc5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
index beb1e16..3e2c84a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
@@ -294,7 +294,7 @@ public class GridNearOptimisticSerializableTxPrepareFuture extends GridNearOptim
boolean txStateCheck = remap ? tx.state() == PREPARING : tx.state(PREPARING);
if (!txStateCheck) {
- if (tx.setRollbackOnly()) {
+ if (tx.isRollbackOnly() || tx.setRollbackOnly()) {
if (tx.timedOut())
onError(null, new IgniteTxTimeoutCheckedException("Transaction timed out and " +
"was rolled back: " + this));
http://git-wip-us.apache.org/repos/asf/ignite/blob/c000fbc5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index 8d8c0b2..2afb096 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -327,7 +327,7 @@ public class GridNearOptimisticTxPrepareFuture extends GridNearOptimisticTxPrepa
boolean txStateCheck = remap ? tx.state() == PREPARING : tx.state(PREPARING);
if (!txStateCheck) {
- if (tx.setRollbackOnly()) {
+ if (tx.isRollbackOnly() || tx.setRollbackOnly()) {
if (tx.remainingTime() == -1)
onError(new IgniteTxTimeoutCheckedException("Transaction timed out and " +
"was rolled back: " + this), false);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c000fbc5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index 4ab6863..54ae85c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -155,7 +155,7 @@ public class GridNearPessimisticTxPrepareFuture extends GridNearTxPrepareFutureA
/** {@inheritDoc} */
@Override public void prepare() {
if (!tx.state(PREPARING)) {
- if (tx.setRollbackOnly()) {
+ if (tx.isRollbackOnly() || tx.setRollbackOnly()) {
if (tx.remainingTime() == -1)
onDone(new IgniteTxTimeoutCheckedException("Transaction timed out and was rolled back: " + tx));
else
http://git-wip-us.apache.org/repos/asf/ignite/blob/c000fbc5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
index 7222697..95e4ded 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
@@ -37,13 +37,17 @@ public class GridNearTxFastFinishFuture extends GridFutureAdapter<IgniteInternal
/** */
private final boolean commit;
+ /** */
+ private final boolean onTimeout;
+
/**
* @param tx Transaction.
* @param commit Commit flag.
*/
- GridNearTxFastFinishFuture(GridNearTxLocal tx, boolean commit) {
+ GridNearTxFastFinishFuture(GridNearTxLocal tx, boolean commit, boolean onTimeout) {
this.tx = tx;
this.commit = commit;
+ this.onTimeout = onTimeout;
}
/** {@inheritDoc} */
@@ -61,7 +65,7 @@ public class GridNearTxFastFinishFuture extends GridFutureAdapter<IgniteInternal
tx.state(PREPARED);
tx.state(COMMITTING);
- tx.context().tm().fastFinishTx(tx, true);
+ tx.context().tm().fastFinishTx(tx, true, true);
tx.state(COMMITTED);
}
@@ -70,7 +74,7 @@ public class GridNearTxFastFinishFuture extends GridFutureAdapter<IgniteInternal
tx.state(PREPARED);
tx.state(ROLLING_BACK);
- tx.context().tm().fastFinishTx(tx, false);
+ tx.context().tm().fastFinishTx(tx, false, !onTimeout);
tx.state(ROLLED_BACK);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c000fbc5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index fc8a9a3..e339750 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -111,6 +111,7 @@ import static org.apache.ignite.internal.processors.cache.transactions.IgniteTxE
import static org.apache.ignite.transactions.TransactionState.COMMITTED;
import static org.apache.ignite.transactions.TransactionState.COMMITTING;
import static org.apache.ignite.transactions.TransactionState.MARKED_ROLLBACK;
+import static org.apache.ignite.transactions.TransactionState.PREPARED;
import static org.apache.ignite.transactions.TransactionState.PREPARING;
import static org.apache.ignite.transactions.TransactionState.ROLLED_BACK;
import static org.apache.ignite.transactions.TransactionState.ROLLING_BACK;
@@ -165,7 +166,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
protected boolean transform;
/** */
- private boolean trackTimeout;
+ private final boolean trackTimeout;
/** */
@GridToStringExclude
@@ -175,7 +176,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
* Empty constructor required for {@link Externalizable}.
*/
public GridNearTxLocal() {
- // No-op.
+ this.trackTimeout = false;
}
/**
@@ -228,8 +229,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
initResult();
- if (timeout() > 0 && !implicit())
- trackTimeout = cctx.time().addTimeoutObject(this);
+ trackTimeout = timeout() > 0 && !implicit() && cctx.time().addTimeoutObject(this);
}
/** {@inheritDoc} */
@@ -3155,8 +3155,13 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
if (!PREP_FUT_UPD.compareAndSet(this, null, fut))
return prepFut;
- if (trackTimeout)
- removeTimeoutHandler();
+ if (trackTimeout) {
+ prepFut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> f) {
+ GridNearTxLocal.this.removeTimeoutHandler();
+ }
+ });
+ }
if (timeout == -1) {
fut.onDone(this, timeoutException());
@@ -3215,7 +3220,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
if (fastFinish()) {
GridNearTxFastFinishFuture fut0;
- if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, true)))
+ if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, true, false)))
return chainFinishFuture(finishFut, true);
fut0.finish();
@@ -3238,7 +3243,9 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
// Make sure that here are no exceptions.
prepareFut.get();
- fut0.finish(true, true);
+ TransactionState state = state();
+
+ fut0.finish(state == PREPARED || state == COMMITTING || state == COMMITTED, true);
}
catch (Error | RuntimeException e) {
COMMIT_ERR_UPD.compareAndSet(GridNearTxLocal.this, null, e);
@@ -3282,7 +3289,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
* @param onTimeout {@code True} if rolled back asynchronously on timeout.
* @return Rollback future.
*/
- private IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final boolean onTimeout) {
+ public IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final boolean onTimeout) {
if (log.isDebugEnabled())
log.debug("Rolling back near tx: " + this);
@@ -3292,13 +3299,13 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
NearTxFinishFuture fut = finishFut;
if (fut != null)
- return chainFinishFuture(finishFut, false);
+ return chainFinishFuture(finishFut, false, !onTimeout);
if (fastFinish()) {
GridNearTxFastFinishFuture fut0;
- if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, false)))
- return chainFinishFuture(finishFut, false);
+ if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, false, onTimeout)))
+ return chainFinishFuture(finishFut, false, !onTimeout);
fut0.finish();
@@ -3308,7 +3315,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
final GridNearTxFinishFuture fut0;
if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFinishFuture<>(cctx, this, false)))
- return chainFinishFuture(finishFut, false);
+ return chainFinishFuture(finishFut, false, !onTimeout);
cctx.mvcc().addFuture(fut0, fut0.futureId());
@@ -3327,7 +3334,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
fut0.finish(false, !onTimeout);
}
- else {
+ else if (!onTimeout) {
prepFut.listen(new CI1<IgniteInternalFuture<?>>() {
@Override public void apply(IgniteInternalFuture<?> f) {
try {
@@ -3339,10 +3346,11 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
log.debug("Got optimistic tx failure [tx=" + this + ", err=" + e + ']');
}
- fut0.finish(false, !onTimeout);
+ fut0.finish(false, true);
}
});
- }
+ } else
+ fut0.finish(false, false);
return fut0;
}
@@ -3352,12 +3360,17 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
return rollbackNearTxLocalAsync();
}
+ /** */
+ private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFinishFuture fut, final boolean commit) {
+ return chainFinishFuture(fut, commit, true);
+ }
+
/**
* @param fut Already started finish future.
* @param commit Commit flag.
* @return Finish future.
*/
- private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFinishFuture fut, final boolean commit) {
+ private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFinishFuture fut, final boolean commit, final boolean clearThreadMap) {
assert fut != null;
if (fut.commit() != commit) {
@@ -3381,7 +3394,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
if (!cctx.mvcc().addFuture(rollbackFut, rollbackFut.futureId()))
return;
- rollbackFut.finish(false, true);
+ rollbackFut.finish(false, clearThreadMap);
}
}
}
@@ -4161,7 +4174,13 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
}
}
- if (state(MARKED_ROLLBACK, true) || (state() == MARKED_ROLLBACK)) {
+ boolean proceed;
+
+ synchronized (this) {
+ proceed = state() != PREPARED && state(MARKED_ROLLBACK, true);
+ }
+
+ if (proceed || (state() == MARKED_ROLLBACK)) {
cctx.kernalContext().closure().runLocalSafe(new Runnable() {
@Override public void run() {
// Note: if rollback asynchronously on timeout should not clear thread map
http://git-wip-us.apache.org/repos/asf/ignite/blob/c000fbc5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index 9fb8777..7fa31bf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -1356,7 +1356,7 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
* @param tx Transaction to finish.
* @param commit {@code True} if transaction is committed, {@code false} if rolled back.
*/
- public void fastFinishTx(GridNearTxLocal tx, boolean commit) {
+ public void fastFinishTx(GridNearTxLocal tx, boolean commit, boolean clearThreadMap) {
assert tx != null;
assert tx.writeMap().isEmpty();
assert tx.optimistic() || tx.readMap().isEmpty();
@@ -1377,7 +1377,8 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
removeObsolete(tx);
// 4. Remove from per-thread storage.
- clearThreadMap(tx);
+ if (clearThreadMap)
+ clearThreadMap(tx);
// 5. Clear context.
resetContext();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c000fbc5/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
index c5be40e..97de81e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
@@ -439,6 +439,8 @@ public class TxRollbackOnTimeoutTest extends GridCommonAbstractTest {
if (delay > 0)
sleep(delay);
+ assert v != null;
+
node.cache(CACHE_NAME).put(k, v + 1);
tx.commit();
[06/50] [abbrv] ignite git commit: IGNITE-8255 Possible name
collisions in WorkersRegistry.
Posted by ag...@apache.org.
IGNITE-8255 Possible name collisions in WorkersRegistry.
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b810bd44
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b810bd44
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b810bd44
Branch: refs/heads/ignite-6083
Commit: b810bd44805b04aa9c6fb3797ecd48b259fa3a8c
Parents: d79c640
Author: Dmitriy Sorokin <sb...@gmail.com>
Authored: Tue Apr 17 14:48:44 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Apr 17 14:48:44 2018 +0300
----------------------------------------------------------------------
.../java/org/apache/ignite/internal/worker/WorkersRegistry.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b810bd44/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
index e8d46fb..16676c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
@@ -36,7 +36,7 @@ public class WorkersRegistry implements GridWorkerListener {
* @param w Worker.
*/
public void register(GridWorker w) {
- if (registeredWorkers.putIfAbsent(w.name(), w) != null)
+ if (registeredWorkers.putIfAbsent(w.runner().getName(), w) != null)
throw new IllegalStateException("Worker is already registered [worker=" + w + ']');
}
@@ -75,6 +75,6 @@ public class WorkersRegistry implements GridWorkerListener {
/** {@inheritDoc} */
@Override public void onStopped(GridWorker w) {
- unregister(w.name());
+ unregister(w.runner().getName());
}
}
[39/50] [abbrv] ignite git commit: IGNITE-7077: Implementation of
Spark query optimization. - Fixes #3397.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/UnionSQLAccumulator.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/UnionSQLAccumulator.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/UnionSQLAccumulator.scala
new file mode 100644
index 0000000..723e17a
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/UnionSQLAccumulator.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization.accumulator
+
+import org.apache.ignite.spark.impl.optimization.{IgniteQueryContext, exprToString, toAttributeReference}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, NamedExpression, SortOrder}
+
+/**
+ * Accumulator to store info about UNION query.
+ */
+private[apache] case class UnionSQLAccumulator(
+ igniteQueryContext: IgniteQueryContext,
+ children: Seq[QueryAccumulator],
+ outputExpressions: Seq[NamedExpression],
+ orderBy: Option[Seq[SortOrder]] = None
+) extends QueryAccumulator {
+ /** @inheritdoc */
+ override def compileQuery(prettyPrint: Boolean = false): String = {
+ val delim = if (prettyPrint) "\n" else " "
+ val tab = if (prettyPrint) " " else ""
+
+ val query = children.map(_.compileQuery(prettyPrint)).mkString(s"${delim}UNION$delim")
+
+ orderBy match {
+ case Some(sortOrders) ⇒
+ query + s"${delim}ORDER BY ${sortOrders.map(exprToString(_)).mkString(s",$delim$tab")}"
+
+ case None ⇒ query
+ }
+ }
+
+ /** @inheritdoc */
+ override def simpleString: String =
+ s"UnionSQLAccumulator(orderBy: ${orderBy.map(_.map(exprToString(_)).mkString(", ")).getOrElse("[]")})"
+
+ /** @inheritdoc */
+ override def withOutputExpressions(outputExpressions: Seq[NamedExpression]): QueryAccumulator =
+ copy(outputExpressions= outputExpressions)
+
+ /** @inheritdoc */
+ override def withOrderBy(orderBy: Seq[SortOrder]): QueryAccumulator = copy(orderBy = Some(orderBy))
+
+ /** @inheritdoc */
+ override def output: Seq[Attribute] = outputExpressions.map(toAttributeReference(_, Seq.empty))
+
+ /** @inheritdoc */
+ override lazy val qualifier: String = igniteQueryContext.uniqueTableAlias
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/package.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/package.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/package.scala
new file mode 100644
index 0000000..4e168f4
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/package.scala
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl
+
+
+import org.apache.ignite.IgniteException
+import org.apache.ignite.spark.impl.optimization.accumulator.{QueryAccumulator, SingleTableSQLAccumulator}
+import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Count}
+import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, ExprId, Expression, NamedExpression}
+import org.apache.spark.sql.types._
+
+import scala.annotation.tailrec
+
+/**
+ */
+package object optimization {
+ /**
+ * Constant to store alias in column metadata.
+ */
+ private[optimization] val ALIAS: String = "alias"
+
+ /**
+ * All `SupportedExpression` implementations.
+ */
+ private val SUPPORTED_EXPRESSIONS: List[SupportedExpressions] = List (
+ SimpleExpressions,
+ SystemExpressions,
+ AggregateExpressions,
+ ConditionExpressions,
+ DateExpressions,
+ MathExpressions,
+ StringExpressions
+ )
+
+ /**
+ * @param expr Expression.
+ * @param useQualifier If true outputs attributes of `expr` with qualifier.
+ * @param useAlias If true outputs `expr` with alias.
+ * @return String representation of expression.
+ */
+ def exprToString(expr: Expression, useQualifier: Boolean = false, useAlias: Boolean = true): String = {
+ @tailrec
+ def exprToString0(expr: Expression, supportedExpressions: List[SupportedExpressions]): Option[String] =
+ if (supportedExpressions.nonEmpty) {
+ val exprStr = supportedExpressions.head.toString(
+ expr,
+ exprToString(_, useQualifier, useAlias = false),
+ useQualifier,
+ useAlias)
+
+ exprStr match {
+ case res: Some[String] ⇒
+ res
+ case None ⇒
+ exprToString0(expr, supportedExpressions.tail)
+ }
+ }
+ else
+ None
+
+ exprToString0(expr, SUPPORTED_EXPRESSIONS) match {
+ case Some(str) ⇒ str
+
+ case None ⇒
+ throw new IgniteException("Unsupporte expression " + expr)
+ }
+ }
+
+ /**
+ * @param exprs Expressions to check.
+ * @return True if `exprs` contains only allowed(i.e. can be pushed down to Ignite) expressions false otherwise.
+ */
+ def exprsAllowed(exprs: Seq[Expression]): Boolean =
+ exprs.forall(exprsAllowed)
+
+ /**
+ * @param expr Expression to check.
+ * @return True if `expr` allowed(i.e. can be pushed down to Ignite) false otherwise.
+ *
+ */
+ def exprsAllowed(expr: Expression): Boolean =
+ SUPPORTED_EXPRESSIONS.exists(_(expr, exprsAllowed))
+
+ /**
+ * Converts `input` into `AttributeReference`.
+ *
+ * @param input Expression to convert.
+ * @param existingOutput Existing output.
+ * @param exprId Optional expression ID to use.
+ * @param alias Optional alias for a result.
+ * @return Converted expression.
+ */
+ def toAttributeReference(input: Expression, existingOutput: Seq[NamedExpression], exprId: Option[ExprId] = None,
+ alias: Option[String] = None): AttributeReference = {
+
+ input match {
+ case attr: AttributeReference ⇒
+ val toCopy = existingOutput.find(_.exprId == attr.exprId).getOrElse(attr)
+
+ AttributeReference(
+ name = toCopy.name,
+ dataType = toCopy.dataType,
+ metadata = alias
+ .map(new MetadataBuilder().withMetadata(toCopy.metadata).putString(ALIAS, _).build())
+ .getOrElse(toCopy.metadata)
+ )(exprId = exprId.getOrElse(toCopy.exprId), qualifier = toCopy.qualifier, isGenerated = toCopy.isGenerated)
+
+ case a: Alias ⇒
+ toAttributeReference(a.child, existingOutput, Some(a.exprId), Some(alias.getOrElse(a.name)))
+
+ case agg: AggregateExpression ⇒
+ agg.aggregateFunction match {
+ case c: Count ⇒
+ if (agg.isDistinct)
+ AttributeReference(
+ name = s"COUNT(DISTINCT ${c.children.map(exprToString(_)).mkString(" ")})",
+ dataType = LongType,
+ metadata = alias
+ .map(new MetadataBuilder().putString(ALIAS, _).build())
+ .getOrElse(Metadata.empty)
+ )(exprId = exprId.getOrElse(agg.resultId))
+ else
+ AttributeReference(
+ name = s"COUNT(${c.children.map(exprToString(_)).mkString(" ")})",
+ dataType = LongType,
+ metadata = alias
+ .map(new MetadataBuilder().putString(ALIAS, _).build())
+ .getOrElse(Metadata.empty)
+ )(exprId = exprId.getOrElse(agg.resultId))
+
+ case _ ⇒
+ toAttributeReference(agg.aggregateFunction, existingOutput, Some(exprId.getOrElse(agg.resultId)), alias)
+ }
+
+ case ne: NamedExpression ⇒
+ AttributeReference(
+ name = exprToString(input),
+ dataType = input.dataType,
+ metadata = alias
+ .map(new MetadataBuilder().withMetadata(ne.metadata).putString(ALIAS, _).build())
+ .getOrElse(Metadata.empty)
+ )(exprId = exprId.getOrElse(ne.exprId))
+
+ case _ if exprsAllowed(input) ⇒
+ AttributeReference(
+ name = exprToString(input),
+ dataType = input.dataType,
+ metadata = alias
+ .map(new MetadataBuilder().putString(ALIAS, _).build())
+ .getOrElse(Metadata.empty)
+ )(exprId = exprId.getOrElse(NamedExpression.newExprId))
+
+ case _ ⇒
+ throw new IgniteException(s"Unsupported column expression $input")
+ }
+ }
+
+ /**
+ * @param dataType Spark data type.
+ * @return SQL data type.
+ */
+ def toSqlType(dataType: DataType): String = dataType match {
+ case BooleanType ⇒ "BOOLEAN"
+ case IntegerType ⇒ "INT"
+ case ByteType ⇒ "TINYINT"
+ case ShortType ⇒ "SMALLINT"
+ case LongType ⇒ "BIGINT"
+ case DecimalType() ⇒ "DECIMAL"
+ case DoubleType ⇒ "DOUBLE"
+ case FloatType ⇒ "REAL"
+ case DateType ⇒ "DATE"
+ case TimestampType ⇒ "TIMESTAMP"
+ case StringType ⇒ "VARCHAR"
+ case BinaryType ⇒ "BINARY"
+ case ArrayType(_, _) ⇒ "ARRAY"
+ case _ ⇒
+ throw new IgniteException(s"$dataType not supported!")
+ }
+
+ /**
+ * @param expr Expression
+ * @return True if expression or some of it children is AggregateExpression, false otherwise.
+ */
+ def hasAggregateInside(expr: Expression): Boolean = {
+ def hasAggregateInside0(expr: Expression): Boolean = expr match {
+ case AggregateExpression(_, _, _, _) ⇒
+ true
+
+ case e: Expression ⇒
+ e.children.exists(hasAggregateInside0)
+ }
+
+ hasAggregateInside0(expr)
+ }
+
+ /**
+ * Check if `acc` representing simple query.
+ * Simple is `SELECT ... FROM table WHERE ... ` like query.
+ * Without aggregation, limits, order, embedded select expressions.
+ *
+ * @param acc Accumulator to check.
+ * @return True if accumulator stores simple query info, false otherwise.
+ */
+ def isSimpleTableAcc(acc: QueryAccumulator): Boolean = acc match {
+ case acc: SingleTableSQLAccumulator if acc.table.isDefined ⇒
+ acc.groupBy.isEmpty &&
+ acc.localLimit.isEmpty &&
+ acc.orderBy.isEmpty &&
+ !acc.distinct &&
+ !acc.outputExpressions.exists(hasAggregateInside)
+
+ case _ ⇒
+ false
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/package.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/package.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/package.scala
index 4634a97..6502c0f 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/package.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/package.scala
@@ -19,12 +19,15 @@ package org.apache.ignite.spark
import org.apache.commons.lang.StringUtils.equalsIgnoreCase
import org.apache.ignite.{Ignite, IgniteException, IgniteState, Ignition}
-import org.apache.ignite.cache.QueryEntity
+import org.apache.ignite.cache.{CacheMode, QueryEntity}
+import org.apache.ignite.cluster.ClusterNode
import org.apache.ignite.configuration.CacheConfiguration
import org.apache.ignite.internal.util.lang.GridFunc.contains
+import org.apache.spark.Partition
import org.apache.spark.sql.catalyst.catalog.SessionCatalog
import scala.collection.JavaConversions._
+import scala.collection.mutable.ArrayBuffer
package object impl {
/**
@@ -129,4 +132,47 @@ package object impl {
*/
def isKeyColumn(table: QueryEntity, column: String): Boolean =
contains(table.getKeyFields, column) || equalsIgnoreCase(table.getKeyFieldName, column)
+
+ /**
+ * Computes spark partitions for a given cache.
+ *
+ * @param ic Ignite context.
+ * @param cacheName Cache name
+ * @return Array of IgniteDataFramPartition
+ */
+ def calcPartitions(ic: IgniteContext, cacheName: String): Array[Partition] = {
+ val cache = ic.ignite().cache[Any, Any](cacheName)
+
+ val ccfg = cache.getConfiguration(classOf[CacheConfiguration[Any, Any]])
+
+ if (ccfg.getCacheMode == CacheMode.REPLICATED) {
+ val serverNodes = ic.ignite().cluster().forCacheNodes(cacheName).forServers().nodes()
+
+ Array(IgniteDataFramePartition(0, serverNodes.head, Stream.from(0).take(1024).toList))
+ }
+ else {
+ val aff = ic.ignite().affinity(cacheName)
+
+ val parts = aff.partitions()
+
+ val nodesToParts = (0 until parts).foldLeft(Map[ClusterNode, ArrayBuffer[Int]]()) {
+ case (nodeToParts, ignitePartIdx) ⇒
+ val primary = aff.mapPartitionToPrimaryAndBackups(ignitePartIdx).head
+
+ if (nodeToParts.contains(primary)) {
+ nodeToParts(primary) += ignitePartIdx
+
+ nodeToParts
+ }
+ else
+ nodeToParts + (primary → ArrayBuffer[Int](ignitePartIdx))
+ }
+
+ val partitions = nodesToParts.zipWithIndex.map { case ((node, nodesParts), i) ⇒
+ IgniteDataFramePartition(i, node, nodesParts.toList)
+ }
+
+ partitions.toArray
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteOptimization.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteOptimization.scala b/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteOptimization.scala
new file mode 100644
index 0000000..b23cd6f
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteOptimization.scala
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.ignite
+
+import org.apache.ignite.IgniteException
+import org.apache.ignite.spark.impl.{IgniteSQLAccumulatorRelation, IgniteSQLRelation, sqlCacheName}
+import org.apache.ignite.spark.impl.optimization.{accumulator, _}
+import org.apache.ignite.spark.impl.optimization.accumulator._
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+
+/**
+ * Query plan optimization for a Ignite based queries.
+ */
+object IgniteOptimization extends Rule[LogicalPlan] with Logging {
+ /** @inheritdoc */
+ override def apply(plan: LogicalPlan): LogicalPlan = {
+ logDebug("")
+ logDebug("== Plan Before Ignite Operator Push Down ==")
+ logDebug(plan.toString())
+
+ val transformed = fixAmbiguousOutput(pushDownOperators(plan))
+
+ logDebug("")
+ logDebug("== Plan After Ignite Operator Push Down ==")
+ logDebug(transformed.toString())
+
+ makeIgniteAccRelation(transformed)
+ }
+
+ /**
+ * Change query plan by accumulating query parts supported by Ignite into `QueryAccumulator`.
+ *
+ * @param plan Query plan.
+ * @return Transformed plan.
+ */
+ private def pushDownOperators(plan: LogicalPlan): LogicalPlan = {
+ val aliasIndexIterator = Stream.from(1).iterator
+
+ //Flag to indicate that some step was skipped due to unsupported expression.
+ //When it true we has to skip entire transformation of higher level Nodes.
+ var stepSkipped = true
+
+ //Applying optimization rules from bottom to up tree nodes.
+ plan.transformUp {
+ //We found basic node to transform.
+ //We create new accumulator and going to the upper layers.
+ case LogicalRelation(igniteSqlRelation: IgniteSQLRelation[_, _], output, _catalogTable) ⇒
+ //Clear flag to optimize each statement separately
+ stepSkipped = false
+
+ val igniteQueryContext = IgniteQueryContext(
+ igniteContext = igniteSqlRelation.ic,
+ sqlContext = igniteSqlRelation.sqlContext,
+ catalogTable = _catalogTable,
+ aliasIndex = aliasIndexIterator,
+ cacheName =
+ sqlCacheName(igniteSqlRelation.ic.ignite(), igniteSqlRelation.tableName)
+ .getOrElse(throw new IgniteException("Unknown table")))
+
+ //Logical Relation is bottomest TreeNode in LogicalPlan.
+ //We replace it with accumulator.
+ //We push all supported SQL operator into it on the higher tree levels.
+ SingleTableSQLAccumulator(
+ igniteQueryContext = igniteQueryContext,
+ table = Some(igniteSqlRelation.tableName),
+ tableExpression = None,
+ outputExpressions = output.map(attr ⇒ attr.withQualifier(Some(igniteSqlRelation.tableName))))
+
+ case project: Project if !stepSkipped && exprsAllowed(project.projectList) ⇒
+ //Project layer just changes output of current query.
+ project.child match {
+ case acc: SelectAccumulator ⇒
+ acc.withOutputExpressions(
+ substituteExpressions(project.projectList, acc.outputExpressions))
+
+ case _ ⇒
+ throw new IgniteException("stepSkipped == true but child is not SelectAccumulator")
+ }
+
+ case sort: Sort if !stepSkipped && isSortPushDownAllowed(sort.order, sort.global) ⇒
+ sort.child match {
+ case acc: QueryAccumulator ⇒
+ acc.withOrderBy(sort.order)
+
+ case _ ⇒
+ throw new IgniteException("stepSkipped == true but child is not SelectAccumulator")
+ }
+
+ case filter: Filter if !stepSkipped && exprsAllowed(filter.condition) ⇒
+
+ filter.child match {
+ case acc: SelectAccumulator ⇒
+ if (hasAggregateInside(filter.condition) || acc.groupBy.isDefined)
+ acc.withHaving(acc.having.getOrElse(Nil) :+ filter.condition)
+ else
+ acc.withWhere(acc.where.getOrElse(Nil) :+ filter.condition)
+
+ case _ ⇒
+ throw new IgniteException("stepSkipped == true but child is not SelectAccumulator")
+ }
+
+ case agg: Aggregate
+ if !stepSkipped && exprsAllowed(agg.groupingExpressions) && exprsAllowed(agg.aggregateExpressions) ⇒
+
+ agg.child match {
+ case acc: SelectAccumulator ⇒
+ if (acc.groupBy.isDefined) {
+ val tableAlias = acc.igniteQueryContext.uniqueTableAlias
+
+ accumulator.SingleTableSQLAccumulator(
+ igniteQueryContext = acc.igniteQueryContext,
+ table = None,
+ tableExpression = Some((acc, tableAlias)),
+ outputExpressions = agg.aggregateExpressions)
+ }
+ else
+ acc
+ .withGroupBy(agg.groupingExpressions)
+ .withOutputExpressions(
+ substituteExpressions(agg.aggregateExpressions, acc.outputExpressions))
+
+ case acc: QueryAccumulator ⇒
+ val tableAlias = acc.igniteQueryContext.uniqueTableAlias
+
+ accumulator.SingleTableSQLAccumulator(
+ igniteQueryContext = acc.igniteQueryContext,
+ table = None,
+ tableExpression = Some((acc, tableAlias)),
+ outputExpressions = agg.aggregateExpressions)
+
+ case _ ⇒
+ throw new IgniteException("stepSkipped == true but child is not SelectAccumulator")
+ }
+
+ case limit: LocalLimit if !stepSkipped && exprsAllowed(limit.limitExpr) ⇒
+ limit.child match {
+ case acc: SelectAccumulator ⇒
+ acc.withLocalLimit(limit.limitExpr)
+
+ case _ ⇒
+ throw new IgniteException("stepSkipped == true but child is not SelectAccumulator")
+ }
+
+ case limit: GlobalLimit if !stepSkipped && exprsAllowed(limit.limitExpr) ⇒
+ limit.child.transformUp {
+ case acc: SelectAccumulator ⇒
+ acc.withLimit(limit.limitExpr)
+
+ case _ ⇒
+ throw new IgniteException("stepSkipped == true but child is not SelectAccumulator")
+ }
+
+ case union: Union if !stepSkipped && isAllChildrenOptimized(union.children) ⇒
+ val first = union.children.head.asInstanceOf[QueryAccumulator]
+
+ val subQueries = union.children.map(_.asInstanceOf[QueryAccumulator])
+
+ UnionSQLAccumulator(
+ first.igniteQueryContext,
+ subQueries,
+ subQueries.head.output)
+
+ case join: Join
+ if !stepSkipped && isAllChildrenOptimized(Seq(join.left, join.right)) &&
+ join.condition.forall(exprsAllowed) ⇒
+
+ val left = join.left.asInstanceOf[QueryAccumulator]
+
+ val (leftOutput, leftAlias) =
+ if (!isSimpleTableAcc(left)) {
+ val tableAlias = left.igniteQueryContext.uniqueTableAlias
+
+ (left.output, Some(tableAlias))
+ }
+ else
+ (left.output, None)
+
+ val right = join.right.asInstanceOf[QueryAccumulator]
+
+ val (rightOutput, rightAlias) =
+ if (!isSimpleTableAcc(right) ||
+ leftAlias.getOrElse(left.qualifier) == right.qualifier) {
+ val tableAlias = right.igniteQueryContext.uniqueTableAlias
+
+ (right.output, Some(tableAlias))
+ }
+ else
+ (right.output, None)
+
+ JoinSQLAccumulator(
+ left.igniteQueryContext,
+ left,
+ right,
+ join.joinType,
+ leftOutput ++ rightOutput,
+ join.condition,
+ leftAlias,
+ rightAlias)
+
+ case unknown ⇒
+ stepSkipped = true
+
+ unknown
+ }
+ }
+
+ /**
+ * Changes qualifiers for an ambiguous columns names.
+ *
+ * @param plan Query plan.
+ * @return Transformed plan.
+ */
+ private def fixAmbiguousOutput(plan: LogicalPlan): LogicalPlan = plan.transformDown {
+ case acc: SingleTableSQLAccumulator if acc.children.exists(_.isInstanceOf[JoinSQLAccumulator]) ⇒
+ val fixedChildOutput =
+ fixAmbiguousOutput(acc.tableExpression.get._1.outputExpressions, acc.igniteQueryContext)
+
+ val newOutput = substituteExpressions(acc.outputExpressions, fixedChildOutput, changeOnlyName = true)
+
+ acc.copy(
+ outputExpressions = newOutput,
+ where = acc.where.map(
+ substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+ groupBy = acc.groupBy.map(
+ substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+ having = acc.having.map(
+ substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+ limit = acc.limit.map(
+ substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+ localLimit = acc.localLimit.map(
+ substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+ orderBy = acc.orderBy.map(
+ substituteExpressions(_, fixedChildOutput, changeOnlyName = true)))
+
+ acc
+
+ case acc: JoinSQLAccumulator
+ if acc.left.isInstanceOf[JoinSQLAccumulator] || acc.right.isInstanceOf[JoinSQLAccumulator] ⇒
+ val leftFixed = acc.left match {
+ case leftJoin: JoinSQLAccumulator ⇒
+ val fixedChildOutput = fixAmbiguousOutput(acc.left.outputExpressions, acc.igniteQueryContext)
+
+ val newOutput =
+ substituteExpressions(acc.outputExpressions, fixedChildOutput, changeOnlyName = true)
+
+ acc.copy(
+ outputExpressions = newOutput,
+ left = leftJoin.copy(outputExpressions = fixedChildOutput),
+ condition = acc.condition.map(
+ substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+ where = acc.where.map(
+ substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+ groupBy = acc.groupBy.map(
+ substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+ having = acc.having.map(
+ substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+ limit = acc.limit.map(
+ substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+ localLimit = acc.localLimit.map(
+ substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+ orderBy = acc.orderBy.map(
+ substituteExpressions(_, fixedChildOutput, changeOnlyName = true)))
+
+ case _ ⇒ acc
+ }
+
+ val fixed = leftFixed.right match {
+ case rightJoin: JoinSQLAccumulator ⇒
+ val fixedChildOutput =
+ fixAmbiguousOutput(leftFixed.outputExpressions, leftFixed.igniteQueryContext)
+
+ val newOutput = substituteExpressions(leftFixed.outputExpressions, fixedChildOutput)
+
+ leftFixed.copy(
+ outputExpressions = newOutput,
+ right = rightJoin.copy(outputExpressions = fixedChildOutput),
+ condition = acc.condition.map(
+ substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+ where = acc.where.map(
+ substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+ groupBy = acc.groupBy.map(
+ substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+ having = acc.having.map(
+ substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+ limit = acc.limit.map(
+ substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+ localLimit = acc.localLimit.map(
+ substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+ orderBy = acc.orderBy.map(
+ substituteExpressions(_, fixedChildOutput, changeOnlyName = true)))
+
+ case _ ⇒ leftFixed
+ }
+
+ fixed.copy(
+ condition = acc.condition.map(
+ substituteExpression(_, acc.outputExpressions, changeOnlyName = true)),
+ where = acc.where.map(
+ substituteExpressions(_, acc.outputExpressions, changeOnlyName = true)),
+ groupBy = acc.groupBy.map(
+ substituteExpressions(_, acc.outputExpressions, changeOnlyName = true)),
+ having = acc.having.map(
+ substituteExpressions(_, acc.outputExpressions, changeOnlyName = true)),
+ limit = acc.limit.map(
+ substituteExpression(_, acc.outputExpressions, changeOnlyName = true)),
+ localLimit = acc.localLimit.map(
+ substituteExpression(_, acc.outputExpressions, changeOnlyName = true)),
+ orderBy = acc.orderBy.map(
+ substituteExpressions(_, acc.outputExpressions, changeOnlyName = true)))
+
+ case unknown ⇒
+ unknown
+ }
+
+ private def fixAmbiguousOutput(exprs: Seq[NamedExpression], ctx: IgniteQueryContext): Seq[NamedExpression] =
+ exprs.foldLeft((Set[String](), Set[NamedExpression]())) {
+ case ((uniqueNames, fixed), next) ⇒
+ if (uniqueNames(next.name))
+ (uniqueNames, fixed + Alias(next, ctx.uniqueColumnAlias(next))(exprId = next.exprId))
+ else
+ (uniqueNames + next.name, fixed + next)
+ }._2.toSeq
+
+ /**
+ * Substitutes each `QueryAccumulator` with a `LogicalRelation` contains `IgniteSQLAccumulatorRelation`.
+ *
+ * @param plan Query plan.
+ * @return Transformed plan.
+ */
+ private def makeIgniteAccRelation(plan: LogicalPlan): LogicalPlan =
+ plan.transformDown {
+ case acc: QueryAccumulator ⇒
+ new LogicalRelation (
+ relation = IgniteSQLAccumulatorRelation(acc),
+ output = acc.outputExpressions.map(toAttributeReference(_, Seq.empty)),
+ catalogTable = acc.igniteQueryContext.catalogTable)
+ }
+
+ /**
+ * @param order Order.
+ * @param global True if order applied to entire result set false if ordering per-partition.
+ * @return True if sort can be pushed down to Ignite, false otherwise.
+ */
+ private def isSortPushDownAllowed(order: Seq[SortOrder], global: Boolean): Boolean =
+ global && order.map(_.child).forall(exprsAllowed)
+
+ /**
+ * @param children Plans to check.
+ * @return True is all plan are `QueryAccumulator`, false otherwise.
+ */
+ private def isAllChildrenOptimized(children: Seq[LogicalPlan]): Boolean =
+ children.forall {
+ case _: QueryAccumulator ⇒
+ true
+
+ case _ ⇒
+ false
+ }
+
+ /**
+ * Changes expression from `exprs` collection to expression with same `exprId` from `substitution`.
+ *
+ * @param exprs Expressions to substitute.
+ * @param substitution Expressions for substitution
+ * @param changeOnlyName If true substitute only expression name.
+ * @tparam T Concrete expression type.
+ * @return Substituted expressions.
+ */
+ private def substituteExpressions[T <: Expression](exprs: Seq[T], substitution: Seq[NamedExpression],
+ changeOnlyName: Boolean = false): Seq[T] = {
+
+ exprs.map(substituteExpression(_, substitution, changeOnlyName))
+ }
+
+ private def substituteExpression[T <: Expression](expr: T, substitution: Seq[NamedExpression],
+ changeOnlyName: Boolean): T = expr match {
+ case ne: NamedExpression ⇒
+ substitution.find(_.exprId == ne.exprId) match {
+ case Some(found) ⇒
+ if (!changeOnlyName)
+ found.asInstanceOf[T]
+ else ne match {
+ case alias: Alias ⇒
+ Alias(
+ AttributeReference(
+ found.name,
+ found.dataType,
+ nullable = found.nullable,
+ metadata = found.metadata)(
+ exprId = found.exprId,
+ qualifier = found.qualifier,
+ isGenerated = found.isGenerated),
+ alias.name) (
+ exprId = alias.exprId,
+ qualifier = alias.qualifier,
+ explicitMetadata = alias.explicitMetadata,
+ isGenerated = alias.isGenerated).asInstanceOf[T]
+
+ case attr: AttributeReference ⇒
+ attr.copy(name = found.name)(
+ exprId = found.exprId,
+ qualifier = found.qualifier,
+ isGenerated = found.isGenerated).asInstanceOf[T]
+
+ case _ ⇒ ne.asInstanceOf[T]
+ }
+
+ case None ⇒
+ expr.withNewChildren(
+ substituteExpressions(expr.children, substitution, changeOnlyName)).asInstanceOf[T]
+ }
+
+ case _ ⇒
+ expr.withNewChildren(
+ substituteExpressions(expr.children, substitution, changeOnlyName)).asInstanceOf[T]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteSparkSession.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteSparkSession.scala b/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteSparkSession.scala
index 8860590..1fccc3a 100644
--- a/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteSparkSession.scala
+++ b/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteSparkSession.scala
@@ -66,8 +66,14 @@ class IgniteSparkSession private(ic: IgniteContext, proxy: SparkSession) extends
new IgniteSharedState(ic, sparkContext)
/** @inheritdoc */
- @transient override lazy val sessionState: SessionState =
- new SessionStateBuilder(self, None).build()
+ @transient override lazy val sessionState: SessionState = {
+ val sessionState = new SessionStateBuilder(self, None).build()
+
+ sessionState.experimentalMethods.extraOptimizations =
+ sessionState.experimentalMethods.extraOptimizations :+ IgniteOptimization
+
+ sessionState
+ }
/** @inheritdoc */
@transient override lazy val conf: RuntimeConfig = proxy.conf
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/test/scala/org/apache/ignite/spark/AbstractDataFrameSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/AbstractDataFrameSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/AbstractDataFrameSpec.scala
index 8613592..29a4e6f 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/AbstractDataFrameSpec.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/AbstractDataFrameSpec.scala
@@ -19,13 +19,16 @@ package org.apache.ignite.spark
import org.apache.ignite.{Ignite, Ignition}
import org.apache.ignite.configuration.{CacheConfiguration, IgniteConfiguration}
-import org.apache.spark.sql.SparkSession
-import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, FunSpec, Matchers}
+import org.apache.spark.sql.{DataFrame, Row, SparkSession}
+import org.scalatest._
import java.lang.{Long ⇒ JLong}
import org.apache.ignite.cache.query.SqlFieldsQuery
import org.apache.ignite.cache.query.annotations.QuerySqlField
-import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.internal.IgnitionEx.loadConfiguration
+import org.apache.ignite.spark.AbstractDataFrameSpec.configuration
+import org.apache.ignite.spark.impl.IgniteSQLAccumulatorRelation
+import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.ignite.spark.AbstractDataFrameSpec._
import scala.annotation.meta.field
@@ -33,12 +36,13 @@ import scala.reflect.ClassTag
/**
*/
-abstract class AbstractDataFrameSpec extends FunSpec with Matchers with BeforeAndAfterAll with BeforeAndAfter {
+abstract class AbstractDataFrameSpec extends FunSpec with Matchers with BeforeAndAfterAll with BeforeAndAfter
+ with Assertions {
var spark: SparkSession = _
var client: Ignite = _
- override protected def beforeAll() = {
+ override protected def beforeAll(): Unit = {
for (i ← 0 to 3)
Ignition.start(configuration("grid-" + i, client = false))
@@ -47,7 +51,7 @@ abstract class AbstractDataFrameSpec extends FunSpec with Matchers with BeforeAn
createSparkSession()
}
- override protected def afterAll() = {
+ override protected def afterAll(): Unit = {
Ignition.stop("client", false)
for (i ← 0 to 3)
@@ -108,6 +112,7 @@ abstract class AbstractDataFrameSpec extends FunSpec with Matchers with BeforeAn
cache.query(qry.setArgs(1L.asInstanceOf[JLong], "Forest Hill")).getAll
cache.query(qry.setArgs(2L.asInstanceOf[JLong], "Denver")).getAll
cache.query(qry.setArgs(3L.asInstanceOf[JLong], "St. Petersburg")).getAll
+ cache.query(qry.setArgs(4L.asInstanceOf[JLong], "St. Petersburg")).getAll
}
def createEmployeeCache(client: Ignite, cacheName: String): Unit = {
@@ -119,6 +124,31 @@ abstract class AbstractDataFrameSpec extends FunSpec with Matchers with BeforeAn
cache.put("key2", Employee(2, "Sarah Connor", 32, 10000))
cache.put("key3", Employee(3, "Arnold Schwarzenegger", 27, 1000))
}
+
+ def checkQueryData[T](res: DataFrame, expectedRes: Product)
+ (implicit ord: T ⇒ Ordered[T]): Unit =
+ checkQueryData(res, expectedRes, _.getAs[T](0))
+
+ def checkQueryData[Ordered](res: DataFrame, expectedRes: Product, sorter: Row => Ordered)
+ (implicit ord: Ordering[Ordered]): Unit = {
+ val data = res.rdd.collect.sortBy(sorter)
+
+ for(i ← 0 until expectedRes.productArity) {
+ val row = data(i)
+
+ if (row.size == 1)
+ assert(row(0) == expectedRes.productElement(i), s"row[$i, 0] = ${row(0)} should be equal ${expectedRes.productElement(i)}")
+ else {
+ val expectedRow: Product = expectedRes.productElement(i).asInstanceOf[Product]
+
+ assert(expectedRow.productArity == row.size, s"Rows size should be equal, but expected.size=${expectedRow.productArity} " +
+ s"and row.size=${row.size}")
+
+ for (j ← 0 until expectedRow.productArity)
+ assert(row(j) == expectedRow.productElement(j), s"row[$i, $j] = ${row(j)} should be equal ${expectedRow.productElement(j)}")
+ }
+ }
+ }
}
object AbstractDataFrameSpec {
@@ -135,7 +165,7 @@ object AbstractDataFrameSpec {
val PERSON_TBL_NAME_2 = "person2"
def configuration(igniteInstanceName: String, client: Boolean): IgniteConfiguration = {
- val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+ val cfg = loadConfiguration(TEST_CONFIG_FILE).get1()
cfg.setClientMode(client)
@@ -167,6 +197,30 @@ object AbstractDataFrameSpec {
}
/**
+ * @param df Data frame.
+ * @param qry SQL Query.
+ */
+ def checkOptimizationResult(df: DataFrame, qry: String = ""): Unit = {
+ df.explain(true)
+
+ val plan = df.queryExecution.optimizedPlan
+
+ val cnt = plan.collectLeaves.count {
+ case LogicalRelation(relation: IgniteSQLAccumulatorRelation[_, _], _, _) ⇒
+ if (qry != "")
+ assert(qry.toLowerCase == relation.acc.compileQuery().toLowerCase,
+ s"Generated query should be equal to expected.\nexpected - $qry\ngenerated - ${relation.acc.compileQuery()}")
+
+ true
+
+ case _ ⇒
+ false
+ }
+
+ assert(cnt != 0, s"Plan should contains IgniteSQLAccumulatorRelation")
+ }
+
+ /**
* Enclose some closure, so it doesn't on outer object(default scala behaviour) while serializing.
*/
def enclose[E, R](enclosed: E)(func: E => R): R = func(enclosed)
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteCatalogSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteCatalogSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteCatalogSpec.scala
index 6077211..d87d234 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteCatalogSpec.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteCatalogSpec.scala
@@ -58,7 +58,7 @@ class IgniteCatalogSpec extends AbstractDataFrameSpec {
it("Should provide ability to query SQL table without explicit registration") {
val res = igniteSession.sql("SELECT id, name FROM city").rdd
- res.count should equal(3)
+ res.count should equal(4)
val cities = res.collect.sortBy(_.getAs[JLong]("id"))
@@ -66,7 +66,8 @@ class IgniteCatalogSpec extends AbstractDataFrameSpec {
Array(
(1, "Forest Hill"),
(2, "Denver"),
- (3, "St. Petersburg")
+ (3, "St. Petersburg"),
+ (4, "St. Petersburg")
)
)
}
@@ -136,7 +137,7 @@ class IgniteCatalogSpec extends AbstractDataFrameSpec {
createEmployeeCache(client, EMPLOYEE_CACHE_NAME)
- val configProvider = enclose(null) (x ⇒ () ⇒ {
+ val configProvider = enclose(null) (_ ⇒ () ⇒ {
val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
cfg.setClientMode(true)
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSchemaSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSchemaSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSchemaSpec.scala
index cdd26cd..c5df901 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSchemaSpec.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSchemaSpec.scala
@@ -17,9 +17,6 @@
package org.apache.ignite.spark
-import java.lang.{Integer ⇒ JInteger, String ⇒ JString}
-
-import org.apache.ignite.Ignite
import org.apache.ignite.spark.AbstractDataFrameSpec._
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.types._
@@ -45,7 +42,7 @@ class IgniteDataFrameSchemaSpec extends AbstractDataFrameSpec {
("IS_RESIDENT", BooleanType, true),
("SALARY", DoubleType, true),
("PENSION", DoubleType, true),
- ("ACCOUNT", DecimalType(10, 0), true),
+ ("ACCOUNT", IgniteRDD.DECIMAL, true),
("AGE", IntegerType, true),
("ID", LongType, false),
("CITY_ID", LongType, false))
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSuite.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSuite.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSuite.scala
index 2ceb44a..b3f7026 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSuite.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSuite.scala
@@ -28,5 +28,12 @@ class IgniteDataFrameSuite extends Suites (
new IgniteSQLDataFrameWriteSpec,
new IgniteSQLDataFrameIgniteSessionWriteSpec,
new IgniteDataFrameWrongConfigSpec,
- new IgniteCatalogSpec
+ new IgniteCatalogSpec,
+ new IgniteOptimizationSpec,
+ new IgniteOptimizationStringFuncSpec,
+ new IgniteOptimizationMathFuncSpec,
+ new IgniteOptimizationAggregationFuncSpec,
+ new IgniteOptimizationSystemFuncSpec,
+ new IgniteOptimizationJoinSpec,
+ new IgniteOptimizationDisableEnableSpec
)
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationAggregationFuncSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationAggregationFuncSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationAggregationFuncSpec.scala
new file mode 100644
index 0000000..d2527c8
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationAggregationFuncSpec.scala
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.Ignite
+import org.apache.ignite.cache.query.SqlFieldsQuery
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import java.lang.{Double ⇒ JDouble, Long ⇒ JLong}
+
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+import org.apache.spark.sql.ignite.IgniteSparkSession
+
+/**
+ */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationAggregationFuncSpec extends AbstractDataFrameSpec {
+ var igniteSession: IgniteSparkSession = _
+
+ describe("Supported optimized aggregation functions") {
+ it("COUNT") {
+ val df = igniteSession.sql("SELECT count(*) FROM numbers")
+
+ checkOptimizationResult(df, "SELECT count(1) FROM numbers")
+
+ val data = Tuple1(21)
+
+ checkQueryData(df, data)
+ }
+
+ it("AVG - DECIMAL") {
+ //TODO: write me
+ }
+
+ it("AVG - DOUBLE") {
+ val df = igniteSession.sql("SELECT AVG(val) FROM numbers WHERE id <= 3")
+
+ checkOptimizationResult(df, "SELECT AVG(val) FROM numbers WHERE id IS NOT NULL and id <= 3")
+
+ val data = Tuple1(.5)
+
+ checkQueryData(df, data)
+ }
+
+ it("MIN - DOUBLE") {
+ val df = igniteSession.sql("SELECT MIN(val) FROM numbers")
+
+ checkOptimizationResult(df, "SELECT MIN(val) FROM numbers")
+
+ val data = Tuple1(-1.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("MAX - DOUBLE") {
+ val df = igniteSession.sql("SELECT MAX(val) FROM numbers")
+
+ checkOptimizationResult(df, "SELECT MAX(val) FROM numbers")
+
+ val data = Tuple1(180.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("SUM - DOUBLE") {
+ val df = igniteSession.sql("SELECT SUM(val) FROM numbers WHERE id <= 3")
+
+ checkOptimizationResult(df, "SELECT SUM(val) FROM numbers WHERE id IS NOT NULL and id <= 3")
+
+ val data = Tuple1(1.5)
+
+ checkQueryData(df, data)
+ }
+
+ it("SUM - DECIMAL - 1") {
+ val df = igniteSession.sql("SELECT SUM(decimal_val) FROM numbers WHERE id IN (18, 19, 20)")
+
+ checkOptimizationResult(df, "SELECT SUM(decimal_val) FROM numbers WHERE id IN (18, 19, 20)")
+
+ df.printSchema()
+
+ val data = Tuple1(new java.math.BigDecimal(10.5).setScale(3))
+
+ checkQueryData(df, data)
+ }
+
+ it("SUM - DECIMAL - 2") {
+ val df = igniteSession.sql("SELECT SUM(decimal_val) FROM numbers WHERE id IN (18, 19, 20, 21)")
+
+ checkOptimizationResult(df, "SELECT SUM(decimal_val) FROM numbers WHERE id IN (18, 19, 20, 21)")
+
+ val data = Tuple1(new java.math.BigDecimal(15).setScale(3))
+
+ checkQueryData(df, data)
+ }
+
+ it("SUM - LONG") {
+ val df = igniteSession.sql("SELECT SUM(int_val) FROM numbers WHERE id in (15, 16, 17)")
+
+ checkOptimizationResult(df, "SELECT CAST(SUM(int_val) AS BIGINT) as \"SUM(int_val)\" " +
+ "FROM numbers WHERE id in (15, 16, 17)")
+
+ val data = Tuple1(6L)
+
+ checkQueryData(df, data)
+ }
+ }
+
+ def createNumberTable(client: Ignite, cacheName: String): Unit = {
+ val cache = client.cache(cacheName)
+
+ cache.query(new SqlFieldsQuery(
+ """
+ | CREATE TABLE numbers (
+ | id LONG,
+ | val DOUBLE,
+ | int_val LONG,
+ | decimal_val DECIMAL(5, 5),
+ | PRIMARY KEY (id)) WITH "backups=1"
+ """.stripMargin)).getAll
+
+ var qry = new SqlFieldsQuery("INSERT INTO numbers (id, val) values (?, ?)")
+
+ cache.query(qry.setArgs(1L.asInstanceOf[JLong], .0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(2L.asInstanceOf[JLong], .5.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(3L.asInstanceOf[JLong], 1.0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(4L.asInstanceOf[JLong], 2.0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(5L.asInstanceOf[JLong], 4.0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(6L.asInstanceOf[JLong], -0.5.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(7L.asInstanceOf[JLong], -1.0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(8L.asInstanceOf[JLong], 42.0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(9L.asInstanceOf[JLong], .51.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(10L.asInstanceOf[JLong], .49.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(11L.asInstanceOf[JLong], 100.0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(12L.asInstanceOf[JLong], (Math.E*Math.E).asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(13L.asInstanceOf[JLong], Math.PI.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(14L.asInstanceOf[JLong], 180.0.asInstanceOf[JDouble])).getAll
+
+ qry = new SqlFieldsQuery("INSERT INTO numbers (id, int_val) values (?, ?)")
+
+ cache.query(qry.setArgs(15L.asInstanceOf[JLong], 1L.asInstanceOf[JLong])).getAll
+ cache.query(qry.setArgs(16L.asInstanceOf[JLong], 2L.asInstanceOf[JLong])).getAll
+ cache.query(qry.setArgs(17L.asInstanceOf[JLong], 3L.asInstanceOf[JLong])).getAll
+
+ qry = new SqlFieldsQuery("INSERT INTO numbers (id, decimal_val) values (?, ?)")
+
+ cache.query(qry.setArgs(18L.asInstanceOf[JLong], new java.math.BigDecimal(2.5))).getAll
+ cache.query(qry.setArgs(19L.asInstanceOf[JLong], new java.math.BigDecimal(3.5))).getAll
+ cache.query(qry.setArgs(20L.asInstanceOf[JLong], new java.math.BigDecimal(4.5))).getAll
+ cache.query(qry.setArgs(21L.asInstanceOf[JLong], new java.math.BigDecimal(4.5))).getAll
+ }
+
+ override protected def beforeAll(): Unit = {
+ super.beforeAll()
+
+ createNumberTable(client, DEFAULT_CACHE)
+
+ val configProvider = enclose(null) (x ⇒ () ⇒ {
+ val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+ cfg.setClientMode(true)
+
+ cfg.setIgniteInstanceName("client-2")
+
+ cfg
+ })
+
+ igniteSession = IgniteSparkSession.builder()
+ .config(spark.sparkContext.getConf)
+ .igniteConfigProvider(configProvider)
+ .getOrCreate()
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDateFuncSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDateFuncSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDateFuncSpec.scala
new file mode 100644
index 0000000..7912cd0
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDateFuncSpec.scala
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.Ignite
+import org.apache.ignite.cache.query.SqlFieldsQuery
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+import org.apache.spark.sql.ignite.IgniteSparkSession
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import java.lang.{Long ⇒ JLong}
+import java.util.{Date ⇒ JDate}
+import java.text.SimpleDateFormat
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.TimeUnit.DAYS
+
+/**
+ */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationDateFuncSpec extends AbstractDataFrameSpec {
+ var igniteSession: IgniteSparkSession = _
+
+ val format = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss")
+
+ describe("Supported optimized date functions") {
+ it(" - CURRENT_TIMESTAMP") {
+ val df = igniteSession.sql("SELECT id, CURRENT_TIMESTAMP() FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = df.rdd.collect
+
+ assert(data(0).getAs[JLong]("id") == 1L)
+
+ val date: JDate = data(0).getAs[JDate]("current_timestamp()")
+ val millisDiff = new JDate().getTime - date.getTime
+
+ assert(millisDiff <= 30000)
+ }
+
+ it(" - CURRENT_DATE") {
+ val df = igniteSession.sql("SELECT id, CURRENT_DATE() FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = df.rdd.collect
+
+ assert(data(0).getAs[JLong]("id") == 1L)
+
+ val date: JDate = data(0).getAs[JDate]("current_date()")
+ val dayDiff = DAYS.convert(new JDate().getTime - date.getTime, TimeUnit.MILLISECONDS)
+
+ assert(dayDiff <= 1)
+ }
+
+ it(" - HOUR") {
+ val df = igniteSession.sql("SELECT HOUR(val) FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = Tuple1(0)
+
+ checkQueryData(df, data)
+ }
+
+ it(" - MINUTE") {
+ val df = igniteSession.sql("SELECT MINUTE(val) FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = Tuple1(0)
+
+ checkQueryData(df, data)
+ }
+
+ it(" - SECOND") {
+ val df = igniteSession.sql("SELECT SECOND(val) FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = Tuple1(0)
+
+ checkQueryData(df, data)
+ }
+
+ it(" - MONTH") {
+ val df = igniteSession.sql("SELECT MONTH(val) FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = Tuple1(0)
+
+ checkQueryData(df, data)
+ }
+
+ it(" - YEAR") {
+ val df = igniteSession.sql("SELECT YEAR(val) FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = Tuple1(2017)
+
+ checkQueryData(df, data)
+ }
+
+ it(" - QUARTER") {
+ val df = igniteSession.sql("SELECT QUARTER(val) FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = Tuple1(1)
+
+ checkQueryData(df, data)
+ }
+
+ it(" - WEEK") {
+ val df = igniteSession.sql("SELECT WEEKOFYEAR(val) FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = Tuple1(1)
+
+ checkQueryData(df, data)
+ }
+
+ it(" - DAY_OF_MONTH") {
+ val df = igniteSession.sql("SELECT DAYOFMONTH(val) FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = Tuple1(1)
+
+ checkQueryData(df, data)
+ }
+
+ it(" - DAY_OF_YEAR") {
+ val df = igniteSession.sql("SELECT DAYOFYEAR(val) FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = Tuple1(1)
+
+ checkQueryData(df, data)
+ }
+
+ it(" - DATE_ADD") {
+ val df = igniteSession.sql("SELECT DATE_ADD(val, 2) FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = Tuple1(format.parse("03.01.2017 00:00:00"))
+
+ checkQueryData(df, data)
+ }
+
+ it(" - DATEDIFF") {
+ val df = igniteSession.sql("SELECT " +
+ "DATEDIFF(val, TO_DATE('2017-01-02 00:00:00.000', 'yyyy-MM-dd HH:mm:ss.SSS')) FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = Tuple1(1)
+
+ checkQueryData(df, data)
+ }
+
+ it(" - FORMATDATETIME") {
+ val df = igniteSession.sql("SELECT DATE_FORMAT(val, 'yyyy-MM-dd HH:mm:ss.SSS') FROM dates WHERE id = 1")
+
+ checkOptimizationResult(df)
+
+ val data = Tuple1("2017-01-01 00:00:00.000")
+
+ checkQueryData(df, data)
+ }
+ }
+
+ def createDateTable(client: Ignite, cacheName: String): Unit = {
+ val cache = client.cache(cacheName)
+
+ cache.query(new SqlFieldsQuery(
+ """
+ | CREATE TABLE dates (
+ | id LONG,
+ | val DATE,
+ | PRIMARY KEY (id)) WITH "backups=1"
+ """.stripMargin)).getAll
+
+ val qry = new SqlFieldsQuery("INSERT INTO dates(id, val) values (?, ?)")
+
+ cache.query(qry.setArgs(1L.asInstanceOf[JLong], format.parse("01.01.2017 00:00:00"))).getAll
+ }
+
+ override protected def beforeAll(): Unit = {
+ super.beforeAll()
+
+ createDateTable(client, DEFAULT_CACHE)
+
+ val configProvider = enclose(null) (x ⇒ () ⇒ {
+ val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+ cfg.setClientMode(true)
+
+ cfg.setIgniteInstanceName("client-2")
+
+ cfg
+ })
+
+ igniteSession = IgniteSparkSession.builder()
+ .config(spark.sparkContext.getConf)
+ .igniteConfigProvider(configProvider)
+ .getOrCreate()
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDisableEnableSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDisableEnableSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDisableEnableSpec.scala
new file mode 100644
index 0000000..033af74
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDisableEnableSpec.scala
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.spark.AbstractDataFrameSpec.TEST_CONFIG_FILE
+import org.apache.ignite.spark.IgniteDataFrameSettings._
+import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.ignite.IgniteOptimization
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+
+/**
+ */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationDisableEnableSpec extends AbstractDataFrameSpec {
+ var personDataFrame: DataFrame = _
+
+ describe("Ignite Optimization Disabling/Enabling") {
+ it("should add Ignite Optimization to a session on a first query") {
+ if (spark.sparkContext.isStopped)
+ createSparkSession()
+
+ assert(!igniteOptimizationExists(spark), "Session shouldn't contains IgniteOptimization")
+
+ personDataFrame = spark.read
+ .format(FORMAT_IGNITE)
+ .option(OPTION_CONFIG_FILE, TEST_CONFIG_FILE)
+ .option(OPTION_TABLE, "person")
+ .load()
+
+ assert(igniteOptimizationExists(spark),
+ "Session should contains IgniteOptimization after executing query over Ignite Data Frame")
+
+ spark.stop()
+ }
+
+ it("should remove Ignite Optimization if it disabled at runtime") {
+ if (!spark.sparkContext.isStopped)
+ spark.stop()
+
+ val newSession = SparkSession.builder()
+ .appName("Ignite Optimization check")
+ .master("local")
+ .config("spark.executor.instances", "2")
+ .getOrCreate()
+
+ assert(!igniteOptimizationExists(newSession), "Session shouldn't contains IgniteOptimization")
+
+ var newPersonDataFrame = newSession.read
+ .format(FORMAT_IGNITE)
+ .option(OPTION_CONFIG_FILE, TEST_CONFIG_FILE)
+ .option(OPTION_TABLE, "person")
+ .load()
+
+ assert(igniteOptimizationExists(newSession),
+ "Session should contains IgniteOptimization after executing query over Ignite Data Frame")
+
+
+ newSession.conf.set(OPTION_DISABLE_SPARK_SQL_OPTIMIZATION, "true")
+
+ newPersonDataFrame = newSession.read
+ .format(FORMAT_IGNITE)
+ .option(OPTION_CONFIG_FILE, TEST_CONFIG_FILE)
+ .option(OPTION_TABLE, "person")
+ .load()
+
+ assert(!igniteOptimizationExists(newSession),
+ "Session shouldn't contains IgniteOptimization")
+
+ newSession.close()
+ }
+
+ it("shouldn't add Ignite Optimization to a session when it's disabled") {
+ if (!spark.sparkContext.isStopped)
+ spark.stop()
+
+ val newSession = SparkSession.builder()
+ .appName("Ignite Optimization check")
+ .master("local")
+ .config("spark.executor.instances", "2")
+ .config(OPTION_DISABLE_SPARK_SQL_OPTIMIZATION, "true")
+ .getOrCreate()
+
+ assert(!igniteOptimizationExists(newSession), "Session shouldn't contains IgniteOptimization")
+
+ val newPersonDataFrame = newSession.read
+ .format(FORMAT_IGNITE)
+ .option(OPTION_CONFIG_FILE, TEST_CONFIG_FILE)
+ .option(OPTION_TABLE, "person")
+ .load()
+
+ newPersonDataFrame.createOrReplaceTempView("person")
+
+ val res = newSession.sqlContext.sql("SELECT name FROM person WHERE id = 2").rdd
+
+ res.count should equal(1)
+
+ assert(!igniteOptimizationExists(newSession), "Session shouldn't contains IgniteOptimization")
+
+ newSession.close()
+ }
+ }
+
+ def igniteOptimizationExists(session: SparkSession): Boolean =
+ session.sessionState.experimentalMethods.extraOptimizations.contains(IgniteOptimization)
+
+ override protected def beforeAll(): Unit = {
+ super.beforeAll()
+
+ createPersonTable(client, "cache1")
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationJoinSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationJoinSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationJoinSpec.scala
new file mode 100644
index 0000000..b4b36a8
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationJoinSpec.scala
@@ -0,0 +1,543 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.Ignite
+import org.apache.ignite.cache.query.SqlFieldsQuery
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+import org.apache.spark.sql.ignite.IgniteSparkSession
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+
+import java.lang.{Long ⇒ JLong}
+
+/**
+ */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationJoinSpec extends AbstractDataFrameSpec {
+ var igniteSession: IgniteSparkSession = _
+
+ describe("Optimized join queries") {
+ it("UNION") {
+ val qry =
+ """
+ | SELECT id, val1 as val FROM jt1 UNION
+ | SELECT id, val2 as val FROM jt2 UNION
+ | SELECT id, val3 as val FROM jt3
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df,
+ "SELECT id, val FROM (SELECT id, val1 as val FROM jt1 UNION " +
+ "SELECT id, val2 as val FROM jt2 UNION " +
+ "SELECT id, val3 as val FROM jt3) table1")
+
+ val data = (
+ (1L, "A"),
+ (1L, "B"),
+ (2L, "B"),
+ (2L, "C"),
+ (2L, "D"),
+ (3L, "C"),
+ (3L, "D"),
+ (3L, "E"))
+
+ checkQueryData(df, data, row ⇒ (row.getAs[JLong](0), row.getAs[String](1)))
+ }
+
+ it("UNION ALL") {
+ val qry =
+ """
+ | SELECT id, val1 as val FROM jt1 UNION ALL
+ | SELECT id, val2 as val FROM jt2 UNION ALL
+ | SELECT id, val3 as val FROM jt3
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df,
+ "SELECT id, val1 as val FROM jt1 UNION " +
+ "SELECT id, val2 as val FROM jt2 UNION " +
+ "SELECT id, val3 as val FROM jt3")
+
+ val data = (
+ (1L, "A"),
+ (1L, "B"),
+ (2L, "B"),
+ (2L, "C"),
+ (2L, "D"),
+ (3L, "C"),
+ (3L, "D"),
+ (3L, "E"))
+
+ checkQueryData(df, data, row ⇒ (row.getAs[JLong](0), row.getAs[String](1)))
+ }
+
+ it("UNION ALL ORDER") {
+ val qry =
+ """
+ | SELECT id, val1 as val FROM jt1 UNION ALL
+ | SELECT id, val2 as val FROM jt2 UNION ALL
+ | SELECT id, val3 as val FROM jt3
+ | ORDER BY id DESC, val
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df,
+ "SELECT id, val1 as val FROM jt1 UNION " +
+ "SELECT id, val2 as val FROM jt2 UNION " +
+ "SELECT id, val3 as val FROM jt3 " +
+ "ORDER BY id DESC, val")
+
+ val data = (
+ (3L, "C"),
+ (3L, "D"),
+ (3L, "E"),
+ (2L, "B"),
+ (2L, "C"),
+ (2L, "D"),
+ (1L, "A"),
+ (1L, "B")
+ )
+
+ checkQueryData(df, data, _ ⇒ 0)
+ }
+
+ it("UNION WITH AGGREGATE") {
+ val qry =
+ """
+ | SELECT VAL, COUNT(*) FROM (
+ | SELECT id, val1 as val FROM jt1 UNION
+ | SELECT id, val2 as val FROM jt2 UNION
+ | SELECT id, val3 as val FROM jt3 ) t1
+ | GROUP BY val HAVING COUNT(*) > 1
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df,
+ "SELECT VAL, count(1) FROM (" +
+ "SELECT id, val1 AS val FROM JT1 UNION " +
+ "SELECT id, val2 AS val FROM JT2 UNION " +
+ "SELECT id, val3 AS val FROM JT3" +
+ ") table1 GROUP BY val HAVING count(1) > 1")
+
+ val data = (
+ ("B", 2L),
+ ("C", 2L),
+ ("D", 2L)
+ )
+
+ checkQueryData(df, data)
+ }
+
+ it("AGGREGATE ON AGGREGATE RESULT") {
+ val qry =
+ """
+ | SELECT SUM(cnt) FROM (
+ | SELECT VAL, COUNT(*) as CNT FROM (
+ | SELECT id, val1 as val FROM jt1 UNION
+ | SELECT id, val2 as val FROM jt2 UNION
+ | SELECT id, val3 as val FROM jt3 ) t1
+ | GROUP BY val HAVING COUNT(*) > 1
+ | ) t1
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df,
+ "SELECT CAST(SUM(cnt) as BIGINT) as \"SUM(cnt)\" FROM (" +
+ "SELECT count(1) as cnt FROM (" +
+ "SELECT id, val1 as val FROM jt1 UNION " +
+ "SELECT id, val2 as val FROM jt2 UNION " +
+ "SELECT id, val3 as val FROM jt3" +
+ ") table1 GROUP BY val HAVING count(1) > 1) table2")
+
+ val data = Tuple1(6.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("SELF INNER JOIN") {
+ val qry =
+ """
+ |SELECT
+ | jt1.id,
+ | jt1.val1,
+ | jt2.id,
+ | jt2.val1
+ |FROM
+ | jt1 JOIN
+ | jt1 as jt2 ON jt1.val1 = jt2.val1
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df, "SELECT JT1.ID, JT1.VAL1, table1.ID, table1.VAL1 " +
+ "FROM JT1 JOIN JT1 AS table1 ON jt1.val1 = table1.val1 " +
+ "WHERE jt1.val1 IS NOT NULL AND table1.val1 IS NOT NULL")
+
+ val data = (
+ (1, "A", 1, "A"),
+ (2, "B", 2, "B"),
+ (3, "C", 3, "C")
+ )
+
+ checkQueryData(df, data)
+ }
+
+
+ it("SELF INNER JOIN WITH WHERE") {
+ val qry =
+ """
+ |SELECT
+ | jt1.id,
+ | jt1.val1,
+ | jt2.id,
+ | jt2.val1
+ |FROM
+ | jt1 JOIN
+ | jt1 as jt2 ON jt1.val1 = jt2.val1
+ |WHERE jt2.val1 = 'A'
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+/* checkOptimizationResult(df, "SELECT JT1.ID, JT1.VAL1, table1.ID, table1.VAL1 " +
+ "FROM JT1 JOIN JT1 as table1 ON JT1.val1 = table1.val1 " +
+ "WHERE JT1.val1 = 'A' AND JT1.val1 IS NOT NULL AND table1.val1 IS NOT NULL AND table1.val1 = 'A'")*/
+
+ val data = Tuple1(
+ (1, "A", 1, "A")
+ )
+
+ checkQueryData(df, data)
+ }
+
+
+ it("INNER JOIN") {
+ val qry =
+ """
+ |SELECT
+ | jt1.id as id1,
+ | jt1.val1,
+ | jt2.id as id2,
+ | jt2.val2
+ |FROM
+ | jt1 JOIN
+ | jt2 ON jt1.val1 = jt2.val2
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df, "SELECT JT1.ID AS id1, JT1.VAL1, JT2.ID AS id2, JT2.VAL2 " +
+ "FROM JT1 JOIN JT2 ON jt1.val1 = jt2.val2 " +
+ "WHERE jt1.val1 IS NOT NULL AND jt2.val2 IS NOT NULL")
+
+ val data = (
+ (2, "B", 1, "B"),
+ (3, "C", 2, "C")
+ )
+
+ checkQueryData(df, data)
+ }
+
+ it("INNER JOIN WITH WHERE") {
+ val qry =
+ """
+ |SELECT
+ | jt1.id as id1,
+ | jt1.val1,
+ | jt2.id as id2,
+ | jt2.val2
+ |FROM
+ | jt1 JOIN
+ | jt2 ON jt1.val1 = jt2.val2
+ |WHERE
+ | jt1.id < 10
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df, "SELECT jt1.id as id1, jt1.val1, jt2.id as id2, jt2.val2 " +
+ "FROM jt1 JOIN jt2 ON jt1.val1 = jt2.val2 " +
+ "WHERE jt1.id IS NOT NULL AND jt1.id < 10 AND jt1.val1 IS NOT NULL and jt2.val2 IS NOT NULL")
+
+ val data = (
+ (2, "B", 1, "B"),
+ (3, "C", 2, "C")
+ )
+
+ checkQueryData(df, data)
+ }
+
+ it("LEFT JOIN") {
+ val qry =
+ """
+ |SELECT
+ | jt1.id as id1,
+ | jt1.val1,
+ | jt2.id as id2,
+ | jt2.val2
+ |FROM
+ | jt1 LEFT JOIN
+ | jt2 ON jt1.val1 = jt2.val2
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df, "SELECT jt1.id as id1, jt1.val1, jt2.id as id2, jt2.val2 " +
+ "FROM jt1 LEFT JOIN jt2 ON jt1.val1 = jt2.val2")
+
+ val data = (
+ (1, "A", null, null),
+ (2, "B", 1, "B"),
+ (3, "C", 2, "C")
+ )
+
+ checkQueryData(df, data)
+ }
+
+ it("RIGHT JOIN") {
+ val qry =
+ """
+ |SELECT
+ | jt1.id as id1,
+ | jt1.val1,
+ | jt2.id as id2,
+ | jt2.val2
+ |FROM
+ | jt1 RIGHT JOIN
+ | jt2 ON jt1.val1 = jt2.val2
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df, "SELECT jt1.id as id1, jt1.val1, jt2.id as id2, jt2.val2 " +
+ "FROM jt1 RIGHT JOIN jt2 ON jt1.val1 = jt2.val2")
+
+ val data = (
+ (2, "B", 1, "B"),
+ (3, "C", 2, "C"),
+ (null, null, 3, "D")
+ )
+
+ checkQueryData(df, data, r ⇒ if (r.get(0) == null) 100L else r.getAs[Long](0))
+ }
+
+ it("JOIN 3 TABLE") {
+ val qry =
+ """
+ |SELECT
+ | jt1.id as id1,
+ | jt1.val1 as val1,
+ | jt2.id as id2,
+ | jt2.val2 as val2,
+ | jt3.id as id3,
+ | jt3.val3 as val3
+ |FROM
+ | jt1 LEFT JOIN
+ | jt2 ON jt1.val1 = jt2.val2 LEFT JOIN
+ | jt3 ON jt1.val1 = jt3.val3
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df,
+ "SELECT table1.id as id1, table1.val1, table1.id_2 as id2, table1.val2, jt3.id as id3, jt3.val3 " +
+ "FROM (" +
+ "SELECT jt1.val1, jt1.id, jt2.val2, jt2.id as id_2 " +
+ "FROM JT1 LEFT JOIN jt2 ON jt1.val1 = jt2.val2) table1 LEFT JOIN " +
+ "jt3 ON table1.val1 = jt3.val3")
+
+ val data = (
+ (1, "A", null, null, 1, "A"),
+ (2, "B", 1, "B", null, null),
+ (3, "C", 2, "C", null, null))
+
+ checkQueryData(df, data)
+ }
+
+ it("JOIN 3 TABLE AND AGGREGATE") {
+ val qry =
+ """
+ |SELECT SUM(id1) FROM (
+ | SELECT
+ | jt1.id as id1,
+ | jt1.val1 as val1,
+ | jt2.id as id2,
+ | jt2.val2 as val2,
+ | jt3.id as id3,
+ | jt3.val3 as val3
+ |FROM
+ | jt1 LEFT JOIN
+ | jt2 ON jt1.val1 = jt2.val2 LEFT JOIN
+ | jt3 ON jt1.val1 = jt3.val3
+ |) WHERE CONCAT(val1, val2) = 'BB' OR CONCAT(val1, val3) = 'AA'
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df,
+ "SELECT CAST(SUM(table1.ID) AS BIGINT) AS \"sum(id1)\" FROM " +
+ "(SELECT JT1.VAL1, JT1.ID, JT2.VAL2 FROM JT1 LEFT JOIN JT2 ON JT1.val1 = JT2.val2) table1 LEFT JOIN " +
+ "JT3 ON table1.val1 = JT3.val3 " +
+ "WHERE CONCAT(table1.val1, table1.val2) = 'BB' OR CONCAT(table1.val1, JT3.val3) = 'AA'")
+
+ val data = Tuple1(3)
+
+ checkQueryData(df, data, _ ⇒ 0)
+ }
+
+ it("INNER JOIN SUBQUERY") {
+ val qry =
+ """
+ |SELECT sum_id, val1, val2 FROM (
+ | SELECT
+ | jt1.id + jt2.id as sum_id,
+ | jt1.val1 as val1,
+ | jt2.val2 as val2
+ | FROM
+ | jt1 JOIN
+ | jt2 ON jt1.val1 = jt2.val2
+ |) t1 WHERE sum_id != 15
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df,
+ "SELECT jt1.id + jt2.id as sum_id, jt1.val1, jt2.val2 FROM " +
+ "jt1 JOIN jt2 ON NOT jt1.id + jt2.id = 15 AND jt1.val1 = jt2.val2 " +
+ "WHERE " +
+ "jt1.id IS NOT NULL AND " +
+ "jt1.val1 IS NOT NULL AND " +
+ "jt2.id IS NOT NULL AND " +
+ "jt2.val2 IS NOT NULL"
+ )
+
+ val data = (
+ (3, "B", "B"),
+ (5, "C", "C")
+ )
+
+ checkQueryData(df, data)
+ }
+
+ it("INNER JOIN SUBQUERY - 2") {
+ val qry =
+ """
+ |SELECT SUM(sum_id) FROM (
+ | SELECT
+ | jt1.id + jt2.id as sum_id
+ | FROM
+ | jt1 JOIN
+ | jt2 ON jt1.val1 = jt2.val2
+ |) t1 WHERE sum_id != 15
+ |""".stripMargin
+
+ val df = igniteSession.sql(qry)
+
+ checkOptimizationResult(df,
+ "SELECT CAST(SUM(JT1.ID + JT2.ID) AS BIGINT) AS \"sum(sum_id)\" " +
+ "FROM JT1 JOIN JT2 ON NOT JT1.id + JT2.id = 15 AND JT1.val1 = JT2.val2 " +
+ "WHERE JT1.id IS NOT NULL AND JT1.val1 IS NOT NULL AND JT2.id IS NOT NULL AND JT2.val2 IS NOT NULL")
+
+ val data = Tuple1(8)
+
+ checkQueryData(df, data)
+ }
+ }
+
+ def createJoinedTables(client: Ignite, cacheName: String): Unit = {
+ val cache = client.cache(cacheName)
+
+ cache.query(new SqlFieldsQuery(
+ """
+ | CREATE TABLE jt1 (
+ | id LONG,
+ | val1 VARCHAR,
+ | PRIMARY KEY (id)) WITH "backups=1"
+ """.stripMargin)).getAll
+
+ cache.query(new SqlFieldsQuery(
+ """
+ | CREATE TABLE jt2 (
+ | id LONG,
+ | val2 VARCHAR,
+ | PRIMARY KEY (id)) WITH "backups=1"
+ """.stripMargin)).getAll
+
+ cache.query(new SqlFieldsQuery(
+ """
+ | CREATE TABLE jt3 (
+ | id LONG,
+ | val3 VARCHAR,
+ | PRIMARY KEY (id)) WITH "backups=1"
+ """.stripMargin)).getAll
+
+ var qry = new SqlFieldsQuery("INSERT INTO jt1 (id, val1) values (?, ?)")
+
+ cache.query(qry.setArgs(1L.asInstanceOf[JLong], "A")).getAll
+ cache.query(qry.setArgs(2L.asInstanceOf[JLong], "B")).getAll
+ cache.query(qry.setArgs(3L.asInstanceOf[JLong], "C")).getAll
+
+ qry = new SqlFieldsQuery("INSERT INTO jt2 (id, val2) values (?, ?)")
+
+ cache.query(qry.setArgs(1L.asInstanceOf[JLong], "B")).getAll
+ cache.query(qry.setArgs(2L.asInstanceOf[JLong], "C")).getAll
+ cache.query(qry.setArgs(3L.asInstanceOf[JLong], "D")).getAll
+
+ qry = new SqlFieldsQuery("INSERT INTO jt3 (id, val3) values (?, ?)")
+
+ cache.query(qry.setArgs(1L.asInstanceOf[JLong], "A")).getAll
+ cache.query(qry.setArgs(2L.asInstanceOf[JLong], "D")).getAll
+ cache.query(qry.setArgs(3L.asInstanceOf[JLong], "E")).getAll
+
+ cache.query(new SqlFieldsQuery("CREATE INDEX idx1 ON jt1(val1)")).getAll
+ cache.query(new SqlFieldsQuery("CREATE INDEX idx2 ON jt2(val2)")).getAll
+ cache.query(new SqlFieldsQuery("CREATE INDEX idx3 ON jt3(val3)")).getAll
+ }
+
+ override protected def beforeAll(): Unit = {
+ super.beforeAll()
+
+ createPersonTable(client, DEFAULT_CACHE)
+
+ createCityTable(client, DEFAULT_CACHE)
+
+ createJoinedTables(client, DEFAULT_CACHE)
+
+ val configProvider = enclose(null) (x ⇒ () ⇒ {
+ val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+ cfg.setClientMode(true)
+
+ cfg.setIgniteInstanceName("client-2")
+
+ cfg
+ })
+
+ igniteSession = IgniteSparkSession.builder()
+ .config(spark.sparkContext.getConf)
+ .igniteConfigProvider(configProvider)
+ .getOrCreate()
+ }
+}
[30/50] [abbrv] ignite git commit: IGNITE-8298 Web Console: Fixed
loader under Safari.
Posted by ag...@apache.org.
IGNITE-8298 Web Console: Fixed loader under Safari.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0897309a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0897309a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0897309a
Branch: refs/heads/ignite-6083
Commit: 0897309a41de385d08ae6548a7e1812f3f409308
Parents: e7716f4
Author: Dmitriy Shabalin <dm...@gmail.com>
Authored: Thu Apr 19 15:16:18 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Apr 19 15:16:18 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/app/modules/loading/loading.scss | 2 +-
.../frontend/app/modules/loading/loading.service.js | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0897309a/modules/web-console/frontend/app/modules/loading/loading.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/loading/loading.scss b/modules/web-console/frontend/app/modules/loading/loading.scss
index 87bbc6a..0584c25 100644
--- a/modules/web-console/frontend/app/modules/loading/loading.scss
+++ b/modules/web-console/frontend/app/modules/loading/loading.scss
@@ -29,7 +29,7 @@
opacity: 0;
visibility: hidden;
background-color: white;
- transition: visibility 0s 0.5s, opacity 0.5s linear;
+ transition: opacity 0.5s linear;
}
.loading-active {
http://git-wip-us.apache.org/repos/asf/ignite/blob/0897309a/modules/web-console/frontend/app/modules/loading/loading.service.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/loading/loading.service.js b/modules/web-console/frontend/app/modules/loading/loading.service.js
index ec31c1a..bdc80b8 100644
--- a/modules/web-console/frontend/app/modules/loading/loading.service.js
+++ b/modules/web-console/frontend/app/modules/loading/loading.service.js
@@ -15,11 +15,11 @@
* limitations under the License.
*/
-export default ['IgniteLoading', ['$timeout', ($timeout) => {
+export default ['IgniteLoading', [() => {
const _overlays = {};
const start = (key) => {
- $timeout(() => {
+ setTimeout(() => {
const loadingOverlay = _overlays[key];
loadingOverlay && loadingOverlay.addClass('loading-active');
@@ -27,7 +27,7 @@ export default ['IgniteLoading', ['$timeout', ($timeout) => {
};
const finish = (key) => {
- $timeout(() => {
+ setTimeout(() => {
const loadingOverlay = _overlays[key];
loadingOverlay && loadingOverlay.removeClass('loading-active');
[08/50] [abbrv] ignite git commit: IGNITE-8282 Direct IO: support
fdatasync, which does not flush modified metadata - Fixes #3833.
Posted by ag...@apache.org.
IGNITE-8282 Direct IO: support fdatasync, which does not flush modified metadata - Fixes #3833.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/63dc75b8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/63dc75b8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/63dc75b8
Branch: refs/heads/ignite-6083
Commit: 63dc75b85b12f1c7bea796714c1e88925c16b1ab
Parents: e254e65
Author: dpavlov <dp...@apache.org>
Authored: Tue Apr 17 16:25:37 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Tue Apr 17 16:25:37 2018 +0300
----------------------------------------------------------------------
.../persistence/file/AlignedBuffersDirectFileIO.java | 6 +++++-
.../cache/persistence/file/IgniteNativeIoLib.java | 10 ++++++++++
2 files changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/63dc75b8/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
----------------------------------------------------------------------
diff --git a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
index 681426c..0168d2a 100644
--- a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
+++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
@@ -471,7 +471,11 @@ public class AlignedBuffersDirectFileIO implements FileIO {
/** {@inheritDoc} */
@Override public void force(boolean withMetadata) throws IOException {
- if (IgniteNativeIoLib.fsync(fdCheckOpened()) < 0)
+ int fd = fdCheckOpened();
+
+ int res = withMetadata ? IgniteNativeIoLib.fsync(fd) : IgniteNativeIoLib.fdatasync(fd);
+
+ if (res < 0)
throw new IOException(String.format("Error fsync()'ing %s, got %s", file, getLastError()));
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/63dc75b8/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoLib.java
----------------------------------------------------------------------
diff --git a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoLib.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoLib.java
index 47f1e6a..65ef8d7 100644
--- a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoLib.java
+++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoLib.java
@@ -316,6 +316,16 @@ public class IgniteNativeIoLib {
public static native int fsync(int fd);
/**
+ * Synchronize a file's in-core state with storage device. See "man 2 fsync".
+ *
+ * Similar to {@link #fsync(int)}, but does not flush modified metadata unless that metadata is needed in order to allow a subsequent data retrieval to be correctly handled
+ *
+ * @param fd file descriptor.
+ * @return On success return zero. On error, -1 is returned, and errno is set appropriately.
+ */
+ public static native int fdatasync(int fd);
+
+ /**
* Allocates size bytes and places the address of the allocated memory in {@code memptr}.
* The address of the allocated memory will be a multiple of {@code alignment}.
*
[27/50] [abbrv] ignite git commit: IGNITE-8243 Fixed possible memory
leak. Added latch manager to diagnostic messages. - Fixes #3850.
Posted by ag...@apache.org.
IGNITE-8243 Fixed possible memory leak. Added latch manager to diagnostic messages. - Fixes #3850.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6c6f0949
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6c6f0949
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6c6f0949
Branch: refs/heads/ignite-6083
Commit: 6c6f0949b7e79bc3d9ddd5416a990109405b1fb8
Parents: 5c71854
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Wed Apr 18 21:03:32 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Wed Apr 18 21:03:32 2018 +0300
----------------------------------------------------------------------
.../GridCachePartitionExchangeManager.java | 2 ++
.../preloader/latch/ExchangeLatchManager.java | 27 ++++++++++++++------
2 files changed, 21 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6c6f0949/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 20a3ccb..7ea161a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -1633,6 +1633,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
}
}
+ U.warn(diagnosticLog, "Latch manager state: " + latchMgr);
+
dumpPendingObjects(exchTopVer, diagCtx);
for (CacheGroupContext grp : cctx.cache().cacheGroups())
http://git-wip-us.apache.org/repos/asf/ignite/blob/6c6f0949/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
index 404f88f..b9c7dee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -64,21 +64,26 @@ public class ExchangeLatchManager {
private final GridKernalContext ctx;
/** Discovery manager. */
+ @GridToStringExclude
private final GridDiscoveryManager discovery;
/** IO manager. */
+ @GridToStringExclude
private final GridIoManager io;
/** Current coordinator. */
- private volatile ClusterNode coordinator;
+ @GridToStringExclude
+ private volatile ClusterNode crd;
/** Pending acks collection. */
private final ConcurrentMap<T2<String, AffinityTopologyVersion>, Set<UUID>> pendingAcks = new ConcurrentHashMap<>();
/** Server latches collection. */
+ @GridToStringInclude
private final ConcurrentMap<T2<String, AffinityTopologyVersion>, ServerLatch> serverLatches = new ConcurrentHashMap<>();
/** Client latches collection. */
+ @GridToStringInclude
private final ConcurrentMap<T2<String, AffinityTopologyVersion>, ClientLatch> clientLatches = new ConcurrentHashMap<>();
/** Lock. */
@@ -97,15 +102,14 @@ public class ExchangeLatchManager {
if (!ctx.clientNode()) {
ctx.io().addMessageListener(GridTopic.TOPIC_EXCHANGE, (nodeId, msg, plc) -> {
- if (msg instanceof LatchAckMessage) {
+ if (msg instanceof LatchAckMessage)
processAck(nodeId, (LatchAckMessage) msg);
- }
});
// First coordinator initialization.
ctx.discovery().localJoinFuture().listen(f -> {
if (f.error() == null)
- this.coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
+ this.crd = getLatchCoordinator(AffinityTopologyVersion.NONE);
});
ctx.event().addDiscoveryEventListener((e, cache) -> {
@@ -288,6 +292,8 @@ public class ExchangeLatchManager {
pendingAcks.computeIfAbsent(latchId, (id) -> new GridConcurrentHashSet<>());
pendingAcks.get(latchId).add(from);
}
+ else if (coordinator.isLocal())
+ serverLatches.remove(latchId);
} else {
if (log.isDebugEnabled())
log.debug("Process ack [latch=" + latchId + ", from=" + from + "]");
@@ -319,7 +325,7 @@ public class ExchangeLatchManager {
*/
private void becomeNewCoordinator() {
if (log.isInfoEnabled())
- log.info("Become new coordinator " + coordinator.id());
+ log.info("Become new coordinator " + crd.id());
List<T2<String, AffinityTopologyVersion>> latchesToRestore = new ArrayList<>();
latchesToRestore.addAll(pendingAcks.keySet());
@@ -347,7 +353,7 @@ public class ExchangeLatchManager {
* @param left Left node.
*/
private void processNodeLeft(ClusterNode left) {
- assert this.coordinator != null : "Coordinator is not initialized";
+ assert this.crd != null : "Coordinator is not initialized";
lock.lock();
@@ -402,8 +408,8 @@ public class ExchangeLatchManager {
}
// Coordinator is changed.
- if (coordinator.isLocal() && this.coordinator.id() != coordinator.id()) {
- this.coordinator = coordinator;
+ if (coordinator.isLocal() && this.crd.id() != coordinator.id()) {
+ this.crd = coordinator;
becomeNewCoordinator();
}
@@ -693,4 +699,9 @@ public class ExchangeLatchManager {
return S.toString(CompletableLatch.class, this);
}
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ExchangeLatchManager.class, this);
+ }
}
[20/50] [abbrv] ignite git commit: IGNITE-8298 Web Console: Fixed
tables UI issues.
Posted by ag...@apache.org.
IGNITE-8298 Web Console: Fixed tables UI issues.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a050436d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a050436d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a050436d
Branch: refs/heads/ignite-6083
Commit: a050436d5b5c814ef386766811e54d9f711afa8d
Parents: 17ed189
Author: Dmitriy Shabalin <dm...@gmail.com>
Authored: Wed Apr 18 18:43:13 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Apr 18 18:43:13 2018 +0700
----------------------------------------------------------------------
.../app/primitives/ui-grid-header/index.scss | 33 +++++++-
.../frontend/app/primitives/ui-grid/index.scss | 11 ++-
modules/web-console/frontend/package-lock.json | 80 ++++++++++----------
3 files changed, 80 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a050436d/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss b/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss
index ac58707..1ff27b2 100644
--- a/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss
+++ b/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss
@@ -33,6 +33,9 @@
}
.ui-grid-header-cell {
+ // Workaround: Fixed cell header offset in IE11.
+ vertical-align: top;
+
.ui-grid-cell-contents > span:not(.ui-grid-header-cell-label) {
right: 3px;
}
@@ -44,7 +47,7 @@
.ui-grid-header-cell [role="columnheader"] {
display: flex;
-
+
flex-wrap: wrap;
align-items: center;
justify-content: center;
@@ -77,15 +80,29 @@
.ui-grid-column-resizer.right {
top: -100px;
}
+
.ng-hide + .ui-grid-header-cell-row .ui-grid-column-resizer.right {
bottom: 0;
}
+ &.ui-grid-header-cell:not(:first-child) {
+ left: 0;
+ box-shadow: -1px 0px 0 0 #d4d4d4;
+ }
+
&.ui-grid-header-cell .ui-grid-header-cell .ui-grid-column-resizer.right {
border-right-width: 0;
}
+
&.ui-grid-header-cell .ui-grid-header-cell:last-child .ui-grid-column-resizer.right {
- border-right-width: 1px;
+ // Hide all right borders, and fix cell offset.
+ right: -1px;
+ border: none;
+ }
+
+ &.ui-grid-header-cell [ng-show] .ui-grid-cell-contents {
+ text-indent: -20px;
+ margin-right: -20px;
}
& > div > .ui-grid-cell-contents {
@@ -97,3 +114,15 @@
line-height: 21px;
}
}
+
+.ui-grid[ui-grid-selection][ui-grid-grouping] {
+ .ui-grid-pinned-container-left {
+ .ui-grid-header--subcategories {
+ .ui-grid-header-span {
+ &.ui-grid-header-cell {
+ box-shadow: none;
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a050436d/modules/web-console/frontend/app/primitives/ui-grid/index.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/primitives/ui-grid/index.scss b/modules/web-console/frontend/app/primitives/ui-grid/index.scss
index b19d5f7..2a5c587 100644
--- a/modules/web-console/frontend/app/primitives/ui-grid/index.scss
+++ b/modules/web-console/frontend/app/primitives/ui-grid/index.scss
@@ -148,12 +148,12 @@
.ui-grid-header--subcategories {
.ui-grid-header-span.ui-grid-header-cell {
+ background: initial;
+
.ui-grid-cell-contents {
padding: 8px 20px;
}
- background: initial;
-
[ng-show] .ui-grid-cell-contents {
text-align: center;
}
@@ -464,6 +464,7 @@
color: #393939;
text-align: left;
font-size: 14px;
+ font-weight: normal;
line-height: 1.42857;
box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
@@ -538,3 +539,9 @@
font-style: italic;
line-height: 16px;
}
+
+.ui-grid {
+ input[type="text"].ui-grid-filter-input {
+ font-weight: normal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a050436d/modules/web-console/frontend/package-lock.json
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/package-lock.json b/modules/web-console/frontend/package-lock.json
index e28ef4d..1fec909 100644
--- a/modules/web-console/frontend/package-lock.json
+++ b/modules/web-console/frontend/package-lock.json
@@ -401,9 +401,9 @@
"integrity": "sha512-/2xvG6vDC+Us8h0baSa1siDKwPj5R2A7LldxxhK2339HInc09bq9shMVCUy9zqnuvwnDUJ/DSgkSaBoSHSZrqg=="
},
"angular-mocks": {
- "version": "1.6.9",
- "resolved": "https://registry.npmjs.org/angular-mocks/-/angular-mocks-1.6.9.tgz",
- "integrity": "sha512-5aEwXmfd5DQvb64pOgP2W2D3ozAQSARkB6q+6NQfUJvJs9bD2YcExrUc1P4EbiIuyWag2OQM+pIKUNojVi3SBg==",
+ "version": "1.6.10",
+ "resolved": "https://registry.npmjs.org/angular-mocks/-/angular-mocks-1.6.10.tgz",
+ "integrity": "sha512-1865/NmqHNogibNoglY1MGBjx882iu2hI46BBhYDWyz0C4TDM5ER8H8SnYwQKUUG4RXMDsJizszEQ2BEoYKV9w==",
"dev": true
},
"angular-motion": {
@@ -1859,9 +1859,9 @@
"integrity": "sha1-c5JncZI7Whl0etZmqlzUv5xunOg="
},
"base64-js": {
- "version": "1.2.3",
- "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.2.3.tgz",
- "integrity": "sha512-MsAhsUW1GxCdgYSO6tAfZrNapmUKk7mWx/k5mFY/A1gBtkaCaNapTg+FExCw1r9yeaZhqx/xPg43xgTFH6KL5w=="
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz",
+ "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw=="
},
"base64id": {
"version": "1.0.0",
@@ -2120,7 +2120,7 @@
"integrity": "sha512-YkIRgwsZwJWTnyQrsBTWefizHh+8GYj3kbL1BTiAQ/9pwpino0G7B2gp5tx/FUBqUlvtxV85KNR3mwfAtv15Yw==",
"dev": true,
"requires": {
- "base64-js": "1.2.3",
+ "base64-js": "1.3.0",
"ieee754": "1.1.11"
}
},
@@ -2278,7 +2278,7 @@
"resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
"integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
"requires": {
- "base64-js": "1.2.3",
+ "base64-js": "1.3.0",
"ieee754": "1.1.11",
"isarray": "1.0.0"
}
@@ -3006,9 +3006,9 @@
}
},
"commander": {
- "version": "2.15.1",
- "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz",
- "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag=="
+ "version": "2.13.0",
+ "resolved": "https://registry.npmjs.org/commander/-/commander-2.13.0.tgz",
+ "integrity": "sha512-MVuS359B+YzaWqjCL/c+22gfryv+mCBPHAv3zyVI2GN8EY6IRP8VwtasXn8jyyhvvq84R4ImN1OKRtcbIasjYA=="
},
"commondir": {
"version": "1.0.1",
@@ -3192,7 +3192,7 @@
"loader-utils": "1.1.0",
"minimatch": "3.0.4",
"p-limit": "1.2.0",
- "serialize-javascript": "1.4.0"
+ "serialize-javascript": "1.5.0"
},
"dependencies": {
"globby": {
@@ -3517,9 +3517,9 @@
"integrity": "sha1-vEZ0gAQ3iyGjYMn8fPUjF5B2L7g="
},
"d3-color": {
- "version": "1.0.3",
- "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.0.3.tgz",
- "integrity": "sha1-vHZD/KjlOoNH4vva/6I2eWtYUJs="
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.1.0.tgz",
+ "integrity": "sha512-IZVcqX5yYFvR2NUBbSfIfbgNcSgAtZ7JbgQWqDXf4CywtN7agvI7Kw6+Q1ETvlHOHWJT55Kyuzt0C3I0GVtRHQ=="
},
"d3-hierarchy": {
"version": "1.1.6",
@@ -3531,7 +3531,7 @@
"resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-1.1.6.tgz",
"integrity": "sha512-mOnv5a+pZzkNIHtw/V6I+w9Lqm9L5bG3OTXPM5A+QO0yyVMQ4W1uZhR+VOJmazaOZXri2ppbiZ5BUNWT0pFM9A==",
"requires": {
- "d3-color": "1.0.3"
+ "d3-color": "1.1.0"
}
},
"dargs": {
@@ -5568,7 +5568,7 @@
"integrity": "sha1-zcvAgYgmWtEZtqWnyKtw7s+10n0=",
"requires": {
"chalk": "1.1.3",
- "commander": "2.15.1",
+ "commander": "2.13.0",
"is-my-json-valid": "2.17.2",
"pinkie-promise": "2.0.1"
},
@@ -5825,7 +5825,7 @@
"integrity": "sha512-KcuaIRWTU0kFjOJCs32a3JsGNCWkeOak0/F/uvJNp3x/N4McXdqHpcK64cYTozK7QLPKKtUqb9h7wR9K9rYRkg==",
"requires": {
"@posthtml/esm": "1.0.0",
- "htmlnano": "0.1.7",
+ "htmlnano": "0.1.8",
"loader-utils": "1.1.0",
"posthtml": "0.11.3",
"schema-utils": "0.4.5"
@@ -5845,6 +5845,11 @@
"uglify-js": "3.3.21"
},
"dependencies": {
+ "commander": {
+ "version": "2.15.1",
+ "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz",
+ "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag=="
+ },
"source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
@@ -5895,16 +5900,16 @@
"dev": true
},
"htmlnano": {
- "version": "0.1.7",
- "resolved": "https://registry.npmjs.org/htmlnano/-/htmlnano-0.1.7.tgz",
- "integrity": "sha512-t8Gy+r/loFP2VXAJl6ClaNIomGI609oyQcT7O3IoJE6VcDCLR6PYWXaSh+hfd/dnoZ6KPbpgPek/Crm3havqig==",
+ "version": "0.1.8",
+ "resolved": "https://registry.npmjs.org/htmlnano/-/htmlnano-0.1.8.tgz",
+ "integrity": "sha512-wwmDRJn5OQ9BqFYy5vWaufUQTKj7Ct6xTv+od7QNNJzJM7K3yqR4lJ8SHSOTcBahlXMO5EzddUdsS+fmdGvXpw==",
"requires": {
"cssnano": "3.10.0",
"object-assign": "4.1.1",
"posthtml": "0.11.3",
"posthtml-render": "1.1.3",
"svgo": "1.0.5",
- "uglify-js": "3.3.21"
+ "uglify-es": "3.3.9"
},
"dependencies": {
"coa": {
@@ -5969,12 +5974,12 @@
"util.promisify": "1.0.0"
}
},
- "uglify-js": {
- "version": "3.3.21",
- "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.21.tgz",
- "integrity": "sha512-uy82472lH8tshK3jS3c5IFb5MmNKd/5qyBd0ih8sM42L3jWvxnE339U9gZU1zufnLVs98Stib9twq8dLm2XYCA==",
+ "uglify-es": {
+ "version": "3.3.9",
+ "resolved": "https://registry.npmjs.org/uglify-es/-/uglify-es-3.3.9.tgz",
+ "integrity": "sha512-r+MU0rfv4L/0eeW3xZrd16t4NZfK8Ld4SWVglYBb7ez5uXFWHuVRs6xCTrf1yirs9a4j4Y27nn7SRfO6v67XsQ==",
"requires": {
- "commander": "2.15.1",
+ "commander": "2.13.0",
"source-map": "0.6.1"
},
"dependencies": {
@@ -8538,7 +8543,7 @@
"stream-browserify": "2.0.1",
"stream-http": "2.8.1",
"string_decoder": "1.1.1",
- "timers-browserify": "2.0.7",
+ "timers-browserify": "2.0.9",
"tty-browserify": "0.0.0",
"url": "0.11.0",
"util": "0.10.3",
@@ -11273,9 +11278,9 @@
}
},
"serialize-javascript": {
- "version": "1.4.0",
- "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-1.4.0.tgz",
- "integrity": "sha1-fJWFFNtqwkQ6irwGLcn3iGp/YAU="
+ "version": "1.5.0",
+ "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-1.5.0.tgz",
+ "integrity": "sha512-Ga8c8NjAAp46Br4+0oZ2WxJCwIzwP60Gq1YPgU+39PiTVxyed/iKE/zyZI6+UlVYH5Q4PaQdHhcegIFPZTUfoQ=="
},
"serve-index": {
"version": "1.9.1",
@@ -12528,9 +12533,9 @@
"integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8="
},
"timers-browserify": {
- "version": "2.0.7",
- "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.7.tgz",
- "integrity": "sha512-U7DtjfsHeYjNAyEz4MdCLGZMY3ySyHIgZZp6ba9uxZlMRMiK5yTHUYc2XfGQHKFgxGcmvBF2jafoNtQYvlDpOw==",
+ "version": "2.0.9",
+ "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.9.tgz",
+ "integrity": "sha512-2DhyvVpCWwY7gk8UmKhYvgHQl9XTlO0Dg0/2UZcLgPnpulhdm2aGIlFy5rU5igmOCA51w6jPHqLRA4UH1YmhcA==",
"requires": {
"setimmediate": "1.0.5"
}
@@ -12790,18 +12795,13 @@
"cacache": "10.0.4",
"find-cache-dir": "1.0.0",
"schema-utils": "0.4.5",
- "serialize-javascript": "1.4.0",
+ "serialize-javascript": "1.5.0",
"source-map": "0.6.1",
"uglify-es": "3.3.9",
"webpack-sources": "1.1.0",
"worker-farm": "1.6.0"
},
"dependencies": {
- "commander": {
- "version": "2.13.0",
- "resolved": "https://registry.npmjs.org/commander/-/commander-2.13.0.tgz",
- "integrity": "sha512-MVuS359B+YzaWqjCL/c+22gfryv+mCBPHAv3zyVI2GN8EY6IRP8VwtasXn8jyyhvvq84R4ImN1OKRtcbIasjYA=="
- },
"source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
[47/50] [abbrv] ignite git commit: IGNITE-8303 Avoided failure
handler invocation when exchange-worker terminates before reconnect.
Posted by ag...@apache.org.
IGNITE-8303 Avoided failure handler invocation when exchange-worker terminates before reconnect.
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3f3239b9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3f3239b9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3f3239b9
Branch: refs/heads/ignite-6083
Commit: 3f3239b907ef0f038d1673f8fdf4f3377d297aaf
Parents: fa84124
Author: Andrey Kuznetsov <st...@gmail.com>
Authored: Fri Apr 20 19:51:59 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Fri Apr 20 19:52:31 2018 +0300
----------------------------------------------------------------------
.../cache/GridCachePartitionExchangeManager.java | 7 ++++++-
.../apache/ignite/internal/IgniteClientRejoinTest.java | 13 ++++++++++++-
2 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3f3239b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 7ea161a..35f14da 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -2092,6 +2092,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
/** */
private boolean stop;
+ /** Indicates that worker terminates because the node needs to reconnect to the cluster. */
+ private boolean reconnectNeeded;
+
/**
* Constructor.
*/
@@ -2303,7 +2306,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
err = e;
}
finally {
- if (err == null && !stop)
+ if (err == null && !stop && !reconnectNeeded)
err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
if (err instanceof OutOfMemoryError)
@@ -2602,6 +2605,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
cctx.discovery().reconnect();
+ reconnectNeeded = true;
+
return;
}
catch (IgniteCheckedException e) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/3f3239b9/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
index 8edbb52..462c8ab 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
@@ -32,6 +32,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteClientDisconnectedException;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.Ignition;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -209,7 +210,17 @@ public class IgniteClientRejoinTest extends GridCommonAbstractTest {
@Override public Ignite call() throws Exception {
latch.await();
- return startGrid("client" + idx);
+ String nodeName = "client" + idx;
+
+ IgniteConfiguration cfg = getConfiguration(nodeName)
+ .setFailureHandler((ignite, failureCtx) -> {
+ // This should _not_ fire when exchange-worker terminates before reconnect.
+ Runtime.getRuntime().halt(Ignition.KILL_EXIT_CODE);
+
+ return false;
+ });
+
+ return startGrid(nodeName, optimize(cfg), null);
}
});
[11/50] [abbrv] ignite git commit: IGNITE-8033 Fixed flaky failure of
TxOptimisticDeadlockDetectionCrossCacheTest
Posted by ag...@apache.org.
IGNITE-8033 Fixed flaky failure of TxOptimisticDeadlockDetectionCrossCacheTest
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7b39f135
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7b39f135
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7b39f135
Branch: refs/heads/ignite-6083
Commit: 7b39f1355cf7b0d4169622cec2936184168aba99
Parents: 2edcb22
Author: Aleksey Plekhanov <Pl...@gmail.com>
Authored: Tue Apr 17 18:27:53 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Apr 17 18:50:15 2018 +0300
----------------------------------------------------------------------
...timisticDeadlockDetectionCrossCacheTest.java | 147 +++++++------------
1 file changed, 50 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7b39f135/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionCrossCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionCrossCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionCrossCacheTest.java
index 5d1374c..056b093 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionCrossCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionCrossCacheTest.java
@@ -18,30 +18,21 @@
package org.apache.ignite.internal.processors.cache.transactions;
import java.util.Collection;
-import java.util.Set;
-import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.internal.managers.communication.GridIoMessage;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareResponse;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.internal.util.GridConcurrentHashSet;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteInClosure;
-import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.spi.IgniteSpiException;
-import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -57,9 +48,6 @@ import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_REA
*
*/
public class TxOptimisticDeadlockDetectionCrossCacheTest extends GridCommonAbstractTest {
- /** Nodes count. */
- private static final int NODES_CNT = 2;
-
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -73,10 +61,6 @@ public class TxOptimisticDeadlockDetectionCrossCacheTest extends GridCommonAbstr
cfg.setDiscoverySpi(discoSpi);
}
- TcpCommunicationSpi commSpi = new TestCommunicationSpi();
-
- cfg.setCommunicationSpi(commSpi);
-
CacheConfiguration ccfg0 = defaultCacheConfiguration();
ccfg0.setName("cache0");
@@ -96,42 +80,46 @@ public class TxOptimisticDeadlockDetectionCrossCacheTest extends GridCommonAbstr
return cfg;
}
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
- startGrids(NODES_CNT);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- super.afterTestsStopped();
-
- stopAllGrids();
- }
-
/**
* @throws Exception If failed.
*/
public void testDeadlock() throws Exception {
- // Sometimes boh transactions perform commit, so we repeat attempt.
- while (!doTestDeadlock()) {}
+ startGrids(2);
+
+ try {
+ doTestDeadlock();
+ }
+ finally {
+ stopAllGrids();
+ }
}
/**
* @throws Exception If failed.
*/
private boolean doTestDeadlock() throws Exception {
- TestCommunicationSpi.init(2);
-
- final CyclicBarrier barrier = new CyclicBarrier(2);
-
final AtomicInteger threadCnt = new AtomicInteger();
final AtomicBoolean deadlock = new AtomicBoolean();
final AtomicInteger commitCnt = new AtomicInteger();
+ grid(0).events().localListen(new CacheLocksListener(), EventType.EVT_CACHE_OBJECT_LOCKED);
+
+ AffinityTopologyVersion waitTopVer = new AffinityTopologyVersion(2, 1);
+
+ IgniteInternalFuture<?> exchFut = grid(0).context().cache().context().exchange().affinityReadyFuture(waitTopVer);
+
+ if (exchFut != null && !exchFut.isDone()) {
+ log.info("Waiting for topology exchange future [waitTopVer=" + waitTopVer + ", curTopVer="
+ + grid(0).context().cache().context().exchange().readyAffinityVersion() + ']');
+
+ exchFut.get();
+ }
+
+ log.info("Finished topology exchange future [curTopVer="
+ + grid(0).context().cache().context().exchange().readyAffinityVersion() + ']');
+
IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
@Override public void run() {
int threadNum = threadCnt.getAndIncrement();
@@ -152,8 +140,6 @@ public class TxOptimisticDeadlockDetectionCrossCacheTest extends GridCommonAbstr
cache1.put(key1, 0);
- barrier.await();
-
int key2 = primaryKey(cache2);
log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
@@ -171,23 +157,23 @@ public class TxOptimisticDeadlockDetectionCrossCacheTest extends GridCommonAbstr
hasCause(e, TransactionDeadlockException.class)
) {
if (deadlock.compareAndSet(false, true))
- U.error(log, "At least one stack trace should contain " +
- TransactionDeadlockException.class.getSimpleName(), e);
+ log.info("Successfully set deadlock flag");
+ else
+ log.info("Deadlock flag was already set");
}
+ else
+ log.warning("Got not deadlock exception", e);
}
}
}, 2, "tx-thread");
fut.get();
- if (commitCnt.get() == 2)
- return false;
+ assertFalse("Commits must fail", commitCnt.get() == 2);
assertTrue(deadlock.get());
- for (int i = 0; i < NODES_CNT ; i++) {
- Ignite ignite = ignite(i);
-
+ for (Ignite ignite : G.allGrids()) {
IgniteTxManager txMgr = ((IgniteKernal)ignite).context().cache().context().tm();
Collection<IgniteInternalFuture<?>> futs = txMgr.deadlockDetectionFutures();
@@ -199,59 +185,26 @@ public class TxOptimisticDeadlockDetectionCrossCacheTest extends GridCommonAbstr
}
/**
+ * Listener for cache lock events.
*
+ * To ensure deadlock this listener blocks transaction thread until both threads acquire first lock.
*/
- private static class TestCommunicationSpi extends TcpCommunicationSpi {
- /** Tx count. */
- private static volatile int TX_CNT;
-
- /** Tx ids. */
- private static final Set<GridCacheVersion> TX_IDS = new GridConcurrentHashSet<>();
-
- /**
- * @param txCnt Tx count.
- */
- private static void init(int txCnt) {
- TX_CNT = txCnt;
- TX_IDS.clear();
- }
+ private static class CacheLocksListener implements IgnitePredicate<Event> {
+ /** Latch. */
+ private final CountDownLatch latch = new CountDownLatch(2);
/** {@inheritDoc} */
- @Override public void sendMessage(
- final ClusterNode node,
- final Message msg,
- final IgniteInClosure<IgniteException> ackC
- ) throws IgniteSpiException {
- if (msg instanceof GridIoMessage) {
- Message msg0 = ((GridIoMessage)msg).message();
-
- if (msg0 instanceof GridNearTxPrepareRequest) {
- final GridNearTxPrepareRequest req = (GridNearTxPrepareRequest)msg0;
-
- GridCacheVersion txId = req.version();
-
- if (TX_IDS.contains(txId)) {
- while (TX_IDS.size() < TX_CNT) {
- try {
- U.sleep(50);
- }
- catch (IgniteInterruptedCheckedException e) {
- e.printStackTrace();
- }
- }
- }
- }
- else if (msg0 instanceof GridNearTxPrepareResponse) {
- GridNearTxPrepareResponse res = (GridNearTxPrepareResponse)msg0;
+ @Override public boolean apply(Event evt) {
+ latch.countDown();
- GridCacheVersion txId = res.version();
-
- TX_IDS.add(txId);
- }
+ try {
+ latch.await();
+ }
+ catch (InterruptedException e) {
+ e.printStackTrace();
}
- super.sendMessage(node, msg, ackC);
+ return true;
}
}
-
}
[28/50] [abbrv] ignite git commit: IGNITE-8302 Reduce test IO
consumption in case Direct IO is enabled,
fixes flakyness of testPageRecoveryAfterFileCorruption - Fixes #3865.
Posted by ag...@apache.org.
IGNITE-8302 Reduce test IO consumption in case Direct IO is enabled, fixes flakyness of testPageRecoveryAfterFileCorruption - Fixes #3865.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d853ebb7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d853ebb7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d853ebb7
Branch: refs/heads/ignite-6083
Commit: d853ebb77d868f2626ff5cd613bbc38b1d13e594
Parents: 6c6f094
Author: dpavlov <dp...@apache.org>
Authored: Wed Apr 18 21:15:17 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Wed Apr 18 21:15:17 2018 +0300
----------------------------------------------------------------------
...gnitePdsRecoveryAfterFileCorruptionTest.java | 29 ++++++++++++--------
.../ignite/testsuites/IgnitePdsTestSuite2.java | 7 +++--
...iveIoPdsRecoveryAfterFileCorruptionTest.java | 12 ++++++++
.../testsuites/IgnitePdsNativeIoTestSuite2.java | 4 +++
4 files changed, 37 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d853ebb7/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
index 5dc0da9..e03cf52 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
@@ -54,14 +54,14 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
/**
- *
+ * This test generates WAL & Page Store with N pages, then rewrites pages with zeroes and tries to acquire all pages.
*/
public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstractTest {
/** Ip finder. */
private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
/** Total pages. */
- private static final int totalPages = 1024;
+ private static final int totalPages = 512;
/** Cache name. */
private final String cacheName = "cache";
@@ -120,7 +120,7 @@ public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstract
public void testPageRecoveryAfterFileCorruption() throws Exception {
IgniteEx ig = startGrid(0);
- ig.active(true);
+ ig.cluster().active(true);
IgniteCache<Integer, Integer> cache = ig.cache(cacheName);
@@ -144,13 +144,15 @@ public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstract
int cacheId = sharedCtx.cache().cache(cacheName).context().cacheId();
- FullPageId[] pages = new FullPageId[totalPages];
+ int pagesCnt = getTotalPagesToTest();
+
+ FullPageId[] pages = new FullPageId[pagesCnt];
// Get lock to prevent assertion. A new page should be allocated under checkpoint lock.
psMgr.checkpointReadLock();
try {
- for (int i = 0; i < totalPages; i++) {
+ for (int i = 0; i < pagesCnt; i++) {
pages[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
initPage(mem, pageIO, pages[i]);
@@ -174,12 +176,19 @@ public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstract
ig = startGrid(0);
- ig.active(true);
+ ig.cluster().active(true);
checkRestore(ig, pages);
}
/**
+ * @return count of pages to test. Note complexity of test is N^2.
+ */
+ protected int getTotalPagesToTest() {
+ return totalPages;
+ }
+
+ /**
* Initializes page.
* @param mem page memory implementation.
* @param pageIO page io implementation.
@@ -287,9 +296,7 @@ public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstract
wal.flush(start, false);
- for (int i = 0; i < totalPages; i++) {
- FullPageId fullId = pages[i];
-
+ for (FullPageId fullId : pages) {
long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
try {
@@ -325,9 +332,7 @@ public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstract
long write = 0;
- for (int i = 0; i < totalPages; i++) {
- FullPageId fullId = pages[i];
-
+ for (FullPageId fullId : pages) {
if (pageIds.contains(fullId)) {
long cpStart = System.nanoTime();
http://git-wip-us.apache.org/repos/asf/ignite/blob/d853ebb7/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index ede537e..6d953cd 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -77,7 +77,7 @@ public class IgnitePdsTestSuite2 extends TestSuite {
*
* @param suite suite to add tests into.
*/
- public static void addRealPageStoreTestsLongRunning(TestSuite suite) {
+ private static void addRealPageStoreTestsLongRunning(TestSuite suite) {
suite.addTestSuite(IgnitePdsTransactionsHangTest.class);
suite.addTestSuite(IgnitePdsPageEvictionDuringPartitionClearTest.class);
@@ -87,6 +87,9 @@ public class IgnitePdsTestSuite2 extends TestSuite {
suite.addTestSuite(IgnitePdsContinuousRestartTest2.class);
suite.addTestSuite(IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes.class);
+
+ // Integrity test.
+ suite.addTestSuite(IgnitePdsRecoveryAfterFileCorruptionTest.class);
}
/**
@@ -95,8 +98,6 @@ public class IgnitePdsTestSuite2 extends TestSuite {
* @param suite suite to add tests into.
*/
public static void addRealPageStoreTests(TestSuite suite) {
- // Integrity test.
- suite.addTestSuite(IgnitePdsRecoveryAfterFileCorruptionTest.class);
suite.addTestSuite(IgnitePdsPageSizesTest.class);
// Metrics test.
http://git-wip-us.apache.org/repos/asf/ignite/blob/d853ebb7/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java
----------------------------------------------------------------------
diff --git a/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java b/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java
new file mode 100644
index 0000000..4d65bf3
--- /dev/null
+++ b/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java
@@ -0,0 +1,12 @@
+package org.apache.ignite.internal.processors.cache.persistence;
+
+/**
+ * Native IO tests can't use page store, so this test limit pages count to 128.
+ * 524288 bytes to be read 128 times, 64Mbytes total read load.
+ */
+public class IgniteNativeIoPdsRecoveryAfterFileCorruptionTest extends IgnitePdsRecoveryAfterFileCorruptionTest {
+ /** {@inheritDoc} */
+ @Override protected int getTotalPagesToTest() {
+ return 128;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d853ebb7/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java b/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java
index 54dd7d3..679e3f7 100644
--- a/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java
+++ b/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java
@@ -17,6 +17,7 @@
package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteNativeIoPdsRecoveryAfterFileCorruptionTest;
/**
* Same as {@link IgnitePdsTestSuite2} but is started with direct-oi jar in classpath.
@@ -31,6 +32,9 @@ public class IgnitePdsNativeIoTestSuite2 extends TestSuite {
IgnitePdsTestSuite2.addRealPageStoreTests(suite);
+ //Integrity test with reduced count of pages.
+ suite.addTestSuite(IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.class);
+
return suite;
}
}
[42/50] [abbrv] ignite git commit: IGNITE-8078 Introduced additional
data storage metrics - Fixes #3845.
Posted by ag...@apache.org.
IGNITE-8078 Introduced additional data storage metrics - Fixes #3845.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ae18817f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ae18817f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ae18817f
Branch: refs/heads/ignite-6083
Commit: ae18817fbce547fcc8da42dc7ab7589f51e652e8
Parents: 8caeb3d
Author: Dmitriy Govorukhin <dm...@gmail.com>
Authored: Fri Apr 20 16:00:00 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Apr 20 16:00:00 2018 +0300
----------------------------------------------------------------------
.../org/apache/ignite/DataRegionMetrics.java | 35 ++++
.../org/apache/ignite/DataStorageMetrics.java | 70 +++++++
.../apache/ignite/internal/IgniteKernal.java | 1 +
.../cache/CacheGroupMetricsMXBeanImpl.java | 86 +++++++-
.../persistence/DataRegionMetricsImpl.java | 98 +++++++++-
.../DataRegionMetricsMXBeanImpl.java | 25 +++
.../persistence/DataRegionMetricsSnapshot.java | 45 +++++
.../persistence/DataStorageMetricsImpl.java | 195 ++++++++++++++++++-
.../persistence/DataStorageMetricsSnapshot.java | 90 +++++++++
.../GridCacheDatabaseSharedManager.java | 7 +-
.../IgniteCacheDatabaseSharedManager.java | 37 +++-
.../persistence/file/FilePageStoreManager.java | 17 +-
.../persistence/pagemem/PageMemoryImpl.java | 6 +
.../wal/FileWriteAheadLogManager.java | 24 ++-
.../wal/FsyncModeFileWriteAheadLogManager.java | 22 ++-
.../ignite/mxbean/CacheGroupMetricsMXBean.java | 24 +++
.../ignite/mxbean/DataRegionMetricsMXBean.java | 20 ++
.../ignite/mxbean/DataStorageMetricsMXBean.java | 40 ++++
.../database/SwapPathConstructionSelfTest.java | 4 +-
.../ApiParity/DataRegionMetricsParityTest.cs | 8 +-
.../ApiParity/DataStorageMetricsParityTest.cs | 14 +-
21 files changed, 849 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java b/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
index dc48f11..ca2fc66 100644
--- a/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
@@ -165,4 +165,39 @@ public interface DataRegionMetrics {
* @return Page size in bytes.
*/
public int getPageSize();
+
+ /**
+ * The number of read pages from last restart.
+ *
+ * @return The number of read pages from last restart.
+ */
+ public long getPagesRead();
+
+ /**
+ * The number of written pages from last restart.
+ *
+ * @return The number of written pages from last restart.
+ */
+ public long getPagesWritten();
+
+ /**
+ * The number of replaced pages from last restart .
+ *
+ * @return The number of replaced pages from last restart .
+ */
+ public long getPagesReplaced();
+
+ /**
+ * Total offheap size in bytes.
+ *
+ * @return Total offheap size in bytes.
+ */
+ public long getOffHeapSize();
+
+ /**
+ * Total used offheap size in bytes.
+ *
+ * @return Total used offheap size in bytes.
+ */
+ public long getOffheapUsedSize();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java b/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java
index 680caba..5fb2b1e 100644
--- a/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java
@@ -67,6 +67,27 @@ public interface DataStorageMetrics {
public long getWalBuffPollSpinsRate();
/**
+ * Total size in bytes for storage wal files.
+ *
+ * @return Total size in bytes for storage wal files.
+ */
+ public long getWalTotalSize();
+
+ /**
+ * Time of the last WAL segment rollover.
+ *
+ * @return Time of the last WAL segment rollover.
+ */
+ public long getWalLastRollOverTime();
+
+ /**
+ * Total checkpoint time from last restart.
+ *
+ * @return Total checkpoint time from last restart.
+ */
+ public long getCheckpointTotalTime();
+
+ /**
* Gets the duration of the last checkpoint in milliseconds.
*
* @return Total checkpoint duration in milliseconds.
@@ -121,4 +142,53 @@ public interface DataStorageMetrics {
* @return Total number of pages copied to a temporary checkpoint buffer during the last checkpoint.
*/
public long getLastCheckpointCopiedOnWritePagesNumber();
+
+ /**
+ * Total dirty pages for the next checkpoint.
+ *
+ * @return Total dirty pages for the next checkpoint.
+ */
+ public long getDirtyPages();
+
+ /**
+ * The number of read pages from last restart.
+ *
+ * @return The number of read pages from last restart.
+ */
+ public long getPagesRead();
+
+ /**
+ * The number of written pages from last restart.
+ *
+ * @return The number of written pages from last restart.
+ */
+ public long getPagesWritten();
+
+ /**
+ * The number of replaced pages from last restart.
+ *
+ * @return The number of replaced pages from last restart.
+ */
+ public long getPagesReplaced();
+
+ /**
+ * Total offheap size in bytes.
+ *
+ * @return Total offheap size in bytes.
+ */
+ public long getOffHeapSize();
+
+ /**
+ * Total used offheap size in bytes.
+ *
+ * @return Total used offheap size in bytes.
+ */
+ public long getOffheapUsedSize();
+
+ /**
+ * Total size of memory allocated in bytes.
+ *
+ * @return Total size of memory allocated in bytes.
+ */
+ public long getTotalAllocatedSize();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 795601e..019938f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -104,6 +104,7 @@ import org.apache.ignite.internal.binary.BinaryMarshaller;
import org.apache.ignite.internal.binary.BinaryUtils;
import org.apache.ignite.internal.cluster.ClusterGroupAdapter;
import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.failure.FailureProcessor;
import org.apache.ignite.internal.managers.GridManager;
import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager;
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
index 0788ee9..e9a2736 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
@@ -25,12 +25,18 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.atomic.LongAdder;
+import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
+import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
import org.apache.ignite.mxbean.CacheGroupMetricsMXBean;
/**
@@ -40,6 +46,9 @@ public class CacheGroupMetricsMXBeanImpl implements CacheGroupMetricsMXBean {
/** Cache group context. */
private final CacheGroupContext ctx;
+ /** */
+ private final GroupAllocationTrucker groupPageAllocationTracker;
+
/** Interface describing a predicate of two integers. */
private interface IntBiPredicate {
/**
@@ -52,12 +61,58 @@ public class CacheGroupMetricsMXBeanImpl implements CacheGroupMetricsMXBean {
}
/**
- * Creates MBean;
+ *
+ */
+ public static class GroupAllocationTrucker implements AllocatedPageTracker {
+ /** */
+ private final LongAdder totalAllocatedPages = new LongAdder();
+
+ /** */
+ private final AllocatedPageTracker delegate;
+
+ /**
+ * @param delegate Delegate allocation trucker.
+ */
+ public GroupAllocationTrucker(AllocatedPageTracker delegate) {
+ this.delegate = delegate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void updateTotalAllocatedPages(long delta) {
+ totalAllocatedPages.add(delta);
+
+ delegate.updateTotalAllocatedPages(delta);
+ }
+ }
+
+ /**
+ *
+ */
+ private static class NoopAllocationTrucker implements AllocatedPageTracker{
+ /** {@inheritDoc} */
+ @Override public void updateTotalAllocatedPages(long delta) {
+ // No-op.
+ }
+ }
+
+ /**
+ * Creates Group metrics MBean.
*
* @param ctx Cache group context.
*/
public CacheGroupMetricsMXBeanImpl(CacheGroupContext ctx) {
this.ctx = ctx;
+
+ DataRegion region = ctx.dataRegion();
+
+ // On client node, region is null.
+ if (region != null) {
+ DataRegionMetricsImpl dataRegionMetrics = ctx.dataRegion().memoryMetrics();
+
+ this.groupPageAllocationTracker = dataRegionMetrics.getOrAllocateGroupPageAllocationTracker(ctx.groupId());
+ }
+ else
+ this.groupPageAllocationTracker = new GroupAllocationTrucker(new NoopAllocationTrucker());
}
/** {@inheritDoc} */
@@ -252,4 +307,33 @@ public class CacheGroupMetricsMXBeanImpl implements CacheGroupMetricsMXBean {
return assignmentMap;
}
+
+ /** {@inheritDoc} */
+ @Override public String getType() {
+ CacheMode type = ctx.config().getCacheMode();
+
+ return String.valueOf(type);
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<Integer> getPartitionIds() {
+ List<GridDhtLocalPartition> parts = ctx.topology().localPartitions();
+
+ List<Integer> partsRes = new ArrayList<>(parts.size());
+
+ for (GridDhtLocalPartition part : parts)
+ partsRes.add(part.id());
+
+ return partsRes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTotalAllocatedPages() {
+ return groupPageAllocationTracker.totalAllocatedPages.longValue();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTotalAllocatedSize() {
+ return getTotalAllocatedPages() * ctx.dataRegion().pageMemory().pageSize();
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
index 7d6b478..cb13747 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
@@ -16,10 +16,13 @@
*/
package org.apache.ignite.internal.processors.cache.persistence;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;
import org.apache.ignite.DataRegionMetrics;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMXBeanImpl.GroupAllocationTrucker;
import org.apache.ignite.internal.processors.cache.ratemetrics.HitRateMetrics;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteOutClosure;
@@ -35,13 +38,28 @@ public class DataRegionMetricsImpl implements DataRegionMetrics, AllocatedPageTr
/** */
private final LongAdder totalAllocatedPages = new LongAdder();
+ /** */
+ private final ConcurrentHashMap<Integer, GroupAllocationTrucker> groupAllocationTruckers = new ConcurrentHashMap<>();
+
/**
* Counter for number of pages occupied by large entries (one entry is larger than one page).
*/
private final LongAdder largeEntriesPages = new LongAdder();
/** Counter for number of dirty pages. */
- private LongAdder dirtyPages = new LongAdder();
+ private final LongAdder dirtyPages = new LongAdder();
+
+ /** */
+ private final LongAdder readPages = new LongAdder();
+
+ /** */
+ private final LongAdder writtenPages = new LongAdder();
+
+ /** */
+ private final LongAdder replacedPages = new LongAdder();
+
+ /** */
+ private final AtomicLong offHeapSize = new AtomicLong();
/** */
private volatile boolean metricsEnabled;
@@ -212,7 +230,7 @@ public class DataRegionMetricsImpl implements DataRegionMetrics, AllocatedPageTr
}
/** {@inheritDoc} */
- public int getPageSize() {
+ @Override public int getPageSize() {
if (!metricsEnabled)
return 0;
@@ -221,6 +239,54 @@ public class DataRegionMetricsImpl implements DataRegionMetrics, AllocatedPageTr
return pageMem.pageSize();
}
+ /** {@inheritDoc} */
+ @Override public long getPagesRead() {
+ if (!metricsEnabled)
+ return 0;
+
+ return readPages.longValue();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPagesWritten() {
+ if (!metricsEnabled)
+ return 0;
+
+ return writtenPages.longValue();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPagesReplaced() {
+ if (!metricsEnabled)
+ return 0;
+
+ return replacedPages.longValue();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getOffHeapSize() {
+ if (!metricsEnabled)
+ return 0;
+
+ return offHeapSize.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getOffheapUsedSize() {
+ if (!metricsEnabled)
+ return 0;
+
+ return pageMem.loadedPages() * pageMem.pageSize();
+ }
+
+ /**
+ *
+ * @param size Region size.
+ */
+ public void updateOffHeapSize(long size) {
+ this.offHeapSize.addAndGet(size);
+ }
+
/**
* Updates pageReplaceRate metric.
*/
@@ -229,10 +295,28 @@ public class DataRegionMetricsImpl implements DataRegionMetrics, AllocatedPageTr
pageReplaceRate.onHit();
pageReplaceAge.onHits(pageAge);
+
+ replacedPages.increment();
}
}
/**
+ * Updates page read.
+ */
+ public void onPageRead(){
+ if (metricsEnabled)
+ readPages.increment();
+ }
+
+ /**
+ * Updates page written.
+ */
+ public void onPageWritten(){
+ if (metricsEnabled)
+ writtenPages.increment();
+ }
+
+ /**
* Increments dirtyPages counter.
*/
public void incrementDirtyPages() {
@@ -274,6 +358,16 @@ public class DataRegionMetricsImpl implements DataRegionMetrics, AllocatedPageTr
}
/**
+ * Get or allocate group allocation trucker.
+ *
+ * @param grpId Group id.
+ * @return Group allocation trucker.
+ */
+ public GroupAllocationTrucker getOrAllocateGroupPageAllocationTracker(int grpId) {
+ return groupAllocationTruckers.getOrDefault(grpId, new GroupAllocationTrucker(this));
+ }
+
+ /**
*
*/
private void updateAllocationRateMetrics(long hits) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
index fc746fe..9ab6824 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
@@ -158,4 +158,29 @@ class DataRegionMetricsMXBeanImpl implements DataRegionMetricsMXBean {
@Override public String getSwapPath() {
return dataRegCfg.getSwapPath();
}
+
+ /** {@inheritDoc} */
+ @Override public long getPagesRead() {
+ return memMetrics.getPagesRead();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPagesWritten() {
+ return memMetrics.getPagesWritten();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPagesReplaced() {
+ return memMetrics.getPagesReplaced();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getOffHeapSize() {
+ return memMetrics.getOffHeapSize();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getOffheapUsedSize() {
+ return memMetrics.getOffheapUsedSize();
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
index d715e82..6126c4b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
@@ -68,6 +68,21 @@ public class DataRegionMetricsSnapshot implements DataRegionMetrics {
/** */
private int pageSize;
+ /** */
+ private long readPages;
+
+ /** */
+ private long writtenPages;
+
+ /** */
+ private long replacedPage;
+
+ /** */
+ private long offHeapSize;
+
+ /** */
+ private long offHeapUsedSize;
+
/**
* @param metrics Metrics instance to take a copy.
*/
@@ -87,6 +102,11 @@ public class DataRegionMetricsSnapshot implements DataRegionMetrics {
checkpointBufferPages = metrics.getCheckpointBufferPages();
checkpointBufferSize = metrics.getCheckpointBufferSize();
pageSize = metrics.getPageSize();
+ readPages = metrics.getPagesRead();
+ writtenPages = metrics.getPagesWritten();
+ replacedPage = metrics.getPagesReplaced();
+ offHeapSize = metrics.getOffHeapSize();
+ offHeapUsedSize = metrics.getOffheapUsedSize();
}
/** {@inheritDoc} */
@@ -163,4 +183,29 @@ public class DataRegionMetricsSnapshot implements DataRegionMetrics {
@Override public int getPageSize() {
return pageSize;
}
+
+ /** {@inheritDoc} */
+ @Override public long getPagesRead() {
+ return readPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPagesWritten() {
+ return writtenPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPagesReplaced() {
+ return replacedPage;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getOffHeapSize() {
+ return offHeapSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getOffheapUsedSize() {
+ return offHeapUsedSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
index 4f32c25..b7f99e5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
@@ -16,8 +16,14 @@
*/
package org.apache.ignite.internal.processors.cache.persistence;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.DataRegionMetrics;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.processors.cache.ratemetrics.HitRateMetrics;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteOutClosure;
import org.apache.ignite.mxbean.DataStorageMetricsMXBean;
/**
@@ -73,7 +79,19 @@ public class DataStorageMetricsImpl implements DataStorageMetricsMXBean {
private volatile boolean metricsEnabled;
/** */
- private IgniteWriteAheadLogManager wal;
+ private volatile IgniteWriteAheadLogManager wal;
+
+ /** */
+ private volatile IgniteOutClosure<Long> walSizeProvider;
+
+ /** */
+ private volatile long lastWalSegmentRollOverTime;
+
+ /** */
+ private final AtomicLong totalCheckpointTime = new AtomicLong();
+
+ /** */
+ private volatile Collection<DataRegionMetrics> regionMetrics;
/**
* @param metricsEnabled Metrics enabled flag.
@@ -226,6 +244,158 @@ public class DataStorageMetricsImpl implements DataStorageMetricsMXBean {
resetRates();
}
+ /** {@inheritDoc} */
+ @Override public long getWalTotalSize() {
+ if (!metricsEnabled)
+ return 0;
+
+ IgniteOutClosure<Long> walSize = this.walSizeProvider;
+
+ return walSize != null ? walSize.apply() : 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getWalLastRollOverTime() {
+ if (!metricsEnabled)
+ return 0;
+
+ return lastWalSegmentRollOverTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getCheckpointTotalTime() {
+ if (!metricsEnabled)
+ return 0;
+
+ return totalCheckpointTime.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getDirtyPages() {
+ if (!metricsEnabled)
+ return 0;
+
+ Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+ if (F.isEmpty(regionMetrics0))
+ return 0;
+
+ long dirtyPages = 0L;
+
+ for (DataRegionMetrics rm : regionMetrics0)
+ dirtyPages += rm.getDirtyPages();
+
+ return dirtyPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPagesRead() {
+ if (!metricsEnabled)
+ return 0;
+
+ Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+ if (F.isEmpty(regionMetrics0))
+ return 0;
+
+ long readPages = 0L;
+
+ for (DataRegionMetrics rm : regionMetrics0)
+ readPages += rm.getPagesRead();
+
+ return readPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPagesWritten() {
+ if (!metricsEnabled)
+ return 0;
+
+ Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+ if (F.isEmpty(regionMetrics0))
+ return 0;
+
+ long writtenPages = 0L;
+
+ for (DataRegionMetrics rm : regionMetrics0)
+ writtenPages += rm.getPagesWritten();
+
+ return writtenPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPagesReplaced() {
+ if (!metricsEnabled)
+ return 0;
+
+ Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+ if (F.isEmpty(regionMetrics0))
+ return 0;
+
+ long replacedPages = 0L;
+
+ for (DataRegionMetrics rm : regionMetrics0)
+ replacedPages += rm.getPagesReplaced();
+
+ return replacedPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getOffHeapSize() {
+ if (!metricsEnabled)
+ return 0;
+
+ Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+ if (F.isEmpty(regionMetrics0))
+ return 0;
+
+ long offHeapSize = 0L;
+
+ for (DataRegionMetrics rm : regionMetrics0)
+ offHeapSize += rm.getOffHeapSize();
+
+ return offHeapSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getOffheapUsedSize() {
+ if (!metricsEnabled)
+ return 0;
+
+ Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+ if (F.isEmpty(regionMetrics0))
+ return 0;
+
+ long offHeapUsedSize = 0L;
+
+ for (DataRegionMetrics rm : regionMetrics0)
+ offHeapUsedSize += rm.getOffheapUsedSize();
+
+ return offHeapUsedSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTotalAllocatedSize() {
+ if (!metricsEnabled)
+ return 0;
+
+ Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+ if (F.isEmpty(regionMetrics0))
+ return 0;
+
+ long totalAllocatedSize = 0L;
+
+ for (DataRegionMetrics rm : regionMetrics0)
+ totalAllocatedSize += rm.getTotalAllocatedSize();
+
+ return totalAllocatedSize;
+ }
+
/**
* @param wal Write-ahead log manager.
*/
@@ -234,6 +404,27 @@ public class DataStorageMetricsImpl implements DataStorageMetricsMXBean {
}
/**
+ * @param walSizeProvider Wal size provider.
+ */
+ public void setWalSizeProvider(IgniteOutClosure<Long> walSizeProvider){
+ this.walSizeProvider = walSizeProvider;
+ }
+
+ /**
+ *
+ */
+ public void onWallRollOver() {
+ this.lastWalSegmentRollOverTime = U.currentTimeMillis();
+ }
+
+ /**
+ *
+ */
+ public void regionMetrics(Collection<DataRegionMetrics> regionMetrics){
+ this.regionMetrics = regionMetrics;
+ }
+
+ /**
* @return Metrics enabled flag.
*/
public boolean metricsEnabled() {
@@ -269,6 +460,8 @@ public class DataStorageMetricsImpl implements DataStorageMetricsMXBean {
lastCpTotalPages = totalPages;
lastCpDataPages = dataPages;
lastCpCowPages = cowPages;
+
+ totalCheckpointTime.addAndGet(duration);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java
index c67eeeb..cb52279 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java
@@ -62,6 +62,36 @@ public class DataStorageMetricsSnapshot implements DataStorageMetrics {
/** */
private long lastCpCowPages;
+ /** */
+ private long walTotalSize;
+
+ /** */
+ private long walLastRollOverTime;
+
+ /** */
+ private long checkpointTotalSize;
+
+ /** */
+ private long dirtyPages;
+
+ /** */
+ private long readPages;
+
+ /** */
+ private long writtenPages;
+
+ /** */
+ private long replacedPages;
+
+ /** */
+ private long offHeapSize;
+
+ /** */
+ private long offHeadUsedSize;
+
+ /** */
+ private long totalAllocatedSize;
+
/**
* @param metrics Metrics.
*/
@@ -79,6 +109,16 @@ public class DataStorageMetricsSnapshot implements DataStorageMetrics {
lastCpTotalPages = metrics.getLastCheckpointTotalPagesNumber();
lastCpDataPages = metrics.getLastCheckpointDataPagesNumber();
lastCpCowPages = metrics.getLastCheckpointCopiedOnWritePagesNumber();
+ walTotalSize = metrics.getWalTotalSize();
+ walLastRollOverTime = metrics.getWalLastRollOverTime();
+ checkpointTotalSize = metrics.getCheckpointTotalTime();
+ dirtyPages = metrics.getDirtyPages();
+ readPages = metrics.getPagesRead();
+ writtenPages = metrics.getPagesWritten();
+ replacedPages = metrics.getPagesReplaced();
+ offHeapSize = metrics.getOffHeapSize();
+ offHeadUsedSize = metrics.getOffheapUsedSize();
+ totalAllocatedSize = metrics.getTotalAllocatedSize();
}
/** {@inheritDoc} */
@@ -147,6 +187,56 @@ public class DataStorageMetricsSnapshot implements DataStorageMetrics {
}
/** {@inheritDoc} */
+ @Override public long getWalTotalSize() {
+ return walTotalSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getWalLastRollOverTime() {
+ return walLastRollOverTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getCheckpointTotalTime() {
+ return checkpointTotalSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getDirtyPages() {
+ return dirtyPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPagesRead() {
+ return readPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPagesWritten() {
+ return writtenPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPagesReplaced() {
+ return replacedPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getOffHeapSize() {
+ return offHeapSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getOffheapUsedSize() {
+ return offHeadUsedSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTotalAllocatedSize() {
+ return totalAllocatedSize;
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return S.toString(DataStorageMetricsSnapshot.class, this);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index d314d50..e20abd6 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -86,6 +86,7 @@ import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.DirectMemoryRegion;
import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
import org.apache.ignite.internal.pagemem.FullPageId;
@@ -470,6 +471,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
createDataRegionConfiguration(memCfg),
false
);
+
+ persStoreMetrics.regionMetrics(memMetricsMap.values());
}
/**
@@ -1009,7 +1012,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
changeTracker = null;
PageMemoryImpl pageMem = new PageMemoryImpl(
- memProvider,
+ wrapMetricsMemoryProvider(memProvider, memMetrics),
calculateFragmentSizes(
memCfg.getConcurrencyLevel(),
cacheSize,
@@ -1018,6 +1021,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
cctx,
memCfg.getPageSize(),
(fullId, pageBuf, tag) -> {
+ memMetrics.onPageWritten();
+
// First of all, write page to disk.
storeMgr.write(fullId.groupId(), fullId.pageId(), pageBuf, tag);
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index bf080b6..a251234 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -37,6 +37,7 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.DirectMemoryRegion;
import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
import org.apache.ignite.internal.pagemem.PageMemory;
@@ -247,6 +248,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
CU.isPersistenceEnabled(memCfg)
);
+
dataRegionsInitialized = true;
}
@@ -957,7 +959,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
PageMemory pageMem = new PageMemoryNoStoreImpl(
log,
- memProvider,
+ wrapMetricsMemoryProvider(memProvider, memMetrics),
cctx,
memCfg.getPageSize(),
memPlcCfg,
@@ -971,6 +973,39 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
}
/**
+ * @param memoryProvider0 Memory provider.
+ * @param memMetrics Memory metrics.
+ * @return Wrapped memory provider.
+ */
+ protected DirectMemoryProvider wrapMetricsMemoryProvider(
+ final DirectMemoryProvider memoryProvider0,
+ final DataRegionMetricsImpl memMetrics
+ ) {
+ return new DirectMemoryProvider() {
+ private final DirectMemoryProvider memProvider = memoryProvider0;
+
+ @Override public void initialize(long[] chunkSizes) {
+ memProvider.initialize(chunkSizes);
+ }
+
+ @Override public void shutdown() {
+ memProvider.shutdown();
+ }
+
+ @Override public DirectMemoryRegion nextRegion() {
+ DirectMemoryRegion nextMemoryRegion = memProvider.nextRegion();
+
+ if (nextMemoryRegion == null)
+ return null;
+
+ memMetrics.updateOffHeapSize(nextMemoryRegion.size());
+
+ return nextMemoryRegion;
+ }
+ };
+ }
+
+ /**
* @param path Path to the working directory.
* @param consId Consistent ID of the local node.
* @return DB storage path.
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 1c1b3e2..661694d 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -52,10 +52,12 @@ import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
import org.apache.ignite.internal.processors.cache.StoredCacheData;
import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.marshaller.Marshaller;
import org.apache.ignite.marshaller.jdk.JdkMarshaller;
@@ -409,13 +411,20 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
File cacheWorkDir = cacheWorkDir(ccfg);
- AllocatedPageTracker allocatedTracker =
- cctx.database().dataRegion(grpDesc.config().getDataRegionName()).memoryMetrics();
+ String dataRegionName = grpDesc.config().getDataRegionName();
- return initDir(cacheWorkDir,
+ DataRegionMetricsImpl regionMetrics = cctx.database().dataRegion(dataRegionName).memoryMetrics();
+
+ int grpId = CU.cacheId(grpDesc.cacheOrGroupName());
+
+ AllocatedPageTracker allocatedTracker = regionMetrics.getOrAllocateGroupPageAllocationTracker(grpId);
+
+ return initDir(
+ cacheWorkDir,
grpDesc.groupId(),
grpDesc.config().getAffinity().partitions(),
- allocatedTracker);
+ allocatedTracker
+ );
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index c684ce1..10d0a3f 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -777,6 +777,8 @@ public class PageMemoryImpl implements PageMemoryEx {
try {
storeMgr.read(grpId, pageId, buf);
+
+ memMetrics.onPageRead();
}
catch (IgniteDataIntegrityViolationException ignore) {
U.warn(log, "Failed to read page (data integrity violation encountered, will try to " +
@@ -785,6 +787,8 @@ public class PageMemoryImpl implements PageMemoryEx {
buf.rewind();
tryToRestorePage(fullId, buf);
+
+ memMetrics.onPageRead();
}
finally {
rwLock.writeUnlock(lockedPageAbsPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
@@ -1173,6 +1177,8 @@ public class PageMemoryImpl implements PageMemoryEx {
assert PageIO.getType(outBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
assert PageIO.getVersion(outBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(fullId.pageId());
+ memMetrics.onPageWritten();
+
return true;
}
finally {
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 9917e85..a28b73b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -66,7 +66,6 @@ import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.events.EventType;
import org.apache.ignite.events.WalSegmentArchivedEvent;
import org.apache.ignite.failure.FailureContext;
-import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
@@ -101,6 +100,7 @@ import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.CIX1;
+import org.apache.ignite.internal.util.typedef.CO;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -367,14 +367,14 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
checkWalConfiguration();
- walWorkDir = initDirectory(
+ final File walWorkDir0 = walWorkDir = initDirectory(
dsCfg.getWalPath(),
DataStorageConfiguration.DFLT_WAL_PATH,
resolveFolders.folderName(),
"write ahead log work directory"
);
- walArchiveDir = initDirectory(
+ final File walArchiveDir0 = walArchiveDir = initDirectory(
dsCfg.getWalArchivePath(),
DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH,
resolveFolders.folderName(),
@@ -389,6 +389,21 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
checkOrPrepareFiles();
+ if (metrics != null)
+ metrics.setWalSizeProvider(new CO<Long>() {
+ @Override public Long apply() {
+ long size = 0;
+
+ for (File f : walWorkDir0.listFiles())
+ size += f.length();
+
+ for (File f : walArchiveDir0.listFiles())
+ size += f.length();
+
+ return size;
+ }
+ });
+
IgniteBiTuple<Long, Long> tup = scanMinMaxArchiveIndices();
lastTruncatedArchiveIdx = tup == null ? -1 : tup.get1() - 1;
@@ -1042,6 +1057,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
return hnd;
if (hnd.close(true)) {
+ if (metrics.metricsEnabled())
+ metrics.onWallRollOver();
+
FileWriteHandle next = initNextWriteHandle(cur);
next.writeHeader();
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 2305389..e354b43 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -94,6 +94,7 @@ import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.CIX1;
+import org.apache.ignite.internal.util.typedef.CO;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.SB;
@@ -320,14 +321,14 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
checkWalConfiguration();
- walWorkDir = initDirectory(
+ final File walWorkDir0 = walWorkDir = initDirectory(
dsCfg.getWalPath(),
DataStorageConfiguration.DFLT_WAL_PATH,
resolveFolders.folderName(),
"write ahead log work directory"
);
- walArchiveDir = initDirectory(
+ final File walArchiveDir0 = walArchiveDir = initDirectory(
dsCfg.getWalArchivePath(),
DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH,
resolveFolders.folderName(),
@@ -342,6 +343,20 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
checkOrPrepareFiles();
+ metrics.setWalSizeProvider(new CO<Long>() {
+ @Override public Long apply() {
+ long size = 0;
+
+ for (File f : walWorkDir0.listFiles())
+ size += f.length();
+
+ for (File f : walArchiveDir0.listFiles())
+ size += f.length();
+
+ return size;
+ }
+ });
+
IgniteBiTuple<Long, Long> tup = scanMinMaxArchiveIndices();
lastTruncatedArchiveIdx = tup == null ? -1 : tup.get1() - 1;
@@ -944,6 +959,9 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
return hnd;
if (hnd.close(true)) {
+ if (metrics.metricsEnabled())
+ metrics.onWallRollOver();
+
FileWriteHandle next = initNextWriteHandle(cur.idx);
boolean swapped = currentHndUpd.compareAndSet(this, hnd, next);
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/mxbean/CacheGroupMetricsMXBean.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/CacheGroupMetricsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/CacheGroupMetricsMXBean.java
index 46b99b1..9973d84 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/CacheGroupMetricsMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/CacheGroupMetricsMXBean.java
@@ -139,4 +139,28 @@ public interface CacheGroupMetricsMXBean {
*/
@MXBeanDescription("Affinity partitions assignment map.")
public Map<Integer, List<String>> getAffinityPartitionsAssignmentMap();
+
+ /**
+ * Cache group type.
+ */
+ @MXBeanDescription("Cache group type.")
+ public String getType();
+
+ /**
+ * Local partition ids.
+ */
+ @MXBeanDescription("Local partition ids.")
+ public List<Integer> getPartitionIds();
+
+ /**
+ * Cache group total allocated pages.
+ */
+ @MXBeanDescription("Cache group total allocated pages.")
+ public long getTotalAllocatedPages();
+
+ /**
+ * Total size of memory allocated for group, in bytes.
+ */
+ @MXBeanDescription("Total size of memory allocated for group, in bytes.")
+ public long getTotalAllocatedSize();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/mxbean/DataRegionMetricsMXBean.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/DataRegionMetricsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/DataRegionMetricsMXBean.java
index 5b2697a..9bb1af6 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/DataRegionMetricsMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/DataRegionMetricsMXBean.java
@@ -89,6 +89,26 @@ public interface DataRegionMetricsMXBean extends DataRegionMetrics {
@MXBeanDescription("Number of pages residing in physical RAM.")
@Override public long getPhysicalMemoryPages();
+ /** {@inheritDoc} */
+ @MXBeanDescription("Number of pages read from last restart.")
+ @Override public long getPagesRead();
+
+ /** {@inheritDoc} */
+ @MXBeanDescription("Number of pages written from last restart.")
+ @Override public long getPagesWritten();
+
+ /** {@inheritDoc} */
+ @MXBeanDescription("Number of pages replaced from last restart.")
+ @Override public long getPagesReplaced();
+
+ /** {@inheritDoc} */
+ @MXBeanDescription("Offheap size in bytes.")
+ @Override public long getOffHeapSize();
+
+ /** {@inheritDoc} */
+ @MXBeanDescription("Offheap used size in bytes.")
+ @Override public long getOffheapUsedSize();
+
/**
* Enables memory metrics collection on an Apache Ignite node.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java
index 2051da3..2450874 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java
@@ -45,6 +45,18 @@ public interface DataStorageMetricsMXBean extends DataStorageMetrics {
@Override long getWalBuffPollSpinsRate();
/** {@inheritDoc} */
+ @MXBeanDescription("Total size in bytes for storage wal files.")
+ @Override long getWalTotalSize();
+
+ /** {@inheritDoc} */
+ @MXBeanDescription("Time of the last WAL segment rollover.")
+ @Override long getWalLastRollOverTime();
+
+ /** {@inheritDoc} */
+ @MXBeanDescription("Total checkpoint time from last restart.")
+ @Override long getCheckpointTotalTime();
+
+ /** {@inheritDoc} */
@MXBeanDescription("Duration of the last checkpoint in milliseconds.")
@Override long getLastCheckpointDuration();
@@ -76,6 +88,34 @@ public interface DataStorageMetricsMXBean extends DataStorageMetrics {
@MXBeanDescription("Number of pages copied to a temporary checkpoint buffer during the last checkpoint.")
@Override long getLastCheckpointCopiedOnWritePagesNumber();
+ /** {@inheritDoc} */
+ @MXBeanDescription("Total dirty pages for the next checkpoint.")
+ @Override long getDirtyPages();
+
+ /** {@inheritDoc} */
+ @MXBeanDescription("The number of read pages from last restart.")
+ @Override long getPagesRead();
+
+ /** {@inheritDoc} */
+ @MXBeanDescription("The number of written pages from last restart.")
+ @Override long getPagesWritten();
+
+ /** {@inheritDoc} */
+ @MXBeanDescription("The number of replaced pages from last restart.")
+ @Override long getPagesReplaced();
+
+ /** {@inheritDoc} */
+ @MXBeanDescription("Total offheap size in bytes.")
+ @Override long getOffHeapSize();
+
+ /** {@inheritDoc} */
+ @MXBeanDescription("Total used offheap size in bytes.")
+ @Override long getOffheapUsedSize();
+
+ /** {@inheritDoc} */
+ @MXBeanDescription("Total size of memory allocated in bytes.")
+ @Override long getTotalAllocatedSize();
+
/**
* Enables persistence metrics collection on an Apache Ignite node.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java
index 416087b..177bcbe 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java
@@ -124,7 +124,9 @@ public class SwapPathConstructionSelfTest extends GridCommonAbstractTest {
Object memProvider = U.field(pageMem, "directMemoryProvider");
- return ((File) U.field(memProvider, "allocationPath")).getAbsolutePath();
+ Object memProvider0 = U.field(memProvider, "memProvider");
+
+ return ((File) U.field(memProvider0, "allocationPath")).getAbsolutePath();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
index bea31b1..22b8986 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
@@ -34,7 +34,13 @@ namespace Apache.Ignite.Core.Tests.ApiParity
};
/** Properties that are missing on .NET side. */
- private static readonly string[] MissingProperties = {};
+ private static readonly string[] MissingProperties = {
+ "PagesRead",
+ "PagesWritten",
+ "PagesReplaced",
+ "OffHeapSize",
+ "OffheapUsedSize"
+ };
/// <summary>
/// Tests the API parity.
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae18817f/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs
index 2206891..300ab25 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs
@@ -1,4 +1,4 @@
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -28,7 +28,17 @@ namespace Apache.Ignite.Core.Tests.ApiParity
private static readonly string[] MissingProperties =
{
// IGNITE-7305
- "WalBuffPollSpinsRate"
+ "WalBuffPollSpinsRate",
+ "WalTotalSize",
+ "WalLastRollOverTime",
+ "CheckpointTotalTime",
+ "DirtyPages",
+ "PagesRead",
+ "PagesWritten",
+ "PagesReplaced",
+ "OffHeapSize",
+ "OffheapUsedSize",
+ "TotalAllocatedSize"
};
/// <summary>
[31/50] [abbrv] ignite git commit: IGNITE-6666: Support Time data
type in BinaryObjectImpl.writeFieldByOrder. This closes #2878.
Posted by ag...@apache.org.
IGNITE-6666: Support Time data type in BinaryObjectImpl.writeFieldByOrder. This closes #2878.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c09c68ad
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c09c68ad
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c09c68ad
Branch: refs/heads/ignite-6083
Commit: c09c68adc36cc2353e11878d30c3ea29fbcbdde7
Parents: 0897309
Author: Nikita Amelchev <ns...@gmail.com>
Authored: Thu Apr 19 12:57:06 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Apr 19 12:57:06 2018 +0300
----------------------------------------------------------------------
.../ignite/internal/binary/BinaryFieldImpl.java | 9 +++++
.../internal/binary/BinaryObjectImpl.java | 1 +
.../binary/BinaryFieldExtractionSelfTest.java | 35 ++++++++++++++++++++
3 files changed, 45 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c09c68ad/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java
index 883576c..de0b2d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java
@@ -21,6 +21,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.sql.Time;
import java.sql.Timestamp;
import java.util.Date;
import java.util.UUID;
@@ -211,6 +212,14 @@ public class BinaryFieldImpl implements BinaryFieldEx {
break;
}
+ case GridBinaryMarshaller.TIME: {
+ long time = buf.getLong();
+
+ val = new Time(time);
+
+ break;
+ }
+
case GridBinaryMarshaller.UUID: {
long most = buf.getLong();
long least = buf.getLong();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c09c68ad/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
index 4f91e25..7ab8b35 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
@@ -545,6 +545,7 @@ public final class BinaryObjectImpl extends BinaryObjectExImpl implements Extern
case GridBinaryMarshaller.LONG:
case GridBinaryMarshaller.DOUBLE:
case GridBinaryMarshaller.DATE:
+ case GridBinaryMarshaller.TIME:
totalLen = 9;
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c09c68ad/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldExtractionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldExtractionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldExtractionSelfTest.java
index 9f39746..a050591 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldExtractionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldExtractionSelfTest.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.binary;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
+import java.sql.Time;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.ignite.configuration.BinaryConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
@@ -100,6 +101,27 @@ public class BinaryFieldExtractionSelfTest extends GridCommonAbstractTest {
/**
* @throws Exception If failed.
*/
+ public void testTimeMarshalling() throws Exception {
+ BinaryMarshaller marsh = createMarshaller();
+
+ TimeValue obj = new TimeValue(11111L);
+
+ BinaryObjectImpl binObj = toBinary(obj, marsh);
+
+ BinaryFieldEx field = (BinaryFieldEx)binObj.type().field("time");
+
+ ByteBuffer buf = ByteBuffer.allocate(16);
+
+ field.writeField(binObj, buf);
+
+ buf.flip();
+
+ assertEquals(field.value(binObj), field.<Time>readField(buf));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testDecimalFieldMarshalling() throws Exception {
BinaryMarshaller marsh = createMarshaller();
@@ -194,6 +216,19 @@ public class BinaryFieldExtractionSelfTest extends GridCommonAbstractTest {
}
}
+ /** */
+ private static class TimeValue {
+ /** */
+ private Time time;
+
+ /**
+ * @param time Time.
+ */
+ TimeValue(long time) {
+ this.time = new Time(time);
+ }
+ }
+
/**
*
*/
[17/50] [abbrv] ignite git commit: IGNITE-8294 Web Console: Move
"Beta" ribbon to the left.
Posted by ag...@apache.org.
IGNITE-8294 Web Console: Move "Beta" ribbon to the left.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/69606e42
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/69606e42
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/69606e42
Branch: refs/heads/ignite-6083
Commit: 69606e4248a69d44f798c9f59865281c19744a69
Parents: eda5fe7
Author: Ilya Borisov <kl...@gmail.com>
Authored: Wed Apr 18 11:39:41 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Apr 18 11:39:41 2018 +0700
----------------------------------------------------------------------
.../web-console/frontend/public/stylesheets/style.scss | 10 ++++++++++
modules/web-console/frontend/views/index.pug | 2 +-
2 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/69606e42/modules/web-console/frontend/public/stylesheets/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index ae1e58c..0c978e9 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -1938,6 +1938,16 @@ treecontrol.tree-classic {
}
}
+.ribbon-wrapper.left {
+ overflow: visible;
+
+ .ribbon {
+ transform: rotate(-45deg);
+ left: -75px;
+ top: 10px;
+ }
+}
+
html, body {
width: 100%;
min-height: 100vh;
http://git-wip-us.apache.org/repos/asf/ignite/blob/69606e42/modules/web-console/frontend/views/index.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/views/index.pug b/modules/web-console/frontend/views/index.pug
index e91af9b..6384592 100644
--- a/modules/web-console/frontend/views/index.pug
+++ b/modules/web-console/frontend/views/index.pug
@@ -39,7 +39,7 @@ html(ng-app='ignite-console' id='app' ng-strict-di)
.splash-wellcome Loading...
- .ribbon-wrapper.right(ng-if='!IgniteDemoMode')
+ .ribbon-wrapper.left(ng-if='!IgniteDemoMode')
.ribbon
label Beta
[12/50] [abbrv] ignite git commit: IGNITE-2766 Fix .net test. - Fixes
#3853.
Posted by ag...@apache.org.
IGNITE-2766 Fix .net test. - Fixes #3853.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/96cb795f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/96cb795f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/96cb795f
Branch: refs/heads/ignite-6083
Commit: 96cb795f3d97af0e3dcb95fa0c81e31257d83431
Parents: 7b39f13
Author: Ilya Kasnacheev <il...@gmail.com>
Authored: Tue Apr 17 19:50:51 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Tue Apr 17 19:50:51 2018 +0300
----------------------------------------------------------------------
.../platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/96cb795f/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs
index 7e6222f..274439e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs
@@ -98,10 +98,8 @@ namespace Apache.Ignite.Core.Tests
cache1[1] = 2;
Assert.AreEqual(2, cache1[1]);
- // Check that old cache instance does not work.
- var cacheEx1 = Assert.Throws<InvalidOperationException>(() => cache.Get(1));
- Assert.IsTrue(cacheEx1.Message.EndsWith("Failed to perform cache operation (cache is stopped): cache"),
- cacheEx1.Message);
+ // Check that old cache instance still works.
+ Assert.AreEqual(2, cache.Get(1));
}
/// <summary>
[32/50] [abbrv] ignite git commit: IGNITE-8017 Javadoc fix for
Disable WAL during initial rebalancing
Posted by ag...@apache.org.
IGNITE-8017 Javadoc fix for Disable WAL during initial rebalancing
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/02a08651
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/02a08651
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/02a08651
Branch: refs/heads/ignite-6083
Commit: 02a0865153f41f0fb0abec0ccdf9adc4b9cc5cbe
Parents: c09c68a
Author: dpavlov <dp...@apache.org>
Authored: Thu Apr 19 15:17:58 2018 +0300
Committer: dpavlov <dp...@apache.org>
Committed: Thu Apr 19 15:17:58 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/processors/cache/CacheGroupContext.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/02a08651/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 849ecc8..5f750d5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -1036,7 +1036,7 @@ public class CacheGroupContext {
}
/**
- * @Global WAL enabled flag.
+ * @return Global WAL enabled flag.
*/
public boolean globalWalEnabled() {
return globalWalEnabled;
[24/50] [abbrv] ignite git commit: IGNITE-8017 Disable WAL during
initial rebalancing
Posted by ag...@apache.org.
IGNITE-8017 Disable WAL during initial rebalancing
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8826fa13
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8826fa13
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8826fa13
Branch: refs/heads/ignite-6083
Commit: 8826fa13edd0ab776b9fa2ab482b5a216b99b103
Parents: ea8bfac
Author: Ilya Lantukh <il...@gridgain.com>
Authored: Wed Apr 18 17:52:20 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Apr 18 17:52:20 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 7 +
.../processors/cache/CacheGroupContext.java | 54 +-
.../processors/cache/WalStateManager.java | 160 +++++-
.../dht/GridClientPartitionTopology.java | 5 +
.../dht/GridDhtPartitionTopology.java | 7 +
.../dht/GridDhtPartitionTopologyImpl.java | 27 +
.../dht/preloader/GridDhtPartitionDemander.java | 24 +-
.../GridDhtPartitionsExchangeFuture.java | 2 +
.../GridCacheDatabaseSharedManager.java | 63 +-
.../IgniteCacheDatabaseSharedManager.java | 4 +-
.../persistence/file/FilePageStoreManager.java | 18 +-
...lWalModeChangeDuringRebalancingSelfTest.java | 571 +++++++++++++++++++
.../ignite/testsuites/IgnitePdsTestSuite2.java | 3 +
13 files changed, 895 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 437f49f..32fed05 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -878,6 +878,13 @@ public final class IgniteSystemProperties {
public static final String IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD = "IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD";
/**
+ * When set to {@code true}, WAL will be automatically disabled during rebalancing if there is no partition in
+ * OWNING state.
+ * Default is {@code false}.
+ */
+ public static final String IGNITE_DISABLE_WAL_DURING_REBALANCING = "IGNITE_DISABLE_WAL_DURING_REBALANCING";
+
+ /**
* Enforces singleton.
*/
private IgniteSystemProperties() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 12636f3..849ecc8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -152,7 +152,10 @@ public class CacheGroupContext {
private CacheGroupMetricsMXBean mxBean;
/** */
- private volatile boolean walEnabled;
+ private volatile boolean localWalEnabled;
+
+ /** */
+ private volatile boolean globalWalEnabled;
/**
* @param grpId Group ID.
@@ -196,9 +199,10 @@ public class CacheGroupContext {
this.reuseList = reuseList;
this.locStartVer = locStartVer;
this.cacheType = cacheType;
- this.walEnabled = walEnabled;
+ this.globalWalEnabled = walEnabled;
+ this.localWalEnabled = true;
- persistWalState(walEnabled);
+ persistGlobalWalState(walEnabled);
ioPlc = cacheType.ioPolicy();
@@ -1021,22 +1025,52 @@ public class CacheGroupContext {
* WAL enabled flag.
*/
public boolean walEnabled() {
- return walEnabled;
+ return localWalEnabled && globalWalEnabled;
+ }
+
+ /**
+ * Local WAL enabled flag.
+ */
+ public boolean localWalEnabled() {
+ return localWalEnabled;
+ }
+
+ /**
+ * @Global WAL enabled flag.
+ */
+ public boolean globalWalEnabled() {
+ return globalWalEnabled;
}
/**
- * @param enabled WAL enabled flag.
+ * @param enabled Global WAL enabled flag.
*/
- public void walEnabled(boolean enabled) {
- persistWalState(enabled);
+ public void globalWalEnabled(boolean enabled) {
+ persistGlobalWalState(enabled);
- this.walEnabled = enabled;
+ this.globalWalEnabled = enabled;
+ }
+
+ /**
+ * @param enabled Local WAL enabled flag.
+ */
+ public void localWalEnabled(boolean enabled) {
+ persistLocalWalState(enabled);
+
+ this.localWalEnabled = enabled;
+ }
+
+ /**
+ * @param enabled Enabled flag..
+ */
+ private void persistGlobalWalState(boolean enabled) {
+ shared().database().walEnabled(grpId, enabled, false);
}
/**
* @param enabled Enabled flag..
*/
- private void persistWalState(boolean enabled) {
- shared().database().walEnabled(grpId, enabled);
+ private void persistLocalWalState(boolean enabled) {
+ shared().database().walEnabled(grpId, enabled, true);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
index 64a6819..4a14730 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
@@ -18,7 +18,9 @@
package org.apache.ignite.internal.processors.cache;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.IgniteConfiguration;
@@ -26,10 +28,13 @@ import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.managers.communication.GridMessageListener;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.persistence.CheckpointFuture;
import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashSet;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -44,15 +49,18 @@ import org.jetbrains.annotations.Nullable;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import static org.apache.ignite.internal.GridTopic.TOPIC_WAL;
import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
/**
* Write-ahead log state manager. Manages WAL enable and disable.
@@ -102,6 +110,9 @@ public class WalStateManager extends GridCacheSharedManagerAdapter {
/** Disconnected flag. */
private boolean disconnected;
+ /** Holder for groups with temporary disabled WAL. */
+ private volatile TemporaryDisabledWal tmpDisabledWal;
+
/**
* Constructor.
*
@@ -328,6 +339,126 @@ public class WalStateManager extends GridCacheSharedManagerAdapter {
}
/**
+ * Change local WAL state before exchange is done. This method will disable WAL for groups without partitions
+ * in OWNING state if such feature is enabled.
+ *
+ * @param topVer Topology version.
+ */
+ public void changeLocalStatesOnExchangeDone(AffinityTopologyVersion topVer) {
+ if (!IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING, false))
+ return;
+
+ Set<Integer> grpsToEnableWal = new HashSet<>();
+ Set<Integer> grpsToDisableWal = new HashSet<>();
+ Set<Integer> grpsWithWalDisabled = new HashSet<>();
+
+ boolean hasNonEmptyOwning = false;
+
+ for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
+ if (grp.isLocal() || !grp.affinityNode() || !grp.persistenceEnabled())
+ continue;
+
+ boolean hasOwning = false;
+
+ for (GridDhtLocalPartition locPart : grp.topology().currentLocalPartitions()) {
+ if (locPart.state() == OWNING) {
+ hasOwning = true;
+
+ if (hasNonEmptyOwning)
+ break;
+
+ if (locPart.updateCounter() > 0) {
+ hasNonEmptyOwning = true;
+
+ break;
+ }
+ }
+ }
+
+ if (hasOwning && !grp.localWalEnabled()) {
+ grpsToEnableWal.add(grp.groupId());
+ }
+ else if (!hasOwning && grp.localWalEnabled()) {
+ grpsToDisableWal.add(grp.groupId());
+
+ grpsWithWalDisabled.add(grp.groupId());
+ }
+ else if (!grp.localWalEnabled())
+ grpsWithWalDisabled.add(grp.groupId());
+ }
+
+ tmpDisabledWal = new TemporaryDisabledWal(grpsWithWalDisabled, topVer);
+
+ if (grpsToEnableWal.isEmpty() && grpsToDisableWal.isEmpty())
+ return;
+
+ try {
+ if (hasNonEmptyOwning && !grpsToEnableWal.isEmpty())
+ triggerCheckpoint(0).finishFuture().get();
+ }
+ catch (IgniteCheckedException ex) {
+ throw new IgniteException(ex);
+ }
+
+ for (Integer grpId : grpsToEnableWal)
+ cctx.cache().cacheGroup(grpId).localWalEnabled(true);
+
+ for (Integer grpId : grpsToDisableWal)
+ cctx.cache().cacheGroup(grpId).localWalEnabled(false);
+ }
+
+ /**
+ * Callback when group rebalancing is finished. If there are no pending groups, it should trigger checkpoint and
+ * change partition states.
+ * @param grpId Group ID.
+ * @param topVer Topology version.
+ */
+ public void onGroupRebalanceFinished(int grpId, AffinityTopologyVersion topVer) {
+ TemporaryDisabledWal session0 = tmpDisabledWal;
+
+ if (session0 == null || !session0.topVer.equals(topVer))
+ return;
+
+ session0.remainingGrps.remove(grpId);
+
+ if (session0.remainingGrps.isEmpty()) {
+ synchronized (mux) {
+ if (tmpDisabledWal != session0)
+ return;
+
+ for (Integer grpId0 : session0.disabledGrps) {
+ CacheGroupContext grp = cctx.cache().cacheGroup(grpId0);
+
+ assert grp != null;
+
+ if (!grp.localWalEnabled())
+ grp.localWalEnabled(true);
+ }
+
+ tmpDisabledWal = null;
+ }
+
+ CheckpointFuture cpFut = triggerCheckpoint(0);
+
+ assert cpFut != null;
+
+ cpFut.finishFuture().listen(new IgniteInClosureX<IgniteInternalFuture>() {
+ @Override public void applyx(IgniteInternalFuture future) {
+ for (Integer grpId0 : session0.disabledGrps) {
+ CacheGroupContext grp = cctx.cache().cacheGroup(grpId0);
+
+ assert grp != null;
+
+ grp.topology().ownMoving(session0.topVer);
+ }
+
+ cctx.exchange().refreshPartitions();
+ }
+ });
+ }
+ }
+
+ /**
* Handle propose message in discovery thread.
*
* @param msg Message.
@@ -455,7 +586,7 @@ public class WalStateManager extends GridCacheSharedManagerAdapter {
"no longer exist: " + msg.caches().keySet());
}
else {
- if (F.eq(msg.enable(), grpCtx.walEnabled()))
+ if (F.eq(msg.enable(), grpCtx.globalWalEnabled()))
// Nothing changed -> no-op.
res = new WalStateResult(msg, false);
else {
@@ -468,7 +599,7 @@ public class WalStateManager extends GridCacheSharedManagerAdapter {
cpFut.beginFuture().get();
if (msg.enable()) {
- grpCtx.walEnabled(true);
+ grpCtx.globalWalEnabled(true);
// Enable: it is enough to release cache operations once mark is finished because
// not-yet-flushed dirty pages have been logged.
@@ -489,7 +620,7 @@ public class WalStateManager extends GridCacheSharedManagerAdapter {
// WAL state is persisted after checkpoint if finished. Otherwise in case of crash
// and restart we will think that WAL is enabled, but data might be corrupted.
- grpCtx.walEnabled(false);
+ grpCtx.globalWalEnabled(false);
}
}
catch (Exception e) {
@@ -917,4 +1048,27 @@ public class WalStateManager extends GridCacheSharedManagerAdapter {
onCompletedLocally(res);
}
}
+
+ /**
+ *
+ */
+ private static class TemporaryDisabledWal {
+ /** Groups with disabled WAL. */
+ private final Set<Integer> disabledGrps;
+
+ /** Remaining groups. */
+ private final Set<Integer> remainingGrps;
+
+ /** Topology version*/
+ private final AffinityTopologyVersion topVer;
+
+ /** */
+ public TemporaryDisabledWal(
+ Set<Integer> disabledGrps,
+ AffinityTopologyVersion topVer) {
+ this.disabledGrps = Collections.unmodifiableSet(disabledGrps);
+ this.remainingGrps = new HashSet<>(disabledGrps);
+ this.topVer = topVer;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index 3e3bb0d..dcb8b96 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -1104,6 +1104,11 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
+ @Override public void ownMoving(AffinityTopologyVersion topVer) {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
@Override public void onEvicted(GridDhtLocalPartition part, boolean updateSeq) {
assert updateSeq || lock.isWriteLockedByCurrentThread();
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index d586a94d..2df2e89 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -357,6 +357,13 @@ public interface GridDhtPartitionTopology {
public boolean own(GridDhtLocalPartition part);
/**
+ * Owns all moving partitions for the given topology version.
+ *
+ * @param topVer Topology version.
+ */
+ public void ownMoving(AffinityTopologyVersion topVer);
+
+ /**
* @param part Evicted partition.
* @param updateSeq Update sequence increment flag.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 164f0bf..68104a5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -2410,6 +2410,33 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
+ @Override public void ownMoving(AffinityTopologyVersion topVer) {
+ lock.writeLock().lock();
+
+ try {
+ for (GridDhtLocalPartition locPart : grp.topology().currentLocalPartitions()) {
+ if (locPart.state() == MOVING) {
+ boolean reserved = locPart.reserve();
+
+ try {
+ if (reserved && locPart.state() == MOVING && lastTopChangeVer.equals(topVer))
+ grp.topology().own(locPart);
+ else // topology changed, rebalancing must be restarted
+ return;
+ }
+ finally {
+ if (reserved)
+ locPart.release();
+ }
+ }
+ }
+ }
+ finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public void onEvicted(GridDhtLocalPartition part, boolean updateSeq) {
ctx.database().checkpointReadLock();
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 337553b..dc4bfe9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -57,6 +57,7 @@ import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.CI1;
@@ -282,6 +283,14 @@ public class GridDhtPartitionDemander {
final RebalanceFuture fut = new RebalanceFuture(grp, assignments, log, rebalanceId);
+ if (!grp.localWalEnabled())
+ fut.listen(new IgniteInClosureX<IgniteInternalFuture<Boolean>>() {
+ @Override public void applyx(IgniteInternalFuture<Boolean> future) throws IgniteCheckedException {
+ if (future.get())
+ ctx.walState().onGroupRebalanceFinished(grp.groupId(), assignments.topologyVersion());
+ }
+ });
+
if (!oldFut.isInitial())
oldFut.cancel();
else
@@ -722,9 +731,7 @@ public class GridDhtPartitionDemander {
// If message was last for this partition,
// then we take ownership.
if (last) {
- top.own(part);
-
- fut.partitionDone(nodeId, p);
+ fut.partitionDone(nodeId, p, true);
if (log.isDebugEnabled())
log.debug("Finished rebalancing partition: " + part);
@@ -737,14 +744,14 @@ public class GridDhtPartitionDemander {
}
else {
if (last)
- fut.partitionDone(nodeId, p);
+ fut.partitionDone(nodeId, p, false);
if (log.isDebugEnabled())
log.debug("Skipping rebalancing partition (state is not MOVING): " + part);
}
}
else {
- fut.partitionDone(nodeId, p);
+ fut.partitionDone(nodeId, p, false);
if (log.isDebugEnabled())
log.debug("Skipping rebalancing partition (it does not belong on current node): " + p);
@@ -762,7 +769,7 @@ public class GridDhtPartitionDemander {
}
for (Integer miss : supply.missed())
- fut.partitionDone(nodeId, miss);
+ fut.partitionDone(nodeId, miss, false);
GridDhtPartitionDemandMessage d = new GridDhtPartitionDemandMessage(
supply.rebalanceId(),
@@ -1064,8 +1071,11 @@ public class GridDhtPartitionDemander {
* @param nodeId Node id.
* @param p Partition number.
*/
- private void partitionDone(UUID nodeId, int p) {
+ private void partitionDone(UUID nodeId, int p, boolean updateState) {
synchronized (this) {
+ if (updateState && grp.localWalEnabled())
+ grp.topology().own(grp.topology().localPartition(p));
+
if (isDone())
return;
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 0d57d48..a21d98e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1702,6 +1702,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
if (!grp.isLocal())
grp.topology().onExchangeDone(this, grp.affinity().readyAffinity(res), false);
}
+
+ cctx.walState().changeLocalStatesOnExchangeDone(exchId.topologyVersion());
}
if (super.onDone(res, err)) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 8f81708..d314d50 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.io.RandomAccessFile;
+import java.io.Serializable;
import java.lang.ref.SoftReference;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
@@ -285,7 +286,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
private static final String MBEAN_GROUP = "Persistent Store";
/** WAL marker prefix for meta store. */
- private static final String WAL_KEY_PREFIX = "grp-wal-disabled-";
+ private static final String WAL_KEY_PREFIX = "grp-wal-";
+
+ /** WAL marker prefix for meta store. */
+ private static final String WAL_GLOBAL_KEY_PREFIX = WAL_KEY_PREFIX + "disabled-";
+
+ /** WAL marker prefix for meta store. */
+ private static final String WAL_LOCAL_KEY_PREFIX = WAL_KEY_PREFIX + "local-disabled-";
/** WAL marker predicate for meta store. */
private static final IgnitePredicate<String> WAL_KEY_PREFIX_PRED = new IgnitePredicate<String>() {
@@ -385,7 +392,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
private List<MetastorageLifecycleListener> metastorageLifecycleLsnrs;
/** Initially disabled cache groups. */
- private Collection<Integer> initiallyWalDisabledGrps;
+ private Collection<Integer> initiallyGlobalWalDisabledGrps = new HashSet<>();
+
+ private Collection<Integer> initiallyLocalWalDisabledGrps = new HashSet<>();
/** File I/O factory for writing checkpoint markers. */
private final FileIOFactory ioFactory;
@@ -596,7 +605,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
applyLastUpdates(status, true);
- initiallyWalDisabledGrps = walDisabledGroups();
+ fillWalDisabledGroups();
notifyMetastorageReadyForRead();
}
@@ -1974,7 +1983,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
int applied = 0;
WALPointer lastRead = null;
- Collection<Integer> ignoreGrps = storeOnly ? Collections.emptySet() : initiallyWalDisabledGrps;
+ Collection<Integer> ignoreGrps = storeOnly ? Collections.emptySet() :
+ F.concat(false, initiallyGlobalWalDisabledGrps, initiallyLocalWalDisabledGrps);
try (WALIterator it = cctx.wal().replay(status.endPtr)) {
while (it.hasNextX()) {
@@ -2228,7 +2238,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
long start = U.currentTimeMillis();
int applied = 0;
- Collection<Integer> ignoreGrps = metastoreOnly ? Collections.emptySet() : initiallyWalDisabledGrps;
+ Collection<Integer> ignoreGrps = metastoreOnly ? Collections.emptySet() :
+ F.concat(false, initiallyGlobalWalDisabledGrps, initiallyLocalWalDisabledGrps);
try (WALIterator it = cctx.wal().replay(status.startPtr)) {
Map<T2<Integer, Integer>, T2<Integer, Long>> partStates = new HashMap<>();
@@ -4401,13 +4412,16 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/** {@inheritDoc} */
- @Override public boolean walEnabled(int grpId) {
- return !initiallyWalDisabledGrps.contains(grpId);
+ @Override public boolean walEnabled(int grpId, boolean local) {
+ if (local)
+ return !initiallyLocalWalDisabledGrps.contains(grpId);
+ else
+ return !initiallyGlobalWalDisabledGrps.contains(grpId);
}
/** {@inheritDoc} */
- @Override public void walEnabled(int grpId, boolean enabled) {
- String key = walGroupIdToKey(grpId);
+ @Override public void walEnabled(int grpId, boolean enabled, boolean local) {
+ String key = walGroupIdToKey(grpId, local);
checkpointReadLock();
@@ -4427,27 +4441,26 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
- * @return List of initially WAL-disabled groups.
+ *
*/
- private Collection<Integer> walDisabledGroups() {
+ private void fillWalDisabledGroups() {
MetaStorage meta = cctx.database().metaStorage();
try {
Set<String> keys = meta.readForPredicate(WAL_KEY_PREFIX_PRED).keySet();
if (keys.isEmpty())
- return Collections.emptySet();
-
- HashSet<Integer> res = new HashSet<>(keys.size());
+ return;
for (String key : keys) {
- int grpId = walKeyToGroupId(key);
+ T2<Integer, Boolean> t2 = walKeyToGroupIdAndLocalFlag(key);
- res.add(grpId);
+ if (t2.get2())
+ initiallyLocalWalDisabledGrps.add(t2.get1());
+ else
+ initiallyGlobalWalDisabledGrps.add(t2.get1());
}
- return res;
-
}
catch (IgniteCheckedException e) {
throw new IgniteException("Failed to read cache groups WAL state.", e);
@@ -4460,8 +4473,11 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
* @param grpId Group ID.
* @return Key.
*/
- private static String walGroupIdToKey(int grpId) {
- return WAL_KEY_PREFIX + grpId;
+ private static String walGroupIdToKey(int grpId, boolean local) {
+ if (local)
+ return WAL_LOCAL_KEY_PREFIX + grpId;
+ else
+ return WAL_GLOBAL_KEY_PREFIX + grpId;
}
/**
@@ -4470,7 +4486,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
* @param key Key.
* @return Group ID.
*/
- private static int walKeyToGroupId(String key) {
- return Integer.parseInt(key.substring(WAL_KEY_PREFIX.length()));
+ private static T2<Integer, Boolean> walKeyToGroupIdAndLocalFlag(String key) {
+ if (key.startsWith(WAL_LOCAL_KEY_PREFIX))
+ return new T2<>(Integer.parseInt(key.substring(WAL_LOCAL_KEY_PREFIX.length())), true);
+ else
+ return new T2<>(Integer.parseInt(key.substring(WAL_GLOBAL_KEY_PREFIX.length())), false);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 8746dca..bf080b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -1035,7 +1035,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
* @param grpId Group ID.
* @return WAL enabled flag.
*/
- public boolean walEnabled(int grpId) {
+ public boolean walEnabled(int grpId, boolean local) {
return false;
}
@@ -1045,7 +1045,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
* @param grpId Group id.
* @param enabled flag.
*/
- public void walEnabled(int grpId, boolean enabled) {
+ public void walEnabled(int grpId, boolean enabled, boolean local) {
// No-op.
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 837f3d0..1c1b3e2 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -845,16 +845,22 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
/** {@inheritDoc} */
@Override public void beforeCacheGroupStart(CacheGroupDescriptor grpDesc) {
- if (grpDesc.persistenceEnabled() && !cctx.database().walEnabled(grpDesc.groupId())) {
- File dir = cacheWorkDir(grpDesc.config());
+ if (grpDesc.persistenceEnabled()) {
+ boolean localEnabled = cctx.database().walEnabled(grpDesc.groupId(), true);
+ boolean globalEnabled = cctx.database().walEnabled(grpDesc.groupId(), false);
- assert dir.exists();
+ if (!localEnabled || !globalEnabled) {
+ File dir = cacheWorkDir(grpDesc.config());
- boolean res = IgniteUtils.delete(dir);
+ assert dir.exists();
- assert res;
+ boolean res = IgniteUtils.delete(dir);
- grpDesc.walEnabled(false);
+ assert res;
+
+ if (!globalEnabled)
+ grpDesc.walEnabled(false);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
new file mode 100644
index 0000000..07653f2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
@@ -0,0 +1,571 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.file.OpenOption;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstractTest {
+ /** */
+ private static boolean disableWalDuringRebalancing = true;
+
+ /** */
+ private static final AtomicReference<CountDownLatch> supplyMessageLatch = new AtomicReference<>();
+
+ /** */
+ private static final AtomicReference<CountDownLatch> fileIOLatch = new AtomicReference<>();
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setMaxSize(200 * 1024 * 1024)
+ .setInitialSize(200 * 1024 * 1024)
+ )
+ // Test verifies checkpoint count, so it is essencial that no checkpoint is triggered by timeout
+ .setCheckpointFrequency(999_999_999_999L)
+ .setFileIOFactory(new TestFileIOFactory(new DataStorageConfiguration().getFileIOFactory()))
+ );
+
+ cfg.setCacheConfiguration(
+ new CacheConfiguration(DEFAULT_CACHE_NAME)
+ // Test checks internal state before and after rebalance, so it is configured to be triggered manually
+ .setRebalanceDelay(-1)
+ );
+
+ cfg.setCommunicationSpi(new TcpCommunicationSpi() {
+ @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+ if (msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof GridDhtPartitionSupplyMessage) {
+ int grpId = ((GridDhtPartitionSupplyMessage)((GridIoMessage)msg).message()).groupId();
+
+ if (grpId == CU.cacheId(DEFAULT_CACHE_NAME)) {
+ CountDownLatch latch0 = supplyMessageLatch.get();
+
+ if (latch0 != null)
+ try {
+ latch0.await();
+ }
+ catch (InterruptedException ex) {
+ throw new IgniteException(ex);
+ }
+ }
+ }
+
+ super.sendMessage(node, msg);
+ }
+
+ @Override public void sendMessage(ClusterNode node, Message msg,
+ IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+ if (msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof GridDhtPartitionSupplyMessage) {
+ int grpId = ((GridDhtPartitionSupplyMessage)((GridIoMessage)msg).message()).groupId();
+
+ if (grpId == CU.cacheId(DEFAULT_CACHE_NAME)) {
+ CountDownLatch latch0 = supplyMessageLatch.get();
+
+ if (latch0 != null)
+ try {
+ latch0.await();
+ }
+ catch (InterruptedException ex) {
+ throw new IgniteException(ex);
+ }
+ }
+ }
+
+ super.sendMessage(node, msg, ackC);
+ }
+ });
+
+ cfg.setConsistentId(igniteInstanceName);
+
+ System.setProperty(IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING,
+ Boolean.toString(disableWalDuringRebalancing));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ CountDownLatch msgLatch = supplyMessageLatch.get();
+
+ if (msgLatch != null) {
+ while (msgLatch.getCount() > 0)
+ msgLatch.countDown();
+
+ supplyMessageLatch.set(null);
+ }
+
+ CountDownLatch fileLatch = fileIOLatch.get();
+
+ if (fileLatch != null) {
+ while (fileLatch.getCount() > 0)
+ fileLatch.countDown();
+
+ fileIOLatch.set(null);
+ }
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+
+ disableWalDuringRebalancing = true;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testWalDisabledDuringRebalancing() throws Exception {
+ doTestSimple();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testWalNotDisabledIfParameterSetToFalse() throws Exception {
+ disableWalDuringRebalancing = false;
+
+ doTestSimple();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void doTestSimple() throws Exception {
+ Ignite ignite = startGrids(3);
+
+ ignite.cluster().active(true);
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+ for (int k = 0; k < 10_000; k++)
+ cache.put(k, k);
+
+ IgniteEx newIgnite = startGrid(3);
+
+ final GridCacheDatabaseSharedManager.CheckpointHistory cpHistory =
+ ((GridCacheDatabaseSharedManager)newIgnite.context().cache().context().database()).checkpointHistory();
+
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return !cpHistory.checkpoints().isEmpty();
+ }
+ }, 10_000);
+
+ U.sleep(10); // To ensure timestamp granularity.
+
+ long newIgniteStartedTimestamp = System.currentTimeMillis();
+
+ ignite.cluster().setBaselineTopology(4);
+
+ CacheGroupContext grpCtx = newIgnite.cachex(DEFAULT_CACHE_NAME).context().group();
+
+ assertEquals(!disableWalDuringRebalancing, grpCtx.walEnabled());
+
+ U.sleep(10); // To ensure timestamp granularity.
+
+ long rebalanceStartedTimestamp = System.currentTimeMillis();
+
+ for (Ignite g : G.allGrids())
+ g.cache(DEFAULT_CACHE_NAME).rebalance();
+
+ awaitPartitionMapExchange();
+
+ assertTrue(grpCtx.walEnabled());
+
+ U.sleep(10); // To ensure timestamp granularity.
+
+ long rebalanceFinishedTimestamp = System.currentTimeMillis();
+
+ for (Integer k = 0; k < 1000; k++)
+ assertEquals("k=" + k, k, cache.get(k));
+
+ int checkpointsBeforeNodeStarted = 0;
+ int checkpointsBeforeRebalance = 0;
+ int checkpointsAfterRebalance = 0;
+
+ for (Long timestamp : cpHistory.checkpoints()) {
+ if (timestamp < newIgniteStartedTimestamp)
+ checkpointsBeforeNodeStarted++;
+ else if (timestamp >= newIgniteStartedTimestamp && timestamp < rebalanceStartedTimestamp)
+ checkpointsBeforeRebalance++;
+ else if (timestamp >= rebalanceStartedTimestamp && timestamp <= rebalanceFinishedTimestamp)
+ checkpointsAfterRebalance++;
+ }
+
+ assertEquals(1, checkpointsBeforeNodeStarted); // checkpoint on start
+ assertEquals(0, checkpointsBeforeRebalance);
+ assertEquals(disableWalDuringRebalancing ? 1 : 0, checkpointsAfterRebalance); // checkpoint if WAL was re-activated
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testLocalAndGlobalWalStateInterdependence() throws Exception {
+ Ignite ignite = startGrids(3);
+
+ ignite.cluster().active(true);
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+ for (int k = 0; k < 10_000; k++)
+ cache.put(k, k);
+
+ IgniteEx newIgnite = startGrid(3);
+
+ ignite.cluster().setBaselineTopology(ignite.cluster().nodes());
+
+ CacheGroupContext grpCtx = newIgnite.cachex(DEFAULT_CACHE_NAME).context().group();
+
+ assertFalse(grpCtx.walEnabled());
+
+ ignite.cluster().disableWal(DEFAULT_CACHE_NAME);
+
+ for (Ignite g : G.allGrids())
+ g.cache(DEFAULT_CACHE_NAME).rebalance();
+
+ awaitPartitionMapExchange();
+
+ assertFalse(grpCtx.walEnabled()); // WAL is globally disabled
+
+ ignite.cluster().enableWal(DEFAULT_CACHE_NAME);
+
+ assertTrue(grpCtx.walEnabled());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testParallelExchangeDuringRebalance() throws Exception {
+ doTestParallelExchange(supplyMessageLatch);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testParallelExchangeDuringCheckpoint() throws Exception {
+ doTestParallelExchange(fileIOLatch);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void doTestParallelExchange(AtomicReference<CountDownLatch> latchRef) throws Exception {
+ Ignite ignite = startGrids(3);
+
+ ignite.cluster().active(true);
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+ for (int k = 0; k < 10_000; k++)
+ cache.put(k, k);
+
+ IgniteEx newIgnite = startGrid(3);
+
+ CacheGroupContext grpCtx = newIgnite.cachex(DEFAULT_CACHE_NAME).context().group();
+
+ CountDownLatch latch = new CountDownLatch(1);
+
+ latchRef.set(latch);
+
+ ignite.cluster().setBaselineTopology(ignite.cluster().nodes());
+
+ for (Ignite g : G.allGrids())
+ g.cache(DEFAULT_CACHE_NAME).rebalance();
+
+ assertFalse(grpCtx.walEnabled());
+
+ // TODO : test with client node as well
+ startGrid(4); // Trigger exchange
+
+ assertFalse(grpCtx.walEnabled());
+
+ latch.countDown();
+
+ assertFalse(grpCtx.walEnabled());
+
+ for (Ignite g : G.allGrids())
+ g.cache(DEFAULT_CACHE_NAME).rebalance();
+
+ awaitPartitionMapExchange();
+
+ assertTrue(grpCtx.walEnabled());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDataClearedAfterRestartWithDisabledWal() throws Exception {
+ Ignite ignite = startGrid(0);
+
+ ignite.cluster().active(true);
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+ for (int k = 0; k < 10_000; k++)
+ cache.put(k, k);
+
+ IgniteEx newIgnite = startGrid(1);
+
+ ignite.cluster().setBaselineTopology(2);
+
+ CacheGroupContext grpCtx = newIgnite.cachex(DEFAULT_CACHE_NAME).context().group();
+
+ assertFalse(grpCtx.localWalEnabled());
+
+ stopGrid(1);
+ stopGrid(0);
+
+ newIgnite = startGrid(1);
+
+ newIgnite.cluster().active(true);
+
+ newIgnite.cluster().setBaselineTopology(newIgnite.cluster().nodes());
+
+ cache = newIgnite.cache(DEFAULT_CACHE_NAME);
+
+ for (int k = 0; k < 10_000; k++)
+ assertFalse("k=" + k +", v=" + cache.get(k), cache.containsKey(k));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testWalNotDisabledAfterShrinkingBaselineTopology() throws Exception {
+ Ignite ignite = startGrids(4);
+
+ ignite.cluster().active(true);
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+ for (int k = 0; k < 10_000; k++)
+ cache.put(k, k);
+
+ for (Ignite g : G.allGrids()) {
+ CacheGroupContext grpCtx = ((IgniteEx)g).cachex(DEFAULT_CACHE_NAME).context().group();
+
+ assertTrue(grpCtx.walEnabled());
+ }
+
+ stopGrid(2);
+
+ ignite.cluster().setBaselineTopology(5);
+
+ for (Ignite g : G.allGrids()) {
+ CacheGroupContext grpCtx = ((IgniteEx)g).cachex(DEFAULT_CACHE_NAME).context().group();
+
+ assertTrue(grpCtx.walEnabled());
+
+ g.cache(DEFAULT_CACHE_NAME).rebalance();
+ }
+
+ awaitPartitionMapExchange();
+
+ for (Ignite g : G.allGrids()) {
+ CacheGroupContext grpCtx = ((IgniteEx)g).cachex(DEFAULT_CACHE_NAME).context().group();
+
+ assertTrue(grpCtx.walEnabled());
+ }
+ }
+
+ /**
+ *
+ */
+ private static class TestFileIOFactory implements FileIOFactory {
+ /** */
+ private final FileIOFactory delegate;
+
+ /**
+ * @param delegate Delegate.
+ */
+ TestFileIOFactory(FileIOFactory delegate) {
+ this.delegate = delegate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file) throws IOException {
+ return new TestFileIO(delegate.create(file));
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+ return new TestFileIO(delegate.create(file, modes));
+ }
+ }
+
+ /**
+ *
+ */
+ private static class TestFileIO implements FileIO {
+ /** */
+ private final FileIO delegate;
+
+ /**
+ * @param delegate Delegate.
+ */
+ TestFileIO(FileIO delegate) {
+ this.delegate = delegate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long position() throws IOException {
+ return delegate.position();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void position(long newPosition) throws IOException {
+ delegate.position(newPosition);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(ByteBuffer destBuf) throws IOException {
+ return delegate.read(destBuf);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(ByteBuffer destBuf, long position) throws IOException {
+ return delegate.read(destBuf, position);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(byte[] buf, int off, int len) throws IOException {
+ return delegate.read(buf, off, len);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int write(ByteBuffer srcBuf) throws IOException {
+ CountDownLatch latch = fileIOLatch.get();
+
+ if (latch != null && Thread.currentThread().getName().contains("checkpoint"))
+ try {
+ latch.await();
+ }
+ catch (InterruptedException ex) {
+ throw new IgniteException(ex);
+ }
+
+ return delegate.write(srcBuf);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int write(ByteBuffer srcBuf, long position) throws IOException {
+ CountDownLatch latch = fileIOLatch.get();
+
+ if (latch != null && Thread.currentThread().getName().contains("checkpoint"))
+ try {
+ latch.await();
+ }
+ catch (InterruptedException ex) {
+ throw new IgniteException(ex);
+ }
+
+ return delegate.write(srcBuf, position);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(byte[] buf, int off, int len) throws IOException {
+ CountDownLatch latch = fileIOLatch.get();
+
+ if (latch != null && Thread.currentThread().getName().contains("checkpoint"))
+ try {
+ latch.await();
+ }
+ catch (InterruptedException ex) {
+ throw new IgniteException(ex);
+ }
+
+ delegate.write(buf, off, len);
+ }
+
+ /** {@inheritDoc} */
+ @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
+ return delegate.map(maxWalSegmentSize);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void force() throws IOException {
+ delegate.force();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void force(boolean withMetadata) throws IOException {
+ delegate.force(withMetadata);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long size() throws IOException {
+ return delegate.size();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void clear() throws IOException {
+ delegate.clear();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IOException {
+ delegate.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8826fa13/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index f955b11..ede537e 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsExchange
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
+import org.apache.ignite.internal.processors.cache.persistence.LocalWalModeChangeDuringRebalancingSelfTest;
import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAllBaselineNodesOnlineFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOfflineBaselineNodeFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOnlineNodeOutOfBaselineFullApiSelfTest;
@@ -135,5 +136,7 @@ public class IgnitePdsTestSuite2 extends TestSuite {
suite.addTestSuite(IgniteCheckpointDirtyPagesForLowLoadTest.class);
suite.addTestSuite(IgnitePdsCorruptedStoreTest.class);
+
+ suite.addTestSuite(LocalWalModeChangeDuringRebalancingSelfTest.class);
}
}
[38/50] [abbrv] ignite git commit: IGNITE-7077: Implementation of
Spark query optimization. - Fixes #3397.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationMathFuncSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationMathFuncSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationMathFuncSpec.scala
new file mode 100644
index 0000000..02793c9
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationMathFuncSpec.scala
@@ -0,0 +1,358 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.Ignite
+import org.apache.ignite.cache.query.SqlFieldsQuery
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+import org.apache.spark.sql.ignite.IgniteSparkSession
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import java.lang.{Double ⇒ JDouble, Long ⇒ JLong}
+
+/**
+ */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationMathFuncSpec extends AbstractDataFrameSpec {
+ var igniteSession: IgniteSparkSession = _
+
+ describe("Supported optimized string functions") {
+ it("ABS") {
+ val df = igniteSession.sql("SELECT ABS(val) FROM numbers WHERE id = 6")
+
+ checkOptimizationResult(df, "SELECT ABS(val) FROM numbers WHERE id is not null AND id = 6")
+
+ val data = Tuple1(.5)
+
+ checkQueryData(df, data)
+ }
+
+ it("ACOS") {
+ val df = igniteSession.sql("SELECT ACOS(val) FROM numbers WHERE id = 7")
+
+ checkOptimizationResult(df, "SELECT ACOS(val) FROM numbers WHERE id is not null AND id = 7")
+
+ val data = Tuple1(Math.PI)
+
+ checkQueryData(df, data)
+ }
+
+ it("ASIN") {
+ val df = igniteSession.sql("SELECT ASIN(val) FROM numbers WHERE id = 7")
+
+ checkOptimizationResult(df, "SELECT ASIN(val) FROM numbers WHERE id is not null AND id = 7")
+
+ val data = Tuple1(-Math.PI/2)
+
+ checkQueryData(df, data)
+ }
+
+ it("ATAN") {
+ val df = igniteSession.sql("SELECT ATAN(val) FROM numbers WHERE id = 7")
+
+ checkOptimizationResult(df, "SELECT ATAN(val) FROM numbers WHERE id is not null AND id = 7")
+
+ val data = Tuple1(-Math.PI/4)
+
+ checkQueryData(df, data)
+ }
+
+ it("COS") {
+ val df = igniteSession.sql("SELECT COS(val) FROM numbers WHERE id = 1")
+
+ checkOptimizationResult(df, "SELECT COS(val) FROM numbers WHERE id is not null AND id = 1")
+
+ val data = Tuple1(1.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("SIN") {
+ val df = igniteSession.sql("SELECT SIN(val) FROM numbers WHERE id = 1")
+
+ checkOptimizationResult(df, "SELECT SIN(val) FROM numbers WHERE id is not null AND id = 1")
+
+ val data = Tuple1(.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("TAN") {
+ val df = igniteSession.sql("SELECT TAN(val) FROM numbers WHERE id = 1")
+
+ checkOptimizationResult(df, "SELECT TAN(val) FROM numbers WHERE id is not null AND id = 1")
+
+ val data = Tuple1(.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("COSH") {
+ val df = igniteSession.sql("SELECT COSH(val) FROM numbers WHERE id = 1")
+
+ checkOptimizationResult(df, "SELECT COSH(val) FROM numbers WHERE id is not null AND id = 1")
+
+ val data = Tuple1(1.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("SINH") {
+ val df = igniteSession.sql("SELECT SINH(val) FROM numbers WHERE id = 1")
+
+ checkOptimizationResult(df, "SELECT SINH(val) FROM numbers WHERE id is not null AND id = 1")
+
+ val data = Tuple1(.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("TANH") {
+ val df = igniteSession.sql("SELECT TANH(val) FROM numbers WHERE id = 1")
+
+ checkOptimizationResult(df, "SELECT TANH(val) FROM numbers WHERE id is not null AND id = 1")
+
+ val data = Tuple1(.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("ATAN2") {
+ val df = igniteSession.sql("SELECT ATAN2(val, 0.0) FROM numbers WHERE id = 1")
+
+ checkOptimizationResult(df, "SELECT ATAN2(val, 0.0) AS \"ATAN2(val, CAST(0.0 AS DOUBLE))\" " +
+ "FROM numbers WHERE id is not null AND id = 1")
+
+ val data = Tuple1(.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("MOD") {
+ val df = igniteSession.sql("SELECT val % 9 FROM numbers WHERE id = 8")
+
+ checkOptimizationResult(df, "SELECT val % 9.0 as \"(val % CAST(9 AS DOUBLE))\" " +
+ "FROM numbers WHERE id is not null AND id = 8")
+
+ val data = Tuple1(6.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("CEIL") {
+ val df = igniteSession.sql("SELECT CEIL(val) FROM numbers WHERE id = 2")
+
+ checkOptimizationResult(df, "SELECT CAST(CEIL(val) AS LONG) as \"CEIL(val)\" " +
+ "FROM numbers WHERE id is not null AND id = 2")
+
+ val data = Tuple1(1)
+
+ checkQueryData(df, data)
+ }
+
+ it("ROUND") {
+ val df = igniteSession.sql("SELECT id, ROUND(val) FROM numbers WHERE id IN (2, 9, 10)")
+
+ checkOptimizationResult(df, "SELECT id, ROUND(val, 0) FROM numbers WHERE id IN (2, 9, 10)")
+
+ val data = (
+ (2, 1.0),
+ (9, 1.0),
+ (10, 0.0))
+
+ checkQueryData(df, data)
+ }
+
+ it("FLOOR") {
+ val df = igniteSession.sql("SELECT FLOOR(val) FROM numbers WHERE id = 2")
+
+ checkOptimizationResult(df, "SELECT CAST(FLOOR(val) AS LONG) as \"FLOOR(val)\" FROM numbers " +
+ "WHERE id is not null AND id = 2")
+
+ val data = Tuple1(0)
+
+ checkQueryData(df, data)
+ }
+
+ it("POWER") {
+ val df = igniteSession.sql("SELECT POWER(val, 3) FROM numbers WHERE id = 4")
+
+ checkOptimizationResult(df, "SELECT POWER(val, 3.0) as \"POWER(val, CAST(3 AS DOUBLE))\" FROM numbers " +
+ "WHERE id is not null AND id = 4")
+
+ val data = Tuple1(8.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("EXP") {
+ val df = igniteSession.sql("SELECT id, EXP(val) FROM numbers WHERE id IN (1, 3)")
+
+ checkOptimizationResult(df, "SELECT id, EXP(val) FROM numbers WHERE id IN (1, 3)")
+
+ val data = (
+ (1, 1),
+ (3, Math.E))
+
+ checkQueryData(df, data)
+ }
+
+ it("LOG") {
+ val df = igniteSession.sql("SELECT LOG(val) FROM numbers WHERE id = 12")
+
+ checkOptimizationResult(df, "SELECT LOG(val) as \"LOG(E(), val)\" FROM numbers " +
+ "WHERE id IS NOT NULL AND id = 12")
+
+ val data = Tuple1(2.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("LOG10") {
+ val df = igniteSession.sql("SELECT LOG10(val) FROM numbers WHERE id = 11")
+
+ checkOptimizationResult(df, "SELECT LOG10(val) FROM numbers WHERE id IS NOT NULL AND id = 11")
+
+ val data = Tuple1(2.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("DEGREES") {
+ val df = igniteSession.sql("SELECT DEGREES(val) FROM numbers WHERE id = 13")
+
+ checkOptimizationResult(df, "SELECT DEGREES(val) FROM numbers WHERE id IS NOT NULL AND id = 13")
+
+ val data = Tuple1(180.0)
+
+ checkQueryData(df, data)
+ }
+
+ it("RADIANS") {
+ val df = igniteSession.sql("SELECT RADIANS(val) FROM numbers WHERE id = 14")
+
+ checkOptimizationResult(df, "SELECT RADIANS(val) FROM numbers WHERE id IS NOT NULL AND id = 14")
+
+ val data = Tuple1(Math.PI)
+
+ checkQueryData(df, data)
+ }
+
+ it("BITAND") {
+ val df = igniteSession.sql("SELECT int_val&1 FROM numbers WHERE id = 15")
+
+ checkOptimizationResult(df, "SELECT BITAND(int_val, 1) as \"(int_val & CAST(1 AS BIGINT))\" FROM numbers " +
+ "WHERE id IS NOT NULL AND id = 15")
+
+ val data = Tuple1(1)
+
+ checkQueryData(df, data)
+ }
+
+ it("BITOR") {
+ val df = igniteSession.sql("SELECT int_val|1 FROM numbers WHERE id = 16")
+
+ checkOptimizationResult(df, "SELECT BITOR(int_val, 1) as \"(int_val | CAST(1 AS BIGINt))\" FROM numbers " +
+ "WHERE id IS NOT NULL AND id = 16")
+
+ val data = Tuple1(3)
+
+ checkQueryData(df, data)
+ }
+
+ it("BITXOR") {
+ val df = igniteSession.sql("SELECT int_val^1 FROM numbers WHERE id = 17")
+
+ checkOptimizationResult(df, "SELECT BITXOR(int_val, 1) AS \"(int_val ^ CAST(1 AS BIGINT))\" FROM numbers " +
+ "WHERE id IS NOT NULL AND id = 17")
+
+ val data = Tuple1(2)
+
+ checkQueryData(df, data)
+ }
+
+ it("RAND") {
+ val df = igniteSession.sql("SELECT id, RAND(1) FROM numbers WHERE id = 17")
+
+ checkOptimizationResult(df, "SELECT id, RAND(1) FROM numbers WHERE id IS NOT NULL AND id = 17")
+
+ val data = df.rdd.collect
+
+ assert(data(0).getAs[JLong]("id") == 17L)
+ assert(data(0).getAs[JDouble]("rand(1)") != null)
+ }
+ }
+
+ def createNumberTable(client: Ignite, cacheName: String): Unit = {
+ val cache = client.cache(cacheName)
+
+ cache.query(new SqlFieldsQuery(
+ """
+ | CREATE TABLE numbers (
+ | id LONG,
+ | val DOUBLE,
+ | int_val LONG,
+ | PRIMARY KEY (id)) WITH "backups=1"
+ """.stripMargin)).getAll
+
+ var qry = new SqlFieldsQuery("INSERT INTO numbers (id, val) values (?, ?)")
+
+ cache.query(qry.setArgs(1L.asInstanceOf[JLong], .0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(2L.asInstanceOf[JLong], .5.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(3L.asInstanceOf[JLong], 1.0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(4L.asInstanceOf[JLong], 2.0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(5L.asInstanceOf[JLong], 4.0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(6L.asInstanceOf[JLong], -0.5.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(7L.asInstanceOf[JLong], -1.0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(8L.asInstanceOf[JLong], 42.0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(9L.asInstanceOf[JLong], .51.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(10L.asInstanceOf[JLong], .49.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(11L.asInstanceOf[JLong], 100.0.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(12L.asInstanceOf[JLong], (Math.E*Math.E).asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(13L.asInstanceOf[JLong], Math.PI.asInstanceOf[JDouble])).getAll
+ cache.query(qry.setArgs(14L.asInstanceOf[JLong], 180.0.asInstanceOf[JDouble])).getAll
+
+ qry = new SqlFieldsQuery("INSERT INTO numbers (id, int_val) values (?, ?)")
+
+ cache.query(qry.setArgs(15L.asInstanceOf[JLong], 1L.asInstanceOf[JLong])).getAll
+ cache.query(qry.setArgs(16L.asInstanceOf[JLong], 2L.asInstanceOf[JLong])).getAll
+ cache.query(qry.setArgs(17L.asInstanceOf[JLong], 3L.asInstanceOf[JLong])).getAll
+ }
+
+ override protected def beforeAll(): Unit = {
+ super.beforeAll()
+
+ createNumberTable(client, DEFAULT_CACHE)
+
+ val configProvider = enclose(null) (x ⇒ () ⇒ {
+ val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+ cfg.setClientMode(true)
+
+ cfg.setIgniteInstanceName("client-2")
+
+ cfg
+ })
+
+ igniteSession = IgniteSparkSession.builder()
+ .config(spark.sparkContext.getConf)
+ .igniteConfigProvider(configProvider)
+ .getOrCreate()
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
new file mode 100644
index 0000000..00075f4
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expdfs or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.spark.sql.ignite.IgniteSparkSession
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.internal.util.IgniteUtils.resolveIgnitePath
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+
+/**
+ */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationSpec extends AbstractDataFrameSpec {
+ var igniteSession: IgniteSparkSession = _
+
+ describe("Optimized queries") {
+ it("SELECT name as city_name FROM city") {
+ val df = igniteSession.sql("SELECT name as city_name FROM city")
+
+ checkOptimizationResult(df, "SELECT name as city_name FROM city")
+ }
+
+ it("SELECT count(*) as city_count FROM city") {
+ val df = igniteSession.sql("SELECT count(1) as city_count FROM city")
+
+ checkOptimizationResult(df, "SELECT count(1) as city_count FROM city")
+ }
+
+ it("SELECT count(*), city_id FROM person p GROUP BY city_id") {
+ val df = igniteSession.sql("SELECT city_id, count(*) FROM person GROUP BY city_id")
+
+ checkOptimizationResult(df, "SELECT city_id, count(1) FROM person GROUP BY city_id")
+
+ val data = (
+ (1, 1),
+ (2, 3),
+ (3, 1)
+ )
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT id, name FROM person WHERE id > 3 ORDER BY id") {
+ val df = igniteSession.sql("SELECT id, name FROM person WHERE id > 3 ORDER BY id")
+
+ checkOptimizationResult(df, "SELECT id, name FROM person WHERE id IS NOT NULL AND id > 3 ORDER BY id")
+
+ val data = (
+ (4, "Richard Miles"),
+ (5, null))
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT id, name FROM person WHERE id > 3 ORDER BY id DESC") {
+ val df = igniteSession.sql("SELECT id, name FROM person WHERE id > 3 ORDER BY id DESC")
+
+ checkOptimizationResult(df, "SELECT id, name FROM person WHERE id IS NOT NULL AND id > 3 ORDER BY id DESC")
+
+ val data = (
+ (5, null),
+ (4, "Richard Miles"))
+
+ checkQueryData(df, data, -_.getAs[Long]("id"))
+ }
+
+ it("SELECT id, test_reverse(name) FROM city ORDER BY id") {
+ igniteSession.udf.register("test_reverse", (str: String) ⇒ str.reverse)
+
+ val df = igniteSession.sql("SELECT id, test_reverse(name) FROM city ORDER BY id")
+
+ checkOptimizationResult(df, "SELECT name, id FROM city")
+
+ val data = (
+ (1, "Forest Hill".reverse),
+ (2, "Denver".reverse),
+ (3, "St. Petersburg".reverse),
+ (4, "St. Petersburg".reverse))
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT count(*), city_id FROM person p GROUP BY city_id HAVING count(*) > 1") {
+ val df = igniteSession.sql("SELECT city_id, count(*) FROM person p GROUP BY city_id HAVING count(*) > 1")
+
+ checkOptimizationResult(df, "SELECT city_id, count(1) FROM person GROUP BY city_id HAVING count(1) > 1")
+
+ val data = Tuple1(
+ (2, 3))
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT id FROM city HAVING id > 1") {
+ val df = igniteSession.sql("SELECT id FROM city HAVING id > 1")
+
+ checkOptimizationResult(df, "SELECT id FROM city WHERE id IS NOT NULL AND id > 1")
+
+ val data = (2, 3, 4)
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT DISTINCT name FROM city ORDER BY name") {
+ val df = igniteSession.sql("SELECT DISTINCT name FROM city ORDER BY name")
+
+ checkOptimizationResult(df, "SELECT name FROM city GROUP BY name ORDER BY name")
+
+ val data = ("Denver", "Forest Hill", "St. Petersburg")
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT id, name FROM city ORDER BY id, name") {
+ val df = igniteSession.sql("SELECT id, name FROM city ORDER BY id, name")
+
+ checkOptimizationResult(df, "SELECT id, name FROM city ORDER BY id, name")
+
+ val data = (
+ (1, "Forest Hill"),
+ (2, "Denver"),
+ (3, "St. Petersburg"),
+ (4, "St. Petersburg"))
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT id, name FROM city WHERE id > 1 ORDER BY id") {
+ val df = igniteSession.sql("SELECT id, name FROM city WHERE id > 1 ORDER BY id")
+
+ checkOptimizationResult(df, "SELECT id, name FROM city WHERE id IS NOT NULL and id > 1 ORDER BY id")
+
+ val data = (
+ (2, "Denver"),
+ (3, "St. Petersburg"),
+ (4, "St. Petersburg"))
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT count(*) FROM city") {
+ val df = igniteSession.sql("SELECT count(*) FROM city")
+
+ checkOptimizationResult(df, "SELECT count(1) FROM city")
+
+ val data = Tuple1(4)
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT count(DISTINCT name) FROM city") {
+ val df = igniteSession.sql("SELECT count(DISTINCT name) FROM city")
+
+ checkOptimizationResult(df, "SELECT count(DISTINCT name) FROM city")
+
+ val data = Tuple1(3)
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT id FROM city LIMIT 2") {
+ val df = igniteSession.sql("SELECT id FROM city LIMIT 2")
+
+ checkOptimizationResult(df, "SELECT id FROM city LIMIT 2")
+
+ val data = (1, 2)
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT CAST(id AS STRING) FROM city") {
+ val df = igniteSession.sql("SELECT CAST(id AS STRING) FROM city")
+
+ checkOptimizationResult(df, "SELECT CAST(id AS varchar) as id FROM city")
+
+ val data = ("1", "2", "3", "4")
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT SQRT(id) FROM city WHERE id = 4 OR id = 1") {
+ val df = igniteSession.sql("SELECT SQRT(id) FROM city WHERE id = 4 OR id = 1")
+
+ checkOptimizationResult(df,
+ "SELECT SQRT(cast(id as double)) FROM city WHERE id = 4 OR id = 1")
+
+ val data = (1, 2)
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT CONCAT(id, \" - this is ID\") FROM city") {
+ val df = igniteSession.sql("SELECT CONCAT(id, \" - this is ID\") FROM city")
+
+ checkOptimizationResult(df,
+ "SELECT CONCAT(cast(id AS VARCHAR), ' - this is ID') as \"CONCAT(cast(id AS STRING), - this is ID)\" " +
+ "FROM city")
+
+ val data = (
+ "1 - this is ID",
+ "2 - this is ID",
+ "3 - this is ID",
+ "4 - this is ID")
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT id FROM city WHERE CONCAT(id, \" - this is ID\") = \"1 - this is ID\"") {
+ val df = igniteSession.sql("SELECT id FROM city WHERE CONCAT(id, \" - this is ID\") = \"1 - this is ID\"")
+
+ checkOptimizationResult(df,
+ "SELECT id FROM city WHERE CONCAT(CAST(id AS VARCHAR), ' - this is ID') = '1 - this is ID'")
+
+ val data = Tuple1(1)
+
+ checkQueryData(df, data)
+ }
+ }
+
+ describe("Not Optimized Queries") {
+ it("SELECT id, name FROM json_cities") {
+ val citiesDataFrame = igniteSession.read.json(
+ resolveIgnitePath("modules/spark/src/test/resources/cities.json").getAbsolutePath)
+
+ citiesDataFrame.createOrReplaceTempView("JSON_CITIES")
+
+ val df = igniteSession.sql("SELECT id, name FROM json_cities")
+
+ val data = (
+ (1, "Forest Hill"),
+ (2, "Denver"),
+ (3, "St. Petersburg"))
+
+ checkQueryData(df, data)
+ }
+
+ it("SELECT id, test_reverse(name) tr FROM city WHERE test_reverse(name) = 'revneD' ORDER BY id") {
+ val df = igniteSession.sql("SELECT id, test_reverse(name) tr " +
+ "FROM city WHERE test_reverse(name) = 'revneD' ORDER BY id")
+
+ checkOptimizationResult(df)
+ }
+
+ it("SELECT id, test_reverse(name) tr FROM city WHERE test_reverse(name) = 'revneD' and id > 0 ORDER BY id") {
+ val df = igniteSession.sql("SELECT id, test_reverse(name) tr " +
+ "FROM city WHERE test_reverse(name) = 'revneD' and id > 0 ORDER BY id")
+
+ checkOptimizationResult(df)
+ }
+
+ it("SELECT id, test_reverse(name) tr FROM city ORDER BY tr") {
+ val df = igniteSession.sql("SELECT id, test_reverse(name) tr FROM city ORDER BY tr")
+
+ checkOptimizationResult(df)
+ }
+
+ it("SELECT count(*), test_reverse(name) tr FROM city GROUP BY test_reverse(name)") {
+ val df = igniteSession.sql("SELECT count(*), test_reverse(name) tr FROM city GROUP BY test_reverse(name)")
+
+ checkOptimizationResult(df)
+ }
+ }
+
+ override protected def beforeAll(): Unit = {
+ super.beforeAll()
+
+ createPersonTable(client, DEFAULT_CACHE)
+
+ createCityTable(client, DEFAULT_CACHE)
+
+ val configProvider = enclose(null) (x ⇒ () ⇒ {
+ val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+ cfg.setClientMode(true)
+
+ cfg.setIgniteInstanceName("client-2")
+
+ cfg
+ })
+
+ igniteSession = IgniteSparkSession.builder()
+ .config(spark.sparkContext.getConf)
+ .igniteConfigProvider(configProvider)
+ .getOrCreate()
+
+ igniteSession.udf.register("test_reverse", (str: String) ⇒ str.reverse)
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationStringFuncSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationStringFuncSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationStringFuncSpec.scala
new file mode 100644
index 0000000..db106f2
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationStringFuncSpec.scala
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.Ignite
+import org.apache.ignite.cache.query.SqlFieldsQuery
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+import org.apache.spark.sql.ignite.IgniteSparkSession
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import java.lang.{Long ⇒ JLong}
+
+/**
+ * === Doesn't supported by Spark ===
+ * CHAR
+ * DIFFERENCE
+ * HEXTORAW
+ * RAWTOHEX
+ * REGEXP_LIKE
+ * SOUNDEX
+ * STRINGDECODE
+ * STRINGENCODE
+ * STRINGTOUTF8
+ * UTF8TOSTRING
+ * XMLATTR
+ * XMLNODE
+ * XMLCOMMENT
+ * XMLCDATA
+ * XMLSTARTDOC
+ * XMLTEXT
+ * TO_CHAR - The function that can format a timestamp, a number, or text.
+ * ====== This functions in spark master but not in release =====
+ * LEFT
+ * RIGHT
+ * INSERT
+ * REPLACE
+ */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationStringFuncSpec extends AbstractDataFrameSpec {
+ var igniteSession: IgniteSparkSession = _
+
+ describe("Supported optimized string functions") {
+ it("LENGTH") {
+ val df = igniteSession.sql("SELECT LENGTH(str) FROM strings WHERE id <= 3")
+
+ checkOptimizationResult(df, "SELECT CAST(LENGTH(str) AS INTEGER) as \"length(str)\" FROM strings " +
+ "WHERE id is not null AND id <= 3")
+
+ val data = (3, 3, 6)
+
+ checkQueryData(df, data)
+ }
+
+ it("RTRIM") {
+ val df = igniteSession.sql("SELECT RTRIM(str) FROM strings WHERE id = 3")
+
+ checkOptimizationResult(df, "SELECT RTRIM(str) FROM strings WHERE id is not null AND id = 3")
+
+ val data = Tuple1("AAA")
+
+ checkQueryData(df, data)
+ }
+
+ it("LTRIM") {
+ val df = igniteSession.sql("SELECT LTRIM(str) FROM strings WHERE id = 4")
+
+ checkOptimizationResult(df, "SELECT LTRIM(str) FROM strings WHERE id is not null AND id = 4")
+
+ val data = Tuple1("AAA")
+
+ checkQueryData(df, data)
+ }
+
+ it("TRIM") {
+ val df = igniteSession.sql("SELECT TRIM(str) FROM strings WHERE id = 5")
+
+ checkOptimizationResult(df, "SELECT TRIM(str) FROM strings WHERE id is not null AND id = 5")
+
+ val data = Tuple1("AAA")
+
+ checkQueryData(df, data)
+ }
+
+ it("LOWER") {
+ val df = igniteSession.sql("SELECT LOWER(str) FROM strings WHERE id = 2")
+
+ checkOptimizationResult(df, "SELECT LOWER(str) FROM strings WHERE id is not null AND id = 2")
+
+ val data = Tuple1("aaa")
+
+ checkQueryData(df, data)
+ }
+
+ it("UPPER") {
+ val df = igniteSession.sql("SELECT UPPER(str) FROM strings WHERE id = 1")
+
+ checkOptimizationResult(df, "SELECT UPPER(str) FROM strings WHERE id is not null AND id = 1")
+
+ val data = Tuple1("AAA")
+
+ checkQueryData(df, data)
+ }
+
+ it("LOWER(RTRIM)") {
+ val df = igniteSession.sql("SELECT LOWER(RTRIM(str)) FROM strings WHERE id = 3")
+
+ checkOptimizationResult(df, "SELECT LOWER(RTRIM(str)) FROM strings WHERE id is not null AND id = 3")
+
+ val data = Tuple1("aaa")
+
+ checkQueryData(df, data)
+ }
+
+ it("LOCATE") {
+ val df = igniteSession.sql("SELECT LOCATE('D', str) FROM strings WHERE id = 6")
+
+ checkOptimizationResult(df, "SELECT LOCATE('D', str, 1) FROM strings WHERE id is not null AND id = 6")
+
+ val data = Tuple1(4)
+
+ checkQueryData(df, data)
+ }
+
+ it("LOCATE - 2") {
+ val df = igniteSession.sql("SELECT LOCATE('A', str) FROM strings WHERE id = 6")
+
+ checkOptimizationResult(df, "SELECT LOCATE('A', str, 1) FROM strings WHERE id is not null AND id = 6")
+
+ val data = Tuple1(1)
+
+ checkQueryData(df, data)
+ }
+
+ it("POSITION") {
+ val df = igniteSession.sql("SELECT instr(str, 'BCD') FROM strings WHERE id = 6")
+
+ checkOptimizationResult(df, "SELECT POSITION('BCD', str) as \"instr(str, BCD)\" FROM strings " +
+ "WHERE id is not null AND id = 6")
+
+ val data = Tuple1(2)
+
+ checkQueryData(df, data)
+ }
+
+ it("CONCAT") {
+ val df = igniteSession.sql("SELECT concat(str, 'XXX') FROM strings WHERE id = 6")
+
+ checkOptimizationResult(df, "SELECT concat(str, 'XXX') FROM strings WHERE id is not null AND id = 6")
+
+ val data = Tuple1("ABCDEFXXX")
+
+ checkQueryData(df, data)
+ }
+
+ it("RPAD") {
+ val df = igniteSession.sql("SELECT RPAD(str, 10, 'X') FROM strings WHERE id = 6")
+
+ checkOptimizationResult(df, "SELECT RPAD(str, 10, 'X') FROM strings WHERE id is not null AND id = 6")
+
+ val data = Tuple1("ABCDEFXXXX")
+
+ checkQueryData(df, data)
+ }
+
+ it("LPAD") {
+ val df = igniteSession.sql("SELECT LPAD(str, 10, 'X') FROM strings WHERE id = 6")
+
+ checkOptimizationResult(df, "SELECT LPAD(str, 10, 'X') FROM strings WHERE id is not null AND id = 6")
+
+ val data = Tuple1("XXXXABCDEF")
+
+ checkQueryData(df, data)
+ }
+
+ it("REPEAT") {
+ val df = igniteSession.sql("SELECT REPEAT(str, 2) FROM strings WHERE id = 6")
+
+ checkOptimizationResult(df, "SELECT REPEAT(str, 2) FROM strings WHERE id is not null AND id = 6")
+
+ val data = Tuple1("ABCDEFABCDEF")
+
+ checkQueryData(df, data)
+ }
+
+ it("SUBSTRING") {
+ val df = igniteSession.sql("SELECT SUBSTRING(str, 4, 3) FROM strings WHERE id = 6")
+
+ checkOptimizationResult(df, "SELECT SUBSTR(str, 4, 3) as \"SUBSTRING(str, 4, 3)\" FROM strings " +
+ "WHERE id is not null AND id = 6")
+
+ val data = Tuple1("DEF")
+
+ checkQueryData(df, data)
+ }
+
+ it("SPACE") {
+ val df = igniteSession.sql("SELECT SPACE(LENGTH(str)) FROM strings WHERE id = 1")
+
+ checkOptimizationResult(df, "SELECT SPACE(CAST(LENGTH(str) AS INTEGER)) as \"SPACE(LENGTH(str))\" " +
+ "FROM strings WHERE id is not null AND id = 1")
+
+ val data = Tuple1(" ")
+
+ checkQueryData(df, data)
+ }
+
+ it("ASCII") {
+ val df = igniteSession.sql("SELECT ASCII(str) FROM strings WHERE id = 7")
+
+ checkOptimizationResult(df, "SELECT ASCII(str) FROM strings WHERE id is not null AND id = 7")
+
+ val data = Tuple1(50)
+
+ checkQueryData(df, data)
+ }
+
+ it("REGEXP_REPLACE") {
+ val df = igniteSession.sql("SELECT REGEXP_REPLACE(str, '(\\\\d+)', 'num') FROM strings WHERE id = 7")
+
+ checkOptimizationResult(df, "SELECT REGEXP_REPLACE(str, '(\\d+)', 'num') FROM strings " +
+ "WHERE id is not null AND id = 7")
+
+ val data = Tuple1("num")
+
+ checkQueryData(df, data)
+ }
+
+ it("CONCAT_WS") {
+ val df = igniteSession.sql("SELECT id, CONCAT_WS(', ', str, 'after') FROM strings " +
+ "WHERE id >= 7 AND id <= 8")
+
+ checkOptimizationResult(df, "SELECT id, CONCAT_WS(', ', str, 'after') FROM strings " +
+ "WHERE id is not null AND id >= 7 AND id <= 8")
+
+ val data = (
+ (7, "222, after"),
+ (8, "after"))
+
+ checkQueryData(df, data)
+ }
+
+ it("TRANSLATE") {
+ val df = igniteSession.sql("SELECT id, TRANSLATE(str, 'DEF', 'ABC') FROM strings WHERE id = 6")
+
+ checkOptimizationResult(df, "SELECT id, TRANSLATE(str, 'DEF', 'ABC') FROM strings " +
+ "WHERE id is not null AND id = 6")
+
+ val data = Tuple1((6, "ABCABC"))
+
+ checkQueryData(df, data)
+ }
+ }
+
+ def createStringTable(client: Ignite, cacheName: String): Unit = {
+ val cache = client.cache(cacheName)
+
+ cache.query(new SqlFieldsQuery(
+ """
+ | CREATE TABLE strings (
+ | id LONG,
+ | str VARCHAR,
+ | PRIMARY KEY (id)) WITH "backups=1"
+ """.stripMargin)).getAll
+
+ val qry = new SqlFieldsQuery("INSERT INTO strings (id, str) values (?, ?)")
+
+ cache.query(qry.setArgs(1L.asInstanceOf[JLong], "aaa")).getAll
+ cache.query(qry.setArgs(2L.asInstanceOf[JLong], "AAA")).getAll
+ cache.query(qry.setArgs(3L.asInstanceOf[JLong], "AAA ")).getAll
+ cache.query(qry.setArgs(4L.asInstanceOf[JLong], " AAA")).getAll
+ cache.query(qry.setArgs(5L.asInstanceOf[JLong], " AAA ")).getAll
+ cache.query(qry.setArgs(6L.asInstanceOf[JLong], "ABCDEF")).getAll
+ cache.query(qry.setArgs(7L.asInstanceOf[JLong], "222")).getAll
+ cache.query(qry.setArgs(8L.asInstanceOf[JLong], null)).getAll
+ }
+
+ override protected def beforeAll(): Unit = {
+ super.beforeAll()
+
+ createStringTable(client, DEFAULT_CACHE)
+
+ val configProvider = enclose(null) (x ⇒ () ⇒ {
+ val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+ cfg.setClientMode(true)
+
+ cfg.setIgniteInstanceName("client-2")
+
+ cfg
+ })
+
+ igniteSession = IgniteSparkSession.builder()
+ .config(spark.sparkContext.getConf)
+ .igniteConfigProvider(configProvider)
+ .getOrCreate()
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a4c48b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSystemFuncSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSystemFuncSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSystemFuncSpec.scala
new file mode 100644
index 0000000..282a45f
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSystemFuncSpec.scala
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.Ignite
+import org.apache.ignite.cache.query.SqlFieldsQuery
+import org.apache.spark.sql.ignite.IgniteSparkSession
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import java.lang.{Double ⇒ JDouble, Long ⇒ JLong}
+
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+
+/**
+ */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationSystemFuncSpec extends AbstractDataFrameSpec {
+ var igniteSession: IgniteSparkSession = _
+
+ describe("Supported optimized system functions") {
+ it("COALESCE") {
+ val df = igniteSession.sql("SELECT COALESCE(int_val1, int_val2) FROM numbers WHERE id IN (1, 2, 3)")
+
+ checkOptimizationResult(df, "SELECT COALESCE(int_val1, int_val2) FROM numbers WHERE id IN (1, 2, 3)")
+
+ val data = (1, 2, 3)
+
+ checkQueryData(df, data)
+ }
+
+ it("GREATEST") {
+ val df = igniteSession.sql("SELECT GREATEST(int_val1, int_val2) FROM numbers WHERE id IN (4, 5)")
+
+ checkOptimizationResult(df, "SELECT GREATEST(int_val1, int_val2) FROM numbers WHERE id IN (4, 5)")
+
+ val data = (4, 6)
+
+ checkQueryData(df, data)
+ }
+
+ it("LEAST") {
+ val df = igniteSession.sql("SELECT LEAST(int_val1, int_val2) FROM numbers WHERE id IN (4, 5)")
+
+ checkOptimizationResult(df, "SELECT LEAST(int_val1, int_val2) FROM numbers WHERE id IN (4, 5)")
+
+ val data = (3, 5)
+
+ checkQueryData(df, data)
+ }
+
+ it("IFNULL") {
+ val df = igniteSession.sql("SELECT IFNULL(int_val1, int_val2) FROM numbers WHERE id IN (1, 2, 3)")
+
+ checkOptimizationResult(df, "SELECT COALESCE(int_val1, int_val2) as \"ifnull(numbers.`int_val1`, numbers.`int_val2`)\" FROM numbers WHERE id IN (1, 2, 3)")
+
+ val data = (1, 2, 3)
+
+ checkQueryData(df, data)
+ }
+
+ it("NULLIF") {
+ val df = igniteSession.sql("SELECT id, NULLIF(int_val1, int_val2) FROM numbers WHERE id IN (6, 7)")
+
+ checkOptimizationResult(df)
+
+ val data = (
+ (6, null),
+ (7, 8))
+
+ checkQueryData(df, data)
+ }
+
+ it("NVL2") {
+ val df = igniteSession.sql("SELECT id, NVL2(int_val1, 'not null', 'null') FROM numbers WHERE id IN (1, 2, 3)")
+
+ checkOptimizationResult(df)
+
+ val data = (
+ (1, "not null"),
+ (2, "null"),
+ (3, "not null"))
+
+ checkQueryData(df, data)
+ }
+ }
+
+ def createNumberTable(client: Ignite, cacheName: String): Unit = {
+ val cache = client.cache(cacheName)
+
+ cache.query(new SqlFieldsQuery(
+ """
+ | CREATE TABLE numbers (
+ | id LONG,
+ | int_val1 LONG,
+ | int_val2 LONG,
+ | PRIMARY KEY (id)) WITH "backups=1"
+ """.stripMargin)).getAll
+
+
+ val qry = new SqlFieldsQuery("INSERT INTO numbers (id, int_val1, int_val2) values (?, ?, ?)")
+
+ cache.query(qry.setArgs(1L.asInstanceOf[JLong], 1L.asInstanceOf[JLong], null)).getAll
+ cache.query(qry.setArgs(2L.asInstanceOf[JLong], null, 2L.asInstanceOf[JLong])).getAll
+ cache.query(qry.setArgs(3L.asInstanceOf[JLong], 3L.asInstanceOf[JLong], null)).getAll
+ cache.query(qry.setArgs(4L.asInstanceOf[JLong], 3L.asInstanceOf[JLong], 4L.asInstanceOf[JLong])).getAll
+ cache.query(qry.setArgs(5L.asInstanceOf[JLong], 6L.asInstanceOf[JLong], 5L.asInstanceOf[JLong])).getAll
+ cache.query(qry.setArgs(6L.asInstanceOf[JLong], 7L.asInstanceOf[JLong], 7L.asInstanceOf[JLong])).getAll
+ cache.query(qry.setArgs(7L.asInstanceOf[JLong], 8L.asInstanceOf[JLong], 9L.asInstanceOf[JLong])).getAll
+ }
+
+ override protected def beforeAll(): Unit = {
+ super.beforeAll()
+
+ createNumberTable(client, DEFAULT_CACHE)
+
+ val configProvider = enclose(null) (x ⇒ () ⇒ {
+ val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+ cfg.setClientMode(true)
+
+ cfg.setIgniteInstanceName("client-2")
+
+ cfg
+ })
+
+ igniteSession = IgniteSparkSession.builder()
+ .config(spark.sparkContext.getConf)
+ .igniteConfigProvider(configProvider)
+ .getOrCreate()
+ }
+}