You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by as...@apache.org on 2020/05/22 05:29:12 UTC
[ignite] branch master updated: IGNITE-13003 Improve data loss
handling. - Fixes #7800.
This is an automated email from the ASF dual-hosted git repository.
ascherbakov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 9e214ee IGNITE-13003 Improve data loss handling. - Fixes #7800.
9e214ee is described below
commit 9e214ee8ce7b87780d904111d359c74e90613e3f
Author: Alexey Scherbakov <al...@gmail.com>
AuthorDate: Fri May 22 08:22:36 2020 +0300
IGNITE-13003 Improve data loss handling. - Fixes #7800.
Signed-off-by: Aleksei Scherbakov <as...@apache.org>
---
.../src/main/java/org/apache/ignite/Ignite.java | 3 +
.../main/java/org/apache/ignite/IgniteCache.java | 2 +-
.../java/org/apache/ignite/cache/CacheMetrics.java | 6 +
.../apache/ignite/cache/PartitionLossPolicy.java | 15 +-
.../ignite/configuration/IgniteConfiguration.java | 2 +-
.../internal/events/DiscoveryCustomEvent.java | 8 +
.../cache/CacheAffinitySharedManager.java | 61 +-
.../processors/cache/CacheGroupContext.java | 17 -
.../processors/cache/DynamicCacheChangeBatch.java | 2 +-
.../cache/GridCachePartitionExchangeManager.java | 14 +-
.../processors/cache/GridCacheProcessor.java | 20 +-
.../dht/ClientCacheDhtTopologyFuture.java | 5 +-
.../cache/distributed/dht/GridDhtCacheAdapter.java | 6 +-
.../cache/distributed/dht/GridDhtGetFuture.java | 13 -
.../distributed/dht/GridDhtGetSingleFuture.java | 13 -
.../distributed/dht/GridDhtTopologyFuture.java | 5 +-
.../dht/GridDhtTopologyFutureAdapter.java | 174 +---
.../distributed/dht/GridDhtTxPrepareFuture.java | 13 +-
.../distributed/dht/GridPartitionedGetFuture.java | 72 +-
.../dht/GridPartitionedSingleGetFuture.java | 50 +-
.../distributed/dht/atomic/GridDhtAtomicCache.java | 6 +-
.../dht/colocated/GridDhtColocatedCache.java | 3 +-
.../dht/preloader/GridDhtPartitionDemander.java | 9 +-
.../preloader/GridDhtPartitionsExchangeFuture.java | 147 +--
.../preloader/GridDhtPartitionsFullMessage.java | 87 +-
.../dht/preloader/GridDhtPreloader.java | 32 +-
.../dht/topology/GridClientPartitionTopology.java | 122 ++-
.../dht/topology/GridDhtLocalPartition.java | 33 +-
.../dht/topology/GridDhtPartitionTopology.java | 22 +-
.../dht/topology/GridDhtPartitionTopologyImpl.java | 285 ++++--
.../dht/topology/PartitionsEvictManager.java | 11 +-
.../GridCacheDatabaseSharedManager.java | 11 +-
.../cache/query/GridCacheQueryAdapter.java | 35 +-
.../cache/query/GridCacheQueryManager.java | 28 +-
.../cache/transactions/IgniteTxHandler.java | 19 +-
.../cache/transactions/IgniteTxStateImpl.java | 20 +-
.../schema/SchemaIndexCachePartitionWorker.java | 3 +-
.../ClientCreateCacheGroupOnJoinNodeMapsTest.java | 89 ++
.../RebalanceAfterResettingLostPartitionTest.java | 177 ++++
.../ignite/cache/ResetLostPartitionTest.java | 37 +-
.../ignite/failure/SystemWorkersBlockingTest.java | 3 +
.../internal/TestDelayingCommunicationSpi.java | 3 +-
.../internal/encryption/MasterKeyChangeTest.java | 7 +
.../processors/cache/CacheMetricsManageTest.java | 3 +
.../cache/CacheValidatorMetricsTest.java | 16 +-
...heWithDifferentDataRegionConfigurationTest.java | 2 -
...eCachePartitionMapUpdateSafeLossPolicyTest.java | 79 ++
.../cache/IgniteCachePartitionMapUpdateTest.java | 70 +-
.../cache/IgniteClusterActivateDeactivateTest.java | 2 +
.../cache/distributed/CacheExchangeMergeTest.java | 95 +-
.../CachePartitionLossDetectionOnNodeLeftTest.java | 115 ---
.../CachePartitionLossWithPersistenceTest.java | 264 +++++
.../CachePartitionLossWithRestartsTest.java | 223 ++++
...CachePartitionLostAfterSupplierHasLeftTest.java | 409 ++++++++
.../CachePartitionLostWhileClearingTest.java | 168 ---
.../CacheResultIsNotNullOnPartitionLossTest.java | 4 +
.../GridCachePartitionNotLoadedEventSelfTest.java | 41 +-
...niteCacheGroupsPartitionLossPolicySelfTest.java | 316 +-----
.../IgniteCachePartitionLossPolicySelfTest.java | 1072 ++++++--------------
.../distributed/IgniteCachePrimarySyncTest.java | 3 +
.../cache/mvcc/CacheMvccTxFailoverTest.java | 10 +-
...gniteLostPartitionsOnLeaveBaselineSelfTest.java | 167 +++
.../IgnitePdsCacheRebalancingAbstractTest.java | 1 +
.../persistence/IgnitePdsCorruptedIndexTest.java | 6 +
...ocalWalModeChangeDuringRebalancingSelfTest.java | 22 +-
.../ClientAffinityAssignmentWithBaselineTest.java | 37 +-
.../IgniteAbsentEvictionNodeOutOfBaselineTest.java | 3 +
.../persistence/db/IgnitePdsCacheRestoreTest.java | 5 +-
...IgnitePdsCacheWalDisabledOnRebalancingTest.java | 15 +-
.../persistence/db/wal/WalCompactionTest.java | 3 +
...AtomicPartitionCounterStateConsistencyTest.java | 10 +-
.../TxPartitionCounterStateAbstractTest.java | 19 +
.../TxPartitionCounterStateConsistencyTest.java | 140 ++-
.../junits/common/GridCommonAbstractTest.java | 156 ++-
.../testsuites/IgniteCacheMvccTestSuite5.java | 5 +
.../testsuites/IgniteCacheMvccTestSuite6.java | 4 +
.../testsuites/IgniteCacheMvccTestSuite7.java | 2 +
.../ignite/testsuites/IgniteCacheTestSuite2.java | 2 +
.../ignite/testsuites/IgniteCacheTestSuite6.java | 4 +-
.../ignite/testsuites/IgniteCacheTestSuite7.java | 8 +-
.../ignite/testsuites/IgniteCacheTestSuite8.java | 2 +
.../query/h2/twostep/GridMapQueryExecutor.java | 4 +
.../h2/twostep/PartitionReservationManager.java | 36 +-
.../query/h2/twostep/ReducePartitionMapper.java | 25 +-
...ockPartitionOnAffinityRunAtomicCacheOpTest.java | 11 +-
...cheLockPartitionOnAffinityRunTxCacheOpTest.java | 3 +-
.../cache/IgniteDynamicSqlRestoreTest.java | 3 +
.../IndexingCachePartitionLossPolicySelfTest.java | 57 +-
...acheQueryReservationOnUnstableTopologyTest.java | 110 ++
.../db/IgniteCacheGroupsWithRestartsTest.java | 2 +
.../persistence/db/wal/IgniteWalRecoveryTest.java | 74 +-
...eLockPartitionOnAffinityRunAtomicCacheTest.java | 4 +-
...elineLockPartitionOnAffinityRunTxCacheTest.java | 4 +-
.../IgniteBinaryCacheQueryTestSuite2.java | 5 +-
.../Cache/PartitionLossTest.cs | 20 +-
95 files changed, 3401 insertions(+), 2150 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java
index b1e3b54..7332eaa 100644
--- a/modules/core/src/main/java/org/apache/ignite/Ignite.java
+++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java
@@ -676,6 +676,9 @@ public interface Ignite extends AutoCloseable {
/**
* Clears partition's lost state and moves caches to a normal mode.
+ * <p>
+ * To avoid permanent data loss for persistent caches it's recommended to return all previously failed baseline
+ * nodes to the topology before calling this method.
*/
public void resetLostPartitions(Collection<String> cacheNames);
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java
index 49c8908..d4d289e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java
@@ -132,7 +132,7 @@ public interface IgniteCache<K, V> extends javax.cache.Cache<K, V>, IgniteAsyncS
public IgniteCache<K, V> withNoRetries();
/**
- * Gets an instance of {@code IgniteCache} that will be allowed to execute cache operations (read, write)
+ * Gets an instance of {@code IgniteCache} that will be allowed to execute cache read operations
* regardless of partition loss policy.
*
* @return Cache without partition loss protection.
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java b/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
index 759af8f..1921dbb 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
@@ -691,6 +691,9 @@ public interface CacheMetrics {
/**
* Checks whether cache topology is valid for read operations.
+ * <p>
+ * Note: the method will return {@code false} if any partition was lost despite the fact others can be awailable
+ * for reading.
*
* @return {@code true} when cache topology is valid for reading.
*/
@@ -698,6 +701,9 @@ public interface CacheMetrics {
/**
* Checks whether cache topology is valid for write operations.
+ * <p>
+ * Note: the method will return {@code false} if any partition was lost despite the fact others can be awailable
+ * for writing according to configured partition loss policy.
*
* @return {@code true} when cache topology is valid for writing.
*/
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/PartitionLossPolicy.java b/modules/core/src/main/java/org/apache/ignite/cache/PartitionLossPolicy.java
index 7c9b2a7..9bc5cd0 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/PartitionLossPolicy.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/PartitionLossPolicy.java
@@ -23,9 +23,11 @@ import org.apache.ignite.IgniteCache;
import org.jetbrains.annotations.Nullable;
/**
- * Partition loss policy. Defines how cache will behave in a case when one or more partitions are lost
+ * Partition loss policy. Defines how a cache will behave in a case when one or more partitions are lost
* because of a node(s) failure.
* <p>
+ * A partition is considered <em>lost</em> if all owning nodes had left a topology.
+ * <p>
* All <code>*_SAFE</code> policies prevent a user from interaction with partial data in lost partitions until
* {@link Ignite#resetLostPartitions(Collection)} method is called. <code>*_ALL</code> policies allow working with
* partial data in lost partitions.
@@ -47,6 +49,8 @@ public enum PartitionLossPolicy {
* All writes to the cache will be failed with an exception. All reads will proceed as if all partitions
* were in a consistent state. The result of reading from a lost partition is undefined and may be different
* on different nodes in the cluster.
+ *
+ * @deprecated {@link #READ_ONLY_SAFE} is used instead.
*/
READ_ONLY_ALL,
@@ -59,13 +63,16 @@ public enum PartitionLossPolicy {
/**
* All reads and writes will proceed as if all partitions were in a consistent state. The result of reading
* from a lost partition is undefined and may be different on different nodes in the cluster.
+ *
+ * @deprecated {@link #READ_WRITE_SAFE} is used instead.
*/
READ_WRITE_ALL,
/**
- * If partition is lost, reset it's state and do not clear intermediate data. The result of reading from
- * a previously lost and not cleared partition is undefined and may be different on different nodes in the
- * cluster.
+ * If a partition was lost silently ignore it and allow any operations with a partition.
+ * Partition loss events are not fired if using this mode.
+ * For pure in-memory caches the policy will work only when baseline auto adjust is enabled with zero timeout.
+ * If persistence is enabled, the policy is always ignored. READ_WRITE_SAFE is used instead.
*/
IGNORE;
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index 9a9810f..38f33c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -2749,7 +2749,7 @@ public class IgniteConfiguration {
* Sets flag indicating whether the cluster will be active on start. This value should be the same on all
* nodes in the cluster.
* <p>
- * This flag is ignored when {@link DataStorageConfiguration} is present:
+ * This flag is ignored when {@link DataStorageConfiguration} has at least one configured persistent region:
* cluster is always inactive on start when Ignite Persistence is enabled.
*
* @param activeOnStart Active on start flag value.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java
index 44f183e..6662d02 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java
@@ -21,6 +21,7 @@ import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -117,6 +118,13 @@ public class DiscoveryCustomEvent extends DiscoveryEvent {
return snapMsg.needExchange() && snapMsg.needAssignPartitions();
}
+ if (msg instanceof DynamicCacheChangeBatch) {
+ DynamicCacheChangeBatch cacheMsg = (DynamicCacheChangeBatch) msg;
+
+ return cacheMsg.exchangeActions() != null &&
+ !cacheMsg.exchangeActions().cachesToResetLostPartitions().isEmpty();
+ }
+
return false;
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index dcd18d8..f9edf86 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -71,6 +71,7 @@ import org.apache.ignite.internal.util.GridPartitionStateMap;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.GridPlainRunnable;
import org.apache.ignite.internal.util.lang.IgniteInClosureX;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -346,8 +347,9 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
}
/**
- * Adds historically rebalancing partitions to wait group.
- * Not doing so could trigger late affinity switching before actual rebalancing will finish.
+ * Adds group partition to wait list.
+ * <p>
+ * Late affinity switch will be triggered as soons as wait list becomes empty.
*
* @param grpId Group id.
* @param part Part.
@@ -500,7 +502,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
grp.topology().updateTopologyVersion(excFut, discoCache, -1, false);
// Exchange free cache creation, just replacing client topology with dht.
- // Topology shouild be initialized before the use.
+ // Topology should be initialized before the use.
grp.topology().beforeExchange(excFut, true, false);
grpHolder = new CacheGroupAffNodeHolder(grp, grpHolder.affinity());
@@ -517,8 +519,8 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
Collections.<Integer>emptySet(),
null,
null,
- null
- );
+ null,
+ clientTop.lostPartitions());
}
assert grpHolder.affinity().lastVersion().equals(grp.affinity().lastVersion());
@@ -582,7 +584,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
-1,
false);
- grp.topology().update(topVer, partMap, null, Collections.emptySet(), null, null, null);
+ grp.topology().update(topVer, partMap, null, Collections.emptySet(), null, null, null, null);
topFut.validate(grp, discoCache.allNodes());
}
@@ -880,7 +882,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
if (notifyTopVer != null) {
final AffinityTopologyVersion topVer = notifyTopVer;
- cctx.kernalContext().closure().runLocalSafe(new Runnable() {
+ cctx.kernalContext().closure().runLocalSafe(new GridPlainRunnable() {
@Override public void run() {
onCacheGroupStopped(topVer);
}
@@ -1115,8 +1117,6 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
assert assignment != null;
- final Map<Object, List<List<ClusterNode>>> affCache = new ConcurrentHashMap<>();
-
forAllCacheGroups(new IgniteInClosureX<GridAffinityAssignmentCache>() {
@Override public void applyx(GridAffinityAssignmentCache aff) {
List<List<ClusterNode>> idealAssignment = aff.idealAssignmentRaw();
@@ -2214,14 +2214,14 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
* @param evts Discovery events processed during exchange.
* @param addedOnExchnage {@code True} if cache group was added during this exchange.
* @param grpHolder Group holder.
- * @param rebalanceInfo Rebalance information.
+ * @param rebalanceInfo Rebalance information on coordinator or null on other nodes.
* @param latePrimary If {@code true} delays primary assignment if it is not owner.
*/
private void initAffinityOnNodeJoin(
ExchangeDiscoveryEvents evts,
boolean addedOnExchnage,
CacheGroupHolder grpHolder,
- WaitRebalanceInfo rebalanceInfo,
+ @Nullable WaitRebalanceInfo rebalanceInfo,
boolean latePrimary
) {
GridAffinityAssignmentCache aff = grpHolder.affinity();
@@ -2270,8 +2270,15 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
GridDhtPartitionTopology top = grpHolder.topology(evts.discoveryCache());
- if (rebalanceInfo != null && !top.owners(p, evts.topologyVersion()).containsAll(idealAssignment.get(p)))
- rebalanceInfo.add(aff.groupId(), p, newNodes);
+ if (rebalanceInfo != null) {
+ List<ClusterNode> owners = top.owners(p, evts.topologyVersion());
+
+ // If current owners are empty no supplier can exist.
+ // A group with lost partitions never gets rebalanced so should not be added to waitInfo.
+ if (!owners.isEmpty() && !owners.containsAll(idealAssignment.get(p)) &&
+ !top.lostPartitions().contains(p))
+ rebalanceInfo.add(aff.groupId(), p, newNodes);
+ }
}
}
@@ -2294,7 +2301,8 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
int part,
ClusterNode curPrimary,
List<ClusterNode> newNodes,
- WaitRebalanceInfo rebalance) {
+ @Nullable WaitRebalanceInfo rebalance
+ ) {
assert curPrimary != null;
assert !F.isEmpty(newNodes);
assert !curPrimary.equals(newNodes.get(0));
@@ -2478,7 +2486,8 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
}
}
- if (!owners.isEmpty() && !owners.containsAll(newAssignment.get(p)))
+ // This will happen if no primary is changed but some backups still need to be rebalanced.
+ if (!owners.isEmpty() && !owners.containsAll(newNodes) && !top.lostPartitions().contains(p))
waitRebalanceInfo.add(grpHolder.groupId(), p, newNodes);
if (newNodes0 != null) {
@@ -2727,7 +2736,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
/**
* Created if cache is not started on coordinator.
*/
- private class CacheGroupNoAffOrFiltredHolder extends CacheGroupHolder {
+ private class CacheGroupNoAffOrFilteredHolder extends CacheGroupHolder {
/** */
private final GridCacheSharedContext cctx;
@@ -2737,7 +2746,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
* @param aff Affinity.
* @param initAff Current affinity.
*/
- CacheGroupNoAffOrFiltredHolder(
+ CacheGroupNoAffOrFilteredHolder(
boolean rebalanceEnabled,
GridCacheSharedContext cctx,
GridAffinityAssignmentCache aff,
@@ -2755,7 +2764,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
* @return Cache holder.
* @throws IgniteCheckedException If failed.
*/
- CacheGroupNoAffOrFiltredHolder create(
+ CacheGroupNoAffOrFilteredHolder create(
GridCacheSharedContext cctx,
CacheGroupDescriptor grpDesc,
AffinityTopologyVersion topVer
@@ -2771,7 +2780,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
* @return Cache holder.
* @throws IgniteCheckedException If failed.
*/
- CacheGroupNoAffOrFiltredHolder create(
+ CacheGroupNoAffOrFilteredHolder create(
GridCacheSharedContext cctx,
CacheGroupDescriptor grpDesc,
AffinityTopologyVersion topVer,
@@ -2804,7 +2813,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
ccfg.getCacheMode() == LOCAL
);
- return new CacheGroupNoAffOrFiltredHolder(ccfg.getRebalanceMode() != NONE, cctx, aff, initAff);
+ return new CacheGroupNoAffOrFilteredHolder(ccfg.getRebalanceMode() != NONE, cctx, aff, initAff);
}
/** {@inheritDoc} */
@@ -2818,7 +2827,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
}
}
- private CacheGroupNoAffOrFiltredHolder createHolder(
+ private CacheGroupNoAffOrFilteredHolder createHolder(
GridCacheSharedContext cctx,
CacheGroupDescriptor grpDesc,
AffinityTopologyVersion topVer,
@@ -2851,11 +2860,12 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
ccfg.getCacheMode() == LOCAL
);
- return new CacheGroupNoAffOrFiltredHolder(ccfg.getRebalanceMode() != NONE, cctx, aff, initAff);
+ return new CacheGroupNoAffOrFilteredHolder(ccfg.getRebalanceMode() != NONE, cctx, aff, initAff);
}
/**
- *
+ * Tracks rebalance state on coordinator.
+ * After all partitions are rebalanced the current affinity is switched to ideal.
*/
class WaitRebalanceInfo {
/** */
@@ -2865,7 +2875,8 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
private final Map<Integer, Set<Integer>> waitGrps = new ConcurrentHashMap<>();
/** */
- private final Map<Integer, Map<Integer, List<ClusterNode>>> assignments = new ConcurrentHashMap<>();
+ private final Map<Integer /** Group id. */, Map<Integer /** Partition id. */, List<ClusterNode>>> assignments =
+ new ConcurrentHashMap<>();
/** */
private final Map<Integer, IgniteUuid> deploymentIds = new ConcurrentHashMap<>();
@@ -2893,6 +2904,8 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
}
/**
+ * Adds a partition to wait set.
+ *
* @param grpId Group ID.
* @param part Partition.
* @param assignment New assignment.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index c1242d2..d4e9aab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -91,9 +91,6 @@ public class CacheGroupContext {
/** Node ID cache group was received from. */
private volatile UUID rcvdFrom;
- /** Flag indicating that this cache group is in a recovery mode due to partitions loss. */
- private boolean needsRecovery;
-
/** */
private volatile AffinityTopologyVersion locStartVer;
@@ -688,20 +685,6 @@ public class CacheGroupContext {
}
/**
- * @return Current cache state. Must only be modified during exchange.
- */
- public boolean needsRecovery() {
- return needsRecovery;
- }
-
- /**
- * @param needsRecovery Needs recovery flag.
- */
- public void needsRecovery(boolean needsRecovery) {
- this.needsRecovery = needsRecovery;
- }
-
- /**
* @return Topology version when group was started on local node.
*/
public AffinityTopologyVersion localStartVersion() {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
index 239ba6b..6b89424 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
@@ -105,7 +105,7 @@ public class DynamicCacheChangeBatch implements DiscoveryCustomMessage {
/**
* @return Cache updates to be executed on exchange.
*/
- ExchangeActions exchangeActions() {
+ public ExchangeActions exchangeActions() {
return exchangeActions;
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index d0bdd6e..30df974 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -227,7 +227,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
private final ConcurrentMap<AffinityTopologyVersion, AffinityReadyFuture> readyFuts =
new ConcurrentSkipListMap<>();
- /** */
+ /** Used for mapping client exchange topology versions to server exchange topology versions. */
private final ConcurrentNavigableMap<AffinityTopologyVersion, AffinityTopologyVersion> lastAffTopVers =
new ConcurrentSkipListMap<>();
@@ -984,7 +984,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
aff.partitions());
GridClientPartitionTopology old = clientTops.putIfAbsent(grpId,
- top = new GridClientPartitionTopology(cctx, discoCache, grpId, aff.partitions(), affKey));
+ top = new GridClientPartitionTopology(cctx, discoCache, grpId, aff.partitions(), affKey,
+ ccfg.getPartitionLossPolicy()));
return old != null ? old : top;
}
@@ -1480,6 +1481,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
m.addPartitionUpdateCounters(grp.groupId(),
CachePartitionFullCountersMap.toCountersMap(cntrsMap));
}
+
+ // Lost partitions can be skipped on node left or activation.
+ m.addLostPartitions(grp.groupId(), grp.topology().lostPartitions());
}
}
}
@@ -1503,6 +1507,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
if (!partSizesMap.isEmpty())
partsSizes.put(top.groupId(), partSizesMap);
+
+ m.addLostPartitions(top.groupId(), top.lostPartitions());
}
}
@@ -1875,8 +1881,8 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
msg.partsToReload(cctx.localNodeId(), grpId),
partsSizes.getOrDefault(grpId, Collections.emptyMap()),
msg.topologyVersion(),
- null
- );
+ null,
+ null);
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 326f24d..ef48b51 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -70,6 +70,7 @@ import org.apache.ignite.internal.IgniteTransactionsEx;
import org.apache.ignite.internal.binary.BinaryContext;
import org.apache.ignite.internal.binary.BinaryMarshaller;
import org.apache.ignite.internal.binary.GridBinaryMarshaller;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.cluster.DetachedClusterNode;
import org.apache.ignite.internal.managers.communication.GridIoPolicy;
import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
@@ -3807,19 +3808,30 @@ public class GridCacheProcessor extends GridProcessorAdapter {
* @param cacheNames Cache names.
* @return Future that will be completed when state is changed for all caches.
*/
- public IgniteInternalFuture<?> resetCacheState(Collection<String> cacheNames) {
+ public IgniteInternalFuture<?> resetCacheState(Collection<String> cacheNames) throws ClusterTopologyCheckedException {
sharedCtx.tm().checkEmptyTransactions(
() -> String.format(CHECK_EMPTY_TRANSACTIONS_ERROR_MSG_FORMAT, cacheNames, "resetCacheState"));
Collection<DynamicCacheChangeRequest> reqs = new ArrayList<>(cacheNames.size());
for (String cacheName : cacheNames) {
- DynamicCacheDescriptor desc = cacheDescriptor(cacheName);
+ final IgniteInternalCache<Object, Object> cache0 = internalCache(cacheName);
- if (desc == null) {
- U.warn(log, "Failed to find cache for reset lost partition request, cache does not exist: " + cacheName);
+ if (cache0 == null)
+ continue;
+
+ // Check if all lost partitions has at least one affinity owner.
+ final Collection<Integer> lostParts = cache0.lostPartitions();
+ if (lostParts.isEmpty())
continue;
+
+ for (Integer part : lostParts) {
+ final Collection<ClusterNode> owners = cache0.affinity().mapPartitionToPrimaryAndBackups(part);
+
+ if (owners.isEmpty())
+ throw new ClusterTopologyCheckedException("Cannot reset lost partitions because no baseline nodes " +
+ "are online [cache=" + cacheName + ", partition=" + part + ']');
}
DynamicCacheChangeRequest req = DynamicCacheChangeRequest.resetLostPartitions(ctx, cacheName);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java
index f468590..8fae639 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java
@@ -62,7 +62,10 @@ public class ClientCacheDhtTopologyFuture extends GridDhtTopologyFutureAdapter {
public void validate(CacheGroupContext grp, Collection<ClusterNode> topNodes) {
grpValidRes = U.newHashMap(1);
- grpValidRes.put(grp.groupId(), validateCacheGroup(grp,topNodes));
+ CacheGroupValidation valRes = validateCacheGroup(grp, topNodes);
+
+ if (!valRes.isValid() || valRes.hasLostPartitions())
+ grpValidRes.put(grp.groupId(), valRes);
}
/** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index e76b6c6..257cbb0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -76,6 +76,7 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.platform.cache.PlatformCacheEntryFilter;
import org.apache.ignite.internal.processors.security.SecurityUtils;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.GridPlainRunnable;
import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.CI2;
import org.apache.ignite.internal.util.typedef.CI3;
@@ -999,11 +1000,14 @@ public abstract class GridDhtCacheAdapter<K, V> extends GridDistributedCacheAdap
}
/**
+ * Initiates process of notifying all interested nodes that TTL was changed.
+ * Directly sends requests to primary nodes and {@link IgniteCacheExpiryPolicy#readers()}.
+ *
* @param expiryPlc Expiry policy.
*/
public void sendTtlUpdateRequest(@Nullable final IgniteCacheExpiryPolicy expiryPlc) {
if (expiryPlc != null && !F.isEmpty(expiryPlc.entries())) {
- ctx.closures().runLocalSafe(new Runnable() {
+ ctx.closures().runLocalSafe(new GridPlainRunnable() {
@SuppressWarnings({"ForLoopReplaceableByForEach"})
@Override public void run() {
Map<KeyCacheObject, GridCacheVersion> entries = expiryPlc.entries();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
index 2a12fef..a715c82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
@@ -37,7 +37,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedExceptio
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.ReaderArguments;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter.LostPolicyValidator;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
@@ -56,8 +55,6 @@ import org.apache.ignite.lang.IgniteUuid;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-import static java.util.Collections.singleton;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter.OperationType.READ;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
@@ -341,16 +338,6 @@ public final class GridDhtGetFuture<K, V> extends GridCompoundIdentityFuture<Col
if (part == null)
return false;
- if (!forceKeys && part.state() == LOST && !recovery) {
- Throwable error = LostPolicyValidator.validate(cctx, key, READ, singleton(part.id()));
-
- if (error != null) {
- onDone(null, error);
-
- return false;
- }
- }
-
if (parts == null || !F.contains(parts, part.id())) {
// By reserving, we make sure that partition won't be unloaded while processed.
if (part.reserve()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java
index 797fd66..b8a6f17 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java
@@ -35,7 +35,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedExceptio
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.ReaderArguments;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter.LostPolicyValidator;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
@@ -48,8 +47,6 @@ import org.apache.ignite.lang.IgniteUuid;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-import static java.util.Collections.singleton;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter.OperationType.READ;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
@@ -313,16 +310,6 @@ public final class GridDhtGetSingleFuture<K, V> extends GridFutureAdapter<GridCa
assert this.part == -1;
- if (!forceKeys && part.state() == LOST && !recovery) {
- Throwable error = LostPolicyValidator.validate(cctx, key, READ, singleton(part.id()));
-
- if (error != null) {
- onDone(null, error);
-
- return false;
- }
- }
-
// By reserving, we make sure that partition won't be unloaded while processed.
if (part.reserve()) {
if (forceKeys || (part.state() == OWNING || part.state() == LOST)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java
index b92a3bd..2b1ef0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.distributed.dht;
import java.util.Collection;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
@@ -77,9 +78,9 @@ public interface GridDhtTopologyFuture extends IgniteInternalFuture<AffinityTopo
* @param read {@code True} if validating read operation, {@code false} if validating write.
* @param key Key (optimization to avoid collection creation).
* @param keys Keys involved in a cache operation.
- * @return valid ot not.
+ * @return Not null exception if a cache is in invalid state.
*/
- @Nullable public Throwable validateCache(
+ @Nullable public CacheInvalidStateException validateCache(
GridCacheContext cctx,
boolean recovery,
boolean read,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
index c57aaff..decc8d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
@@ -20,7 +20,6 @@ package org.apache.ignite.internal.processors.cache.distributed.dht;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
-import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cache.PartitionLossPolicy;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.TopologyValidator;
@@ -29,14 +28,11 @@ import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.typedef.F;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_ALL;
import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_SAFE;
-import static org.apache.ignite.cache.PartitionLossPolicy.READ_WRITE_SAFE;
import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isSystemCache;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter.OperationType.WRITE;
/**
*
@@ -71,7 +67,7 @@ public abstract class GridDhtTopologyFutureAdapter extends GridFutureAdapter<Aff
}
/** {@inheritDoc} */
- @Override public final @Nullable Throwable validateCache(
+ @Override public final @Nullable CacheInvalidStateException validateCache(
GridCacheContext cctx,
boolean recovery,
boolean read,
@@ -83,29 +79,23 @@ public abstract class GridDhtTopologyFutureAdapter extends GridFutureAdapter<Aff
Throwable err = error();
if (err != null)
- return err;
+ return new CacheInvalidStateException(err);
- if (!clusterIsActive)
+ if (!clusterIsActive) {
return new CacheInvalidStateException(
- "Failed to perform cache operation (cluster is not activated): " + cctx.name());
+ "Failed to perform cache operation (cluster is not activated): " + cctx.name());
+ }
if (cctx.cache() == null)
return new CacheInvalidStateException(
"Failed to perform cache operation (cache is stopped): " + cctx.name());
- OperationType opType = read ? OperationType.READ : WRITE;
-
CacheGroupContext grp = cctx.group();
- PartitionLossPolicy lossPlc = grp.config().getPartitionLossPolicy();
-
- if (cctx.shared().readOnlyMode() && opType == WRITE && !isSystemCache(cctx.name()))
- return new IgniteClusterReadOnlyException("Failed to perform cache operation (cluster is in read-only mode)");
-
- if (grp.needsRecovery() && !recovery) {
- if (opType == WRITE && (lossPlc == READ_ONLY_SAFE || lossPlc == READ_ONLY_ALL))
- return new IgniteCheckedException(
- "Failed to write to cache (cache is moved to a read-only state): " + cctx.name());
+ if (cctx.shared().readOnlyMode() && !read && !isSystemCache(cctx.name())) {
+ return new CacheInvalidStateException(
+ new IgniteClusterReadOnlyException("Failed to perform cache operation (cluster is in " +
+ "read-only mode) [cacheGrp=" + cctx.group().name() + ", cache=" + cctx.name() + ']'));
}
CacheGroupValidation validation = grpValidRes.get(grp.groupId());
@@ -113,23 +103,41 @@ public abstract class GridDhtTopologyFutureAdapter extends GridFutureAdapter<Aff
if (validation == null)
return null;
- if (opType == WRITE && !validation.isValid()) {
- return new IgniteCheckedException("Failed to perform cache operation " +
- "(cache topology is not valid): " + cctx.name());
+ if (!read && !validation.isValid()) {
+ return new CacheInvalidStateException("Failed to perform cache operation " +
+ "(cache topology is not valid): " + cctx.name());
+ }
+
+ if (!validation.hasLostPartitions())
+ return null;
+
+ PartitionLossPolicy lossPlc = grp.config().getPartitionLossPolicy();
+
+ if (!read && (lossPlc == READ_ONLY_SAFE || lossPlc == READ_ONLY_ALL)) {
+ return new CacheInvalidStateException(
+ "Failed to write to cache (cache is moved to a read-only state): " + cctx.name());
}
- if (recovery)
+ // Reads from any partition are allowed in recovery mode.
+ if (read && recovery)
return null;
- if (validation.hasLostPartitions()) {
- if (key != null)
- return LostPolicyValidator.validate(cctx, key, opType, validation.lostPartitions());
+ if (key != null)
+ return validateKey(cctx, key, validation.lostPartitions());
+
+ if (keys != null) {
+ for (Object key0 : keys) {
+ final CacheInvalidStateException res = validateKey(cctx, key0, validation.lostPartitions());
- if (keys != null)
- return LostPolicyValidator.validate(cctx, keys, opType, validation.lostPartitions());
+ if (res != null)
+ return res;
+ }
+
+ return null;
}
- return null;
+ return new CacheInvalidStateException("Failed to perform a cache operation " +
+ "(the cache has lost partitions [cacheGrp=" + cctx.group().name() + ", cache=" + cctx.name() + ']');
}
/**
@@ -170,7 +178,7 @@ public abstract class GridDhtTopologyFutureAdapter extends GridFutureAdapter<Aff
* @return True if lost partition is present, False if not.
*/
public boolean hasLostPartitions() {
- return !F.isEmpty(lostParts);
+ return !lostParts.isEmpty();
}
/**
@@ -182,100 +190,20 @@ public abstract class GridDhtTopologyFutureAdapter extends GridFutureAdapter<Aff
}
/**
- *
- */
- public enum OperationType {
- /**
- * Read operation.
- */
- READ,
- /**
- * Write operation.
- */
- WRITE
- }
-
- /**
- * Lost policy validator.
+ * @param cctx Context.
+ * @param key Key.
+ * @param lostParts Lost partitions.
*/
- public static class LostPolicyValidator {
- /**
- *
- */
- public static Throwable validate(
- GridCacheContext cctx,
- Object key,
- OperationType opType,
- Collection<Integer> lostParts
- ) {
- CacheGroupContext grp = cctx.group();
-
- PartitionLossPolicy lostPlc = grp.config().getPartitionLossPolicy();
-
- int partition = cctx.affinity().partition(key);
-
- return validate(cctx, key, partition, opType, lostPlc, lostParts);
- }
-
- /**
- *
- */
- public static Throwable validate(
- GridCacheContext cctx,
- Collection<?> keys,
- OperationType opType,
- Collection<Integer> lostParts
- ) {
- CacheGroupContext grp = cctx.group();
-
- PartitionLossPolicy lostPlc = grp.config().getPartitionLossPolicy();
-
- for (Object key : keys) {
- int partition = cctx.affinity().partition(key);
-
- Throwable res = validate(cctx, key, partition, opType, lostPlc, lostParts);
-
- if (res != null)
- return res;
- }
-
- return null;
- }
-
- /**
- *
- */
- private static Throwable validate(
- GridCacheContext cctx,
- Object key,
- int partition,
- OperationType opType,
- PartitionLossPolicy lostPlc,
- Collection<Integer> lostParts
- ) {
- if (opType == WRITE) {
- if (lostPlc == READ_ONLY_SAFE || lostPlc == READ_ONLY_ALL) {
- return new IgniteCheckedException(
- "Failed to write to cache (cache is moved to a read-only state): " + cctx.name()
- );
- }
-
- if (lostParts.contains(partition) && lostPlc == READ_WRITE_SAFE) {
- return new CacheInvalidStateException("Failed to execute cache operation " +
- "(all partition owners have left the grid, partition data has been lost) [" +
- "cacheName=" + cctx.name() + ", part=" + partition + ", key=" + key + ']');
- }
- }
-
- if (opType == OperationType.READ) {
- if (lostParts.contains(partition) && (lostPlc == READ_ONLY_SAFE || lostPlc == READ_WRITE_SAFE))
- return new CacheInvalidStateException("Failed to execute cache operation " +
- "(all partition owners have left the grid, partition data has been lost) [" +
- "cacheName=" + cctx.name() + ", part=" + partition + ", key=" + key + ']'
- );
- }
+ private CacheInvalidStateException validateKey(
+ GridCacheContext cctx,
+ Object key,
+ Collection<Integer> lostParts
+ ) {
+ final int part = cctx.affinity().partition(key);
- return null;
- }
+ return lostParts.contains(part) ?
+ new CacheInvalidStateException("Failed to execute the cache operation " +
+ "(all partition owners have left the grid, partition data has been lost) [" +
+ "cacheName=" + cctx.name() + ", partition=" + part + ", key=" + key + ']') : null;
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index d237cb8..2ad04b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -1062,14 +1062,11 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
if (validateCache) {
GridDhtTopologyFuture topFut = cctx.exchange().lastFinishedFuture();
- if (topFut != null && !writesEmpty) {
- // All caches either read only or not. So validation of one cache context is enough.
- GridCacheContext ctx = F.first(req.writes()).context();
-
- Throwable err = topFut.validateCache(ctx, req.recovery(), writesEmpty, null, null);
+ if (topFut != null) {
+ err = tx.txState().validateTopology(cctx, writesEmpty, topFut);
if (err != null)
- onDone(null, new IgniteCheckedException(err));
+ onDone(null, err);
}
}
@@ -1311,7 +1308,7 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
TxCounters counters = tx.txCounters(true);
// Assign keys to primary nodes.
- if (req.writes() != null) {
+ if (!F.isEmpty(req.writes())) {
for (IgniteTxEntry write : req.writes()) {
IgniteTxEntry entry = tx.entry(write.txKey());
@@ -1328,7 +1325,7 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
}
}
- if (req.reads() != null) {
+ if (!F.isEmpty(req.reads())) {
for (IgniteTxEntry read : req.reads())
map(tx.entry(read.txKey()));
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index da8a738..e3ba39a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -40,6 +40,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedExceptio
import org.apache.ignite.internal.processors.cache.GridCacheMessage;
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetRequest;
import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
@@ -47,10 +48,12 @@ import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.GridLeanMap;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.lang.GridPlainRunnable;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgniteUuid;
import org.jetbrains.annotations.Nullable;
@@ -151,16 +154,7 @@ public class GridPartitionedGetFuture<K, V> extends CacheDistributedGetFutureAda
topVer = topVer.topologyVersion() > 0 ? topVer : cctx.affinity().affinityTopologyVersion();
}
- initialMap(topVer);
- }
-
- /**
- * @param topVer Topology version.
- */
- private void initialMap(AffinityTopologyVersion topVer) {
map(keys, Collections.emptyMap(), topVer);
-
- markInitialized();
}
/** {@inheritDoc} */
@@ -192,9 +186,34 @@ public class GridPartitionedGetFuture<K, V> extends CacheDistributedGetFutureAda
Map<ClusterNode, LinkedHashMap<KeyCacheObject, Boolean>> mapped,
AffinityTopologyVersion topVer
) {
+ GridDhtPartitionsExchangeFuture fut = cctx.shared().exchange().lastTopologyFuture();
+
+ // Finished DHT future is required for topology validation.
+ if (!fut.isDone()) {
+ if (fut.initialVersion().after(topVer) || (fut.exchangeActions() != null && fut.exchangeActions().hasStop()))
+ fut = cctx.shared().exchange().lastFinishedFuture();
+ else {
+ fut.listen(new IgniteInClosure<IgniteInternalFuture<AffinityTopologyVersion>>() {
+ @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> fut) {
+ if (fut.error() != null)
+ onDone(fut.error());
+ else {
+ cctx.closures().runLocalSafe(new GridPlainRunnable() {
+ @Override public void run() {
+ map(keys, mapped, topVer);
+ }
+ }, true);
+ }
+ }
+ });
+
+ return;
+ }
+ }
+
Collection<ClusterNode> cacheNodes = CU.affinityNodes(cctx, topVer);
- validate(cacheNodes, topVer);
+ validate(cacheNodes, fut);
// Future can be already done with some exception.
if (isDone())
@@ -238,7 +257,7 @@ public class GridPartitionedGetFuture<K, V> extends CacheDistributedGetFutureAda
// If this is the primary or backup node for the keys.
if (n.isLocal()) {
- GridDhtFuture<Collection<GridCacheEntryInfo>> fut = cache()
+ GridDhtFuture<Collection<GridCacheEntryInfo>> fut0 = cache()
.getDhtAsync(
n.id(),
-1,
@@ -255,7 +274,7 @@ public class GridPartitionedGetFuture<K, V> extends CacheDistributedGetFutureAda
mvccSnapshot()
);
- Collection<Integer> invalidParts = fut.invalidPartitions();
+ Collection<Integer> invalidParts = fut0.invalidPartitions();
if (!F.isEmpty(invalidParts)) {
Collection<KeyCacheObject> remapKeys = new ArrayList<>(keysSize);
@@ -277,12 +296,12 @@ public class GridPartitionedGetFuture<K, V> extends CacheDistributedGetFutureAda
}
// Add new future.
- add(fut.chain(f -> {
+ add(fut0.chain(f -> {
try {
return createResultMap(f.get());
}
catch (Exception e) {
- U.error(log, "Failed to get values from dht cache [fut=" + fut + "]", e);
+ U.error(log, "Failed to get values from dht cache [fut=" + fut0 + "]", e);
onDone(e);
@@ -309,6 +328,8 @@ public class GridPartitionedGetFuture<K, V> extends CacheDistributedGetFutureAda
}
}
}
+
+ markInitialized();
}
/**
@@ -581,24 +602,23 @@ public class GridPartitionedGetFuture<K, V> extends CacheDistributedGetFutureAda
}
/**
- *
* @param cacheNodes Cache affynity nodes.
- * @param topVer Topology version.
+ * @param topFut Topology future.
*/
- private void validate(Collection<ClusterNode> cacheNodes, AffinityTopologyVersion topVer) {
- if (cacheNodes.isEmpty()) {
- onDone(new ClusterTopologyServerNotFoundException("Failed to map keys for cache " +
- "(all partition nodes left the grid) [topVer=" + topVer + ", cache=" + cctx.name() + ']'));
+ private void validate(Collection<ClusterNode> cacheNodes, GridDhtTopologyFuture topFut) {
+ assert topFut.isDone() : topFut;
- return;
- }
+ Throwable err = topFut.validateCache(cctx, recovery, true, null, keys);
- GridDhtTopologyFuture topFut = cctx.shared().exchange().lastFinishedFuture();
+ if (err != null) {
+ onDone(err);
- Throwable err = topFut != null ? topFut.validateCache(cctx, recovery, true, null, keys) : null;
+ return;
+ }
- if (err != null)
- onDone(err);
+ if (cacheNodes.isEmpty())
+ onDone(new ClusterTopologyServerNotFoundException("Failed to map keys for cache " +
+ "(all partition nodes left the grid) [topVer=" + topFut.topologyVersion() + ", cache=" + cctx.name() + ']'));
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index 4ae143a..24d4bf3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -47,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheMessage;
import org.apache.ignite.internal.processors.cache.GridCacheUtils.BackupPostProcessingClosure;
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
import org.apache.ignite.internal.processors.cache.distributed.near.CacheVersionedValue;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetResponse;
@@ -55,11 +56,13 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearSing
import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.lang.GridPlainRunnable;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.jetbrains.annotations.Nullable;
@@ -246,7 +249,32 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter<Objec
*/
@SuppressWarnings("unchecked")
private void map(AffinityTopologyVersion topVer) {
- if (!validate(topVer))
+ GridDhtPartitionsExchangeFuture fut = cctx.shared().exchange().lastTopologyFuture();
+
+ // Finished DHT future is required for topology validation.
+ if (!fut.isDone()) {
+ if (fut.initialVersion().after(topVer) || (fut.exchangeActions() != null && fut.exchangeActions().hasStop()))
+ fut = cctx.shared().exchange().lastFinishedFuture();
+ else {
+ fut.listen(new IgniteInClosure<IgniteInternalFuture<AffinityTopologyVersion>>() {
+ @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> fut) {
+ if (fut.error() != null)
+ onDone(fut.error());
+ else {
+ cctx.closures().runLocalSafe(new GridPlainRunnable() {
+ @Override public void run() {
+ map(topVer);
+ }
+ }, true);
+ }
+ }
+ });
+
+ return;
+ }
+ }
+
+ if (!validate(fut))
return;
ClusterNode node = mapKeyToNode(topVer);
@@ -262,7 +290,7 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter<Objec
// Read value if node is localNode.
if (node.isLocal()) {
- GridDhtFuture<GridCacheEntryInfo> fut = cctx.dht()
+ GridDhtFuture<GridCacheEntryInfo> fut0 = cctx.dht()
.getDhtSingleAsync(
node.id(),
-1,
@@ -279,7 +307,7 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter<Objec
mvccSnapshot
);
- Collection<Integer> invalidParts = fut.invalidPartitions();
+ Collection<Integer> invalidParts = fut0.invalidPartitions();
if (!F.isEmpty(invalidParts)) {
addNodeAsInvalid(node);
@@ -290,14 +318,14 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter<Objec
map(updTopVer);
}
else {
- fut.listen(f -> {
+ fut0.listen(f -> {
try {
GridCacheEntryInfo info = f.get();
setResult(info);
}
catch (Exception e) {
- U.error(log, "Failed to get values from dht cache [fut=" + fut + "]", e);
+ U.error(log, "Failed to get values from dht cache [fut=" + fut0 + "]", e);
onDone(e);
}
@@ -819,16 +847,16 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter<Objec
}
/**
- * @param topVer Topology version.
+ * @param topFut Ready topology future for validation.
* @return True if validate success, False is not.
*/
- private boolean validate(AffinityTopologyVersion topVer) {
- if (!checkRetryPermits(topVer))
- return false;
+ private boolean validate(GridDhtTopologyFuture topFut) {
+ assert topFut.isDone() : topFut;
- GridDhtTopologyFuture lastFut = cctx.shared().exchange().lastFinishedFuture();
+ if (!checkRetryPermits(topFut.topologyVersion()))
+ return false;
- Throwable error = lastFut.validateCache(cctx, recovery, true, key, null);
+ Throwable error = topFut.validateCache(cctx, recovery, true, key, null);
if (error != null) {
onDone(error);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 8c5d84c9..5e3cad2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1521,7 +1521,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
}
// Optimisation: try to resolve value locally and escape 'get future' creation.
- if (!forcePrimary && ctx.affinityNode()) {
+ if (!forcePrimary && ctx.config().isReadFromBackup() && ctx.affinityNode() &&
+ ctx.group().topology().lostPartitions().isEmpty()) {
ctx.shared().database().checkpointReadLock();
try {
@@ -1917,7 +1918,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
assert topFut.isDone() : topFut;
- Throwable err = topFut.validateCache(ctx, req.recovery(), false, null, null);
+ Throwable err =
+ topFut.validateCache(ctx, req.recovery(), false, null, req.keys());
if (err != null) {
IgniteCheckedException e = new IgniteCheckedException(err);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index 217b968..18eba12 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -527,7 +527,8 @@ public class GridDhtColocatedCache<K, V> extends GridDhtTransactionalCacheAdapte
expiryPlc = expiryPolicy(null);
// Optimization: try to resolve value locally and escape 'get future' creation.
- if (!forcePrimary && ctx.affinityNode()) {
+ if (!forcePrimary && ctx.config().isReadFromBackup() && ctx.affinityNode() &&
+ ctx.topology().lostPartitions().isEmpty()) {
ctx.shared().database().checkpointReadLock();
try {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 2c64e6c..120b2c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -1593,7 +1593,7 @@ public class GridDhtPartitionDemander {
if (log.isDebugEnabled())
log.debug("Partitions have been scheduled to resend [reason=" +
- "Rebalance is done [grp=" + grp.cacheOrGroupName() + "]");
+ "Rebalance is done, grp=" + grp.cacheOrGroupName() + "]");
if (!cancelled)
ctx.exchange().scheduleResendPartitions();
@@ -1606,9 +1606,12 @@ public class GridDhtPartitionDemander {
}
if (!m.isEmpty()) {
- U.log(log, ("Reassigning partitions that were missed: " + m));
+ U.log(log, "Reassigning partitions that were missed [parts=" + m +
+ ", grpId=" + grp.groupId() +
+ ", grpName=" + grp.cacheOrGroupName() +
+ ", topVer=" + topVer + ']');
- onDone(false); //Finished but has missed partitions, will force dummy exchange
+ onDone(false); // Finished but has missed partitions, will force dummy exchange
ctx.exchange().forceReassign(exchId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 50d0677..8dde2bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -36,7 +36,6 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
@@ -168,8 +167,9 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
private static final IgniteProductVersion FORCE_AFF_REASSIGNMENT_SINCE = IgniteProductVersion.fromString("2.4.3");
/**
- * This may be useful when per-entry (not per-cache based) partition policy is in use. See {@link
- * IgniteSystemProperties#IGNITE_SKIP_PARTITION_SIZE_VALIDATION} for details. Default value is {@code false}.
+ * This may be useful when per-entry (not per-cache based) partition policy is in use.
+ * See {@link IgniteSystemProperties#IGNITE_SKIP_PARTITION_SIZE_VALIDATION} for details.
+ * Default value is {@code false}.
*/
private static final boolean SKIP_PARTITION_SIZE_VALIDATION = Boolean.getBoolean(IgniteSystemProperties.IGNITE_SKIP_PARTITION_SIZE_VALIDATION);
@@ -224,9 +224,9 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
private volatile ExchangeType exchangeType;
/**
- * Discovery event receive latch. There is a race between discovery event processing and single message processing,
- * so it is possible to create an exchange future before the actual discovery event is received. This latch is
- * notified when the discovery event arrives.
+ * Discovery event receive latch. There is a race between discovery event processing and single message
+ * processing, so it is possible to create an exchange future before the actual discovery event is received.
+ * This latch is notified when the discovery event arrives.
*/
@GridToStringExclude
private final CountDownLatch evtLatch = new CountDownLatch(1);
@@ -245,14 +245,15 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
private AtomicReference<GridCacheVersion> lastVer = new AtomicReference<>();
/**
- * Message received from node joining cluster (if this is 'node join' exchange), needed if this exchange is merged
- * with another one.
+ * Message received from node joining cluster (if this is 'node join' exchange),
+ * needed if this exchange is merged with another one.
*/
@GridToStringExclude
private GridDhtPartitionsSingleMessage pendingJoinMsg;
/**
- * Messages received on non-coordinator are stored in case if this node becomes coordinator.
+ * Messages received on non-coordinator are stored in case if this node
+ * becomes coordinator.
*/
private final Map<UUID, GridDhtPartitionsSingleMessage> pendingSingleMsgs = new ConcurrentHashMap<>();
@@ -312,7 +313,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
private volatile IgniteDhtPartitionHistorySuppliersMap partHistSuppliers = new IgniteDhtPartitionHistorySuppliersMap();
/** Reserved max available history for calculation of history supplier on coordinator. */
- private volatile Map<Integer, Map<Integer, Long>> partHistReserved;
+ private volatile Map<Integer /** Group. */, Map<Integer /** Partition */, Long /** Counter. */>> partHistReserved;
/** */
@GridToStringExclude
@@ -354,16 +355,19 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
private volatile boolean partitionsReceived;
/** Latest (by update sequences) full message with exchangeId == null, need to be processed right after future is done. */
+ @GridToStringExclude
private GridDhtPartitionsFullMessage delayedLatestMsg;
/** Future for wait all exchange listeners comepleted. */
+ @GridToStringExclude
private final GridFutureAdapter<?> afterLsnrCompleteFut = new GridFutureAdapter<>();
/** Time bag to measure and store exchange stages times. */
+ @GridToStringExclude
private final TimeBag timeBag;
/** Start time of exchange. */
- private long startTime = System.nanoTime();
+ private long startTime = System.currentTimeMillis();
/** Init time of exchange in milliseconds. */
private volatile long initTime;
@@ -371,7 +375,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
/** Discovery lag / Clocks discrepancy, calculated on coordinator when all single messages are received. */
private T2<Long, UUID> discoveryLag;
- /** Partitions scheduled for clearing before rebalance for this topology version. */
+ /** Partitions scheduled for clearing before rebalancing for this topology version. */
private Map<Integer, Set<Integer>> clearingPartitions;
/** Specified only in case of 'cluster is fully rebalanced' state achieved. */
@@ -1074,7 +1078,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
/**
- * @return {@code true} if Event node is in baseline and failed and {@code false} otherwise.
+ * @return {@code True} if event node is in baseline and failed and {@code false} otherwise.
*/
public boolean isBaselineNodeFailed() {
BaselineTopology top = firstEvtDiscoCache.state().baselineTopology();
@@ -1138,7 +1142,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
Collections.emptySet(),
null,
null,
- null);
+ null,
+ clientTop.lostPartitions());
}
finally {
cctx.exchange().exchangerBlockingSectionEnd();
@@ -1634,7 +1639,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
// Pre-create missing partitions using current affinity.
- if (!exchCtx.mergeExchanges()) {
+ if (!exchCtx.mergeExchanges() && !exchCtx.exchangeFreeSwitch()) {
for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
if (grp.isLocal() || cacheGroupStopping(grp.groupId()))
continue;
@@ -1675,7 +1680,27 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
cctx.exchange().exchangerBlockingSectionBegin();
try {
- if (!context().exchangeFreeSwitch()) {
+ if (context().exchangeFreeSwitch()) {
+ // Update local maps, see CachePartitionLossWithRestartsTest.
+ doInParallel(
+ U.availableThreadCount(cctx.kernalContext(), GridIoPolicy.SYSTEM_POOL, 2),
+ cctx.kernalContext().getSystemExecutorService(),
+ cctx.affinity().cacheGroups().values(),
+ desc -> {
+ if (desc.config().getCacheMode() == CacheMode.LOCAL)
+ return null;
+
+ CacheGroupContext grp = cctx.cache().cacheGroup(desc.groupId());
+
+ GridDhtPartitionTopology top = grp != null ? grp.topology() :
+ cctx.exchange().clientTopology(desc.groupId(), events().discoveryCache());
+
+ top.beforeExchange(this, true, false); // Not expecting new moving partitions.
+
+ return null;
+ });
+ }
+ else {
if (crd.isLocal()) {
if (remaining.isEmpty()) {
initFut.onDone(true);
@@ -2378,15 +2403,17 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
}
- boolean locNodeNotCrd = crd == null || !crd.isLocal();
-
- if ((locNodeNotCrd && (serverNodeDiscoveryEvent() || localJoinExchange())) || exchCtx.exchangeFreeSwitch())
+ if (exchCtx.events().hasServerLeft() || activateCluster())
detectLostPartitions(res);
Map<Integer, CacheGroupValidation> m = U.newHashMap(cctx.cache().cacheGroups().size());
- for (CacheGroupContext grp : cctx.cache().cacheGroups())
- m.put(grp.groupId(), validateCacheGroup(grp, events().lastEvent().topologyNodes()));
+ for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
+ CacheGroupValidation valRes = validateCacheGroup(grp, events().lastEvent().topologyNodes());
+
+ if (!valRes.isValid() || valRes.hasLostPartitions())
+ m.put(grp.groupId(), valRes);
+ }
grpValidRes = m;
}
@@ -2575,7 +2602,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
latestStartedNode = nodeId;
}
- return new T2<>(U.nanosToMillis(maxStartTime - minStartTime), latestStartedNode);
+ return new T2<>(maxStartTime - minStartTime, latestStartedNode);
}
/**
@@ -3407,38 +3434,29 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
* @param resTopVer Result topology version.
*/
private void detectLostPartitions(AffinityTopologyVersion resTopVer) {
- AtomicInteger detected = new AtomicInteger();
-
try {
// Reserve at least 2 threads for system operations.
doInParallelUninterruptibly(
U.availableThreadCount(cctx.kernalContext(), GridIoPolicy.SYSTEM_POOL, 2),
cctx.kernalContext().getSystemExecutorService(),
- cctx.cache().cacheGroups(),
- grp -> {
- if (!grp.isLocal()) {
- // Do not trigger lost partition events on start.
- boolean evt = !localJoinExchange() && !activateCluster();
+ cctx.affinity().cacheGroups().values(),
+ desc -> {
+ if (desc.config().getCacheMode() == CacheMode.LOCAL)
+ return null;
- if (grp.topology().detectLostPartitions(resTopVer, evt ? events().lastEvent() : null))
- detected.incrementAndGet();
- }
+ CacheGroupContext grp = cctx.cache().cacheGroup(desc.groupId());
+
+ GridDhtPartitionTopology top = grp != null ? grp.topology() :
+ cctx.exchange().clientTopology(desc.groupId(), events().discoveryCache());
+
+ top.detectLostPartitions(resTopVer, this);
return null;
});
- }
- catch (IgniteCheckedException e) {
+ } catch (IgniteCheckedException e) {
throw new IgniteException(e);
}
- if (detected.get() > 0) {
- if (log.isDebugEnabled())
- log.debug("Partitions have been scheduled to resend [reason=" +
- "Lost partitions detect on " + resTopVer + "]");
-
- cctx.exchange().scheduleResendPartitions();
- }
-
timeBag.finishGlobalStage("Detect lost partitions");
}
@@ -3449,21 +3467,21 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
assert !exchCtx.mergeExchanges();
try {
- // Reserve at least 2 threads for system operations.
- U.doInParallel(
+ doInParallelUninterruptibly(
U.availableThreadCount(cctx.kernalContext(), GridIoPolicy.SYSTEM_POOL, 2),
cctx.kernalContext().getSystemExecutorService(),
- cctx.cache().cacheGroups(),
- grp -> {
- if (grp.isLocal())
+ cctx.affinity().caches().values(),
+ desc -> {
+ if (desc.cacheConfiguration().getCacheMode() == CacheMode.LOCAL)
return null;
- for (String cacheName : cacheNames) {
- if (grp.hasCache(cacheName)) {
- grp.topology().resetLostPartitions(initialVersion());
+ if (cacheNames.contains(desc.cacheName())) {
+ CacheGroupContext grp = cctx.cache().cacheGroup(desc.groupId());
- break;
- }
+ GridDhtPartitionTopology top = grp != null ? grp.topology() :
+ cctx.exchange().clientTopology(desc.groupId(), events().discoveryCache());
+
+ top.resetLostPartitions(initialVersion());
}
return null;
@@ -3687,16 +3705,16 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
if (activateCluster() || changedBaseline())
assignPartitionsStates();
- DiscoveryCustomMessage discoveryCustomMessage = ((DiscoveryCustomEvent)firstDiscoEvt).customMessage();
+ DiscoveryCustomMessage discoveryCustomMessage = ((DiscoveryCustomEvent) firstDiscoEvt).customMessage();
if (discoveryCustomMessage instanceof DynamicCacheChangeBatch) {
if (exchActions != null) {
- assignPartitionsStates();
-
Set<String> caches = exchActions.cachesToResetLostPartitions();
if (!F.isEmpty(caches))
resetLostPartitions(caches);
+
+ assignPartitionsStates();
}
}
else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
@@ -3706,10 +3724,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
assignPartitionsStates();
}
}
- else {
- if (exchCtx.events().hasServerJoin())
- assignPartitionsStates();
- }
+ else if (exchCtx.events().hasServerJoin())
+ assignPartitionsStates();
// Recalculate new affinity based on partitions availability.
if (!exchCtx.mergeExchanges() && forceAffReassignment) {
@@ -3737,12 +3753,6 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
if (!cctx.affinity().rebalanceRequired() && !deactivateCluster())
msg.rebalanced(true);
- // Lost partition detection should be done after full message is prepared otherwise in case of IGNORE policy
- // lost partitions will be moved to OWNING state and after what send to other nodes resulting in
- // wrong lost state calculation (another possibility to consider - calculate lost state only on coordinator).
- if (firstDiscoEvt.type() != EVT_DISCOVERY_CUSTOM_EVT && exchCtx.events().hasServerLeft())
- detectLostPartitions(resTopVer);
-
if (exchCtx.mergeExchanges()) {
assert !centralizedAff;
@@ -4520,7 +4530,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
msg.partsToReload(cctx.localNodeId(), grpId),
partsSizes.getOrDefault(grpId, Collections.emptyMap()),
null,
- this);
+ this,
+ msg.lostPartitions(grpId));
}
else {
GridDhtPartitionTopology top = cctx.exchange().clientTopology(grpId, events().discoveryCache());
@@ -4534,7 +4545,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
Collections.emptySet(),
null,
null,
- this);
+ this,
+ msg.lostPartitions(grpId));
}
return null;
@@ -5296,6 +5308,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
return "GridDhtPartitionsExchangeFuture [topVer=" + initialVersion() +
", evt=" + (firstDiscoEvt != null ? IgniteUtils.gridEventName(firstDiscoEvt.type()) : -1) +
", evtNode=" + (firstDiscoEvt != null ? firstDiscoEvt.eventNode() : null) +
+ ", rebalanced=" + rebalanced() +
", done=" + isDone() +
", newCrdFut=" + this.newCrdFut + ']';
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index d34bebf..59a9857 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -27,6 +27,8 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.cluster.ClusterNode;
@@ -40,6 +42,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.lang.IgniteThrowableFunction;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.T2;
@@ -135,6 +138,11 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
/** */
private byte flags;
+ /** */
+ @GridDirectMap(keyType = Integer.class, valueType = int[].class)
+ @GridToStringExclude
+ private Map<Integer, int[]> lostParts;
+
/**
* Required by {@link Externalizable}.
*/
@@ -146,8 +154,8 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
* @param id Exchange ID.
* @param lastVer Last version.
* @param topVer Topology version. For messages not related to exchange may be {@link AffinityTopologyVersion#NONE}.
- * @param partHistSuppliers
- * @param partsToReload
+ * @param partHistSuppliers Suppliers.
+ * @param partsToReload Partitions to reload.
*/
public GridDhtPartitionsFullMessage(@Nullable GridDhtPartitionExchangeId id,
@Nullable GridCacheVersion lastVer,
@@ -188,6 +196,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
cp.joinedNodeAff = joinedNodeAff;
cp.idealAffDiff = idealAffDiff;
cp.flags = flags;
+ cp.lostParts = lostParts;
}
/**
@@ -317,6 +326,36 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
}
/**
+ * @param grpId Group id.
+ * @param lostParts Lost parts.
+ */
+ public void addLostPartitions(int grpId, Collection<Integer> lostParts) {
+ if (lostParts.isEmpty())
+ return;
+
+ if (this.lostParts == null)
+ this.lostParts = new HashMap<>();
+
+ this.lostParts.put(grpId, lostParts.stream().mapToInt(v -> v).toArray());
+ }
+
+ /**
+ * @param grpId Group id.
+ * @return Lost partitions for a group.
+ */
+ @Nullable public Set<Integer> lostPartitions(int grpId) {
+ if (lostParts == null)
+ return null;
+
+ int[] parts = lostParts.get(grpId);
+
+ if (parts == null)
+ return Collections.emptySet();
+
+ return IntStream.of(parts).boxed().collect(Collectors.toSet());
+ }
+
+ /**
* @param grpId Cache group ID.
* @param partsCnt Total cache partitions.
* @return Partition update counters.
@@ -662,48 +701,54 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
writer.incrementState();
case 11:
- if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes))
+ if (!writer.writeMap("lostParts", lostParts, MessageCollectionItemType.INT, MessageCollectionItemType.INT_ARR))
return false;
writer.incrementState();
case 12:
- if (!writer.writeByteArray("partCntrsBytes2", partCntrsBytes2))
+ if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes))
return false;
writer.incrementState();
case 13:
- if (!writer.writeByteArray("partHistSuppliersBytes", partHistSuppliersBytes))
+ if (!writer.writeByteArray("partCntrsBytes2", partCntrsBytes2))
return false;
writer.incrementState();
case 14:
- if (!writer.writeByteArray("partsBytes", partsBytes))
+ if (!writer.writeByteArray("partHistSuppliersBytes", partHistSuppliersBytes))
return false;
writer.incrementState();
case 15:
- if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes))
+ if (!writer.writeByteArray("partsBytes", partsBytes))
return false;
writer.incrementState();
case 16:
- if (!writer.writeByteArray("partsToReloadBytes", partsToReloadBytes))
+ if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes))
return false;
writer.incrementState();
case 17:
- if (!writer.writeAffinityTopologyVersion("resTopVer", resTopVer))
+ if (!writer.writeByteArray("partsToReloadBytes", partsToReloadBytes))
return false;
writer.incrementState();
case 18:
+ if (!writer.writeAffinityTopologyVersion("resTopVer", resTopVer))
+ return false;
+
+ writer.incrementState();
+
+ case 19:
if (!writer.writeAffinityTopologyVersion("topVer", topVer))
return false;
@@ -766,7 +811,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 11:
- partCntrsBytes = reader.readByteArray("partCntrsBytes");
+ lostParts = reader.readMap("lostParts", MessageCollectionItemType.INT, MessageCollectionItemType.INT_ARR, false);
if (!reader.isLastRead())
return false;
@@ -774,7 +819,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 12:
- partCntrsBytes2 = reader.readByteArray("partCntrsBytes2");
+ partCntrsBytes = reader.readByteArray("partCntrsBytes");
if (!reader.isLastRead())
return false;
@@ -782,7 +827,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 13:
- partHistSuppliersBytes = reader.readByteArray("partHistSuppliersBytes");
+ partCntrsBytes2 = reader.readByteArray("partCntrsBytes2");
if (!reader.isLastRead())
return false;
@@ -790,7 +835,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 14:
- partsBytes = reader.readByteArray("partsBytes");
+ partHistSuppliersBytes = reader.readByteArray("partHistSuppliersBytes");
if (!reader.isLastRead())
return false;
@@ -798,7 +843,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 15:
- partsSizesBytes = reader.readByteArray("partsSizesBytes");
+ partsBytes = reader.readByteArray("partsBytes");
if (!reader.isLastRead())
return false;
@@ -806,7 +851,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 16:
- partsToReloadBytes = reader.readByteArray("partsToReloadBytes");
+ partsSizesBytes = reader.readByteArray("partsSizesBytes");
if (!reader.isLastRead())
return false;
@@ -814,7 +859,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 17:
- resTopVer = reader.readAffinityTopologyVersion("resTopVer");
+ partsToReloadBytes = reader.readByteArray("partsToReloadBytes");
if (!reader.isLastRead())
return false;
@@ -822,6 +867,14 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 18:
+ resTopVer = reader.readAffinityTopologyVersion("resTopVer");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 19:
topVer = reader.readAffinityTopologyVersion("topVer");
if (!reader.isLastRead())
@@ -841,7 +894,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 19;
+ return 20;
}
/** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 8f0d390..17a500b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -47,11 +47,11 @@ import org.apache.ignite.internal.util.typedef.CI1;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISABLE_REBALANCING_CANCELLATION_OPTIMIZATION;
-import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_UNLOADED;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
/**
@@ -169,8 +169,10 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter {
}
/** {@inheritDoc} */
- @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
- GridDhtPartitionsExchangeFuture exchFut) {
+ @Override public GridDhtPreloaderAssignments generateAssignments(
+ GridDhtPartitionExchangeId exchId,
+ GridDhtPartitionsExchangeFuture exchFut
+ ) {
assert exchFut == null || exchFut.isDone();
// No assignments for disabled preloader.
@@ -239,6 +241,11 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter {
part.resetUpdateCounter();
}
+ if (part.state() != MOVING && part.state() != OWNING) {
+ throw new AssertionError("Partition has invalid state for rebalance "
+ + aff.topologyVersion() + " " + part);
+ }
+
ClusterNode histSupplier = null;
if (grp.persistenceEnabled() && exchFut != null) {
@@ -263,26 +270,13 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter {
}
// TODO FIXME https://issues.apache.org/jira/browse/IGNITE-11790
- msg.partitions().addHistorical(p, part.initialUpdateCounter(), countersMap.updateCounter(p), partitions);
+ msg.partitions().
+ addHistorical(p, part.initialUpdateCounter(), countersMap.updateCounter(p), partitions);
}
else {
List<ClusterNode> picked = remoteOwners(p, topVer);
- if (picked.isEmpty()) {
- top.own(part);
-
- if (grp.eventRecordable(EVT_CACHE_REBALANCE_PART_DATA_LOST)) {
- grp.addRebalanceEvent(p,
- EVT_CACHE_REBALANCE_PART_DATA_LOST,
- exchId.eventNode(),
- exchId.event(),
- exchId.eventTimestamp());
- }
-
- if (log.isDebugEnabled())
- log.debug("Owning partition as there are no other owners: " + part);
- }
- else {
+ if (!picked.isEmpty()) {
ClusterNode n = picked.get(p % picked.size());
GridDhtPartitionDemandMessage msg = assignments.get(n);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java
index 2574c25..cf89476a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java
@@ -26,11 +26,13 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.PartitionLossPolicy;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
@@ -48,17 +50,20 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.Gri
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
import org.apache.ignite.internal.util.F0;
import org.apache.ignite.internal.util.GridAtomicLong;
import org.apache.ignite.internal.util.GridPartitionStateMap;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
@@ -124,19 +129,27 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
/** */
private volatile Map<Integer, Long> globalPartSizes;
+ /** */
+ private Set<Integer> lostParts;
+
+ /** */
+ private PartitionLossPolicy partLossPlc;
+
/**
* @param cctx Context.
* @param discoCache Discovery data cache.
* @param grpId Group ID.
* @param parts Number of partitions in the group.
* @param similarAffKey Key to find caches with similar affinity.
+ * @param partLossPlc Loss policy.
*/
public GridClientPartitionTopology(
GridCacheSharedContext<?, ?> cctx,
DiscoCache discoCache,
int grpId,
int parts,
- Object similarAffKey
+ Object similarAffKey,
+ PartitionLossPolicy partLossPlc
) {
this.cctx = cctx;
this.discoCache = discoCache;
@@ -153,6 +166,8 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
updateSeq.get());
cntrMap = new CachePartitionFullCountersMap(parts);
+
+ this.partLossPlc = partLossPlc;
}
/** {@inheritDoc} */
@@ -724,7 +739,9 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
Set<Integer> partsToReload,
@Nullable Map<Integer, Long> partSizes,
@Nullable AffinityTopologyVersion msgTopVer,
- @Nullable GridDhtPartitionsExchangeFuture exchFut) {
+ @Nullable GridDhtPartitionsExchangeFuture exchFut,
+ @Nullable Set<Integer> lostParts
+ ) {
if (log.isDebugEnabled())
log.debug("Updating full partition map [exchVer=" + exchangeVer + ", parts=" + fullMapString() + ']');
@@ -832,6 +849,8 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
consistencyCheck();
+ this.lostParts = lostParts;
+
if (log.isDebugEnabled())
log.debug("Partition map after full update: " + fullMapString());
@@ -852,11 +871,11 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
for (int i = 0; i < cntrMap.size(); i++) {
int pId = cntrMap.partitionAt(i);
- long initialUpdateCntr = cntrMap.initialUpdateCounterAt(i);
+ long initUpdateCntr = cntrMap.initialUpdateCounterAt(i);
long updateCntr = cntrMap.updateCounterAt(i);
if (this.cntrMap.updateCounter(pId) < updateCntr) {
- this.cntrMap.initialUpdateCounter(pId, initialUpdateCntr);
+ this.cntrMap.initialUpdateCounter(pId, initUpdateCntr);
this.cntrMap.updateCounter(pId, updateCntr);
}
}
@@ -997,22 +1016,103 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
- @Override public boolean detectLostPartitions(AffinityTopologyVersion affVer, DiscoveryEvent discoEvt) {
- assert false : "detectLostPartitions should never be called on client topology";
+ @Override public boolean detectLostPartitions(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture fut) {
+ lock.writeLock().lock();
- return false;
+ try {
+ if (node2part == null)
+ node2part = new GridDhtPartitionFullMap();
+
+ final GridClusterStateProcessor state = cctx.kernalContext().state();
+
+ boolean isInMemoryCluster = CU.isInMemoryCluster(
+ cctx.kernalContext().discovery().allNodes(),
+ cctx.kernalContext().marshallerContext().jdkMarshaller(),
+ U.resolveClassLoader(cctx.kernalContext().config())
+ );
+
+ boolean compatibleWithIgnorePlc = isInMemoryCluster
+ && state.isBaselineAutoAdjustEnabled() && state.baselineAutoAdjustTimeout() == 0L;
+
+ // Calculate how loss data is handled.
+ boolean safe = partLossPlc != PartitionLossPolicy.IGNORE || !compatibleWithIgnorePlc;
+
+ boolean changed = false;
+
+ for (int part = 0; part < parts; part++) {
+ boolean lost = F.contains(lostParts, part);
+
+ if (!lost) {
+ boolean hasOwner = false;
+
+ // Detect if all owners are left.
+ for (GridDhtPartitionMap partMap : node2part.values()) {
+ if (partMap.get(part) == OWNING) {
+ hasOwner = true;
+
+ break;
+ }
+ }
+
+ if (!hasOwner) {
+ lost = true;
+
+ // Do not detect and record lost partition in IGNORE mode.
+ if (safe) {
+ if (lostParts == null)
+ lostParts = new TreeSet<>();
+
+ lostParts.add(part);
+ }
+ }
+ }
+
+ if (lost) {
+ // Update remote maps according to policy.
+ for (Map.Entry<UUID, GridDhtPartitionMap> entry : node2part.entrySet()) {
+ if (entry.getValue().get(part) != null)
+ entry.getValue().put(part, safe ? LOST : OWNING);
+ }
+ }
+ }
+
+ return changed;
+ }
+ finally {
+ lock.writeLock().unlock();
+ }
}
/** {@inheritDoc} */
@Override public void resetLostPartitions(AffinityTopologyVersion affVer) {
- assert false : "resetLostPartitions should never be called on client topology";
+ lock.writeLock().lock();
+
+ try {
+ for (Map.Entry<UUID, GridDhtPartitionMap> e : node2part.entrySet()) {
+ for (Map.Entry<Integer, GridDhtPartitionState> e0 : e.getValue().entrySet()) {
+ if (e0.getValue() != LOST)
+ continue;
+
+ e0.setValue(OWNING);
+ }
+ }
+
+ lostParts = null;
+ } finally {
+ lock.writeLock().unlock();
+ }
}
/** {@inheritDoc} */
- @Override public Collection<Integer> lostPartitions() {
- assert false : "lostPartitions should never be called on client topology";
+ @Override public Set<Integer> lostPartitions() {
+ lock.readLock().lock();
- return Collections.emptyList();
+ try {
+ return lostParts == null ? Collections.emptySet() : lostParts;
+ }
+ finally {
+ lock.readLock().unlock();
+ }
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java
index 3e18737..20a2492 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java
@@ -551,13 +551,19 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
synchronized (this) {
GridDhtPartitionState prevState = state();
- boolean update = this.state.compareAndSet(state, setPartState(state, toState));
+ boolean updated = this.state.compareAndSet(state, setPartState(state, toState));
- if (update) {
+ if (updated) {
assert toState != EVICTED || reservations() == 0 : this;
try {
- ctx.wal().log(new PartitionMetaStateRecord(grp.groupId(), id, toState, 0));
+ // Optimization: do not log OWNING -> OWNING.
+ if (prevState == OWNING && toState == LOST)
+ return true;
+
+ // Log LOST partitions as OWNING.
+ ctx.wal().log(
+ new PartitionMetaStateRecord(grp.groupId(), id, toState == LOST ? OWNING : toState, 0));
}
catch (IgniteCheckedException e) {
U.error(log, "Failed to log partition state change to WAL.", e);
@@ -570,15 +576,15 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
+ ", p=" + id + ", prev=" + prevState + ", to=" + toState + "]");
}
- return update;
+ return updated;
}
}
else {
GridDhtPartitionState prevState = state();
- boolean update = this.state.compareAndSet(state, setPartState(state, toState));
+ boolean updated = this.state.compareAndSet(state, setPartState(state, toState));
- if (update) {
+ if (updated) {
assert toState != EVICTED || reservations() == 0 : this;
if (log.isDebugEnabled())
@@ -586,7 +592,7 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
+ ", p=" + id + ", prev=" + prevState + ", to=" + toState + "]");
}
- return update;
+ return updated;
}
}
@@ -940,7 +946,7 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
finally {
clearEvicting();
- clearFuture.finish();
+ clearFuture.finish(); // Invokes clear future listeners.
}
}
@@ -1105,6 +1111,13 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
}
/**
+ * Reset partition initial update counter.
+ */
+ public void resetInitialUpdateCounter() {
+ store.resetInitialUpdateCounter();
+ }
+
+ /**
* @return Total size of all caches.
*/
public long fullSize() {
@@ -1146,7 +1159,9 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
// Do not clear fresh rows in case of partition reloading.
// This is required because normal updates are possible to moving partition which is currently cleared.
- if (row.version().compareTo(clearVer) >= 0 && state() == MOVING)
+ // We can clean OWNING partition if a partition has been reset from lost state.
+ // In this case new updates must be preserved.
+ if (row.version().compareTo(clearVer) >= 0 && (state() == MOVING || state() == OWNING))
continue;
if (grp.sharedGroup() && (hld == null || hld.cctx.cacheId() != row.cacheId()))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java
index 1097294..dfc6230 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java
@@ -17,15 +17,12 @@
package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
-import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.events.DiscoveryEvent;
-import org.apache.ignite.events.EventType;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.managers.discovery.DiscoCache;
import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
@@ -291,9 +288,11 @@ public interface GridDhtPartitionTopology {
* @param partMap Update partition map.
* @param cntrMap Partition update counters.
* @param partsToReload Set of partitions that need to be reloaded.
+ * @param partSizes Global partition sizes.
* @param msgTopVer Topology version from incoming message. This value is not null only for case message is not
* related to exchange. Value should be not less than previous 'Topology version from exchange'.
* @param exchFut Future which is not null for initial partition update on exchange.
+ * @param lostParts Lost partitions.
* @return {@code True} if local state was changed.
*/
public boolean update(
@@ -303,8 +302,8 @@ public interface GridDhtPartitionTopology {
Set<Integer> partsToReload,
@Nullable Map<Integer, Long> partSizes,
@Nullable AffinityTopologyVersion msgTopVer,
- @Nullable GridDhtPartitionsExchangeFuture exchFut
- );
+ @Nullable GridDhtPartitionsExchangeFuture exchFut,
+ @Nullable Set<Integer> lostParts);
/**
* @param exchId Exchange ID.
@@ -329,16 +328,15 @@ public interface GridDhtPartitionTopology {
public void applyUpdateCounters();
/**
- * Checks if there is at least one owner for each partition in the cache topology.
- * If not, marks such a partition as LOST.
- * <p>
- * This method should be called on topology coordinator after all partition messages are received.
+ * Checks if there is at least one owner for each partition in the cache topology for a local node.
+ * If not marks such a partition as LOST or OWNING depending on a policy.
*
* @param resTopVer Exchange result version.
- * @param discoEvt Discovery event for which we detect lost partitions if {@link EventType#EVT_CACHE_REBALANCE_PART_DATA_LOST} event should be fired.
+ * @param fut Exchange futute for topology events to detect.
+ *
* @return {@code True} if partitions state got updated.
*/
- public boolean detectLostPartitions(AffinityTopologyVersion resTopVer, @Nullable DiscoveryEvent discoEvt);
+ public boolean detectLostPartitions(AffinityTopologyVersion resTopVer, GridDhtPartitionsExchangeFuture fut);
/**
* Resets the state of all LOST partitions to OWNING.
@@ -350,7 +348,7 @@ public interface GridDhtPartitionTopology {
/**
* @return Collection of lost partitions, if any.
*/
- public Collection<Integer> lostPartitions();
+ public Set<Integer> lostPartitions();
/**
* Pre-processes partition update counters before exchange.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
index a3cd8b3..7920ac1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
@@ -35,7 +35,6 @@ import java.util.stream.Collectors;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.cache.PartitionLossPolicy;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.EventType;
@@ -58,6 +57,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.Gri
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
+import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
import org.apache.ignite.internal.util.F0;
import org.apache.ignite.internal.util.GridAtomicLong;
import org.apache.ignite.internal.util.GridLongList;
@@ -74,10 +75,12 @@ import org.apache.ignite.lang.IgniteInClosure;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.cache.PartitionLossPolicy.IGNORE;
import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.ExchangeType.ALL;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.ExchangeType.NONE;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
@@ -113,7 +116,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
/** */
private final AtomicReferenceArray<GridDhtLocalPartition> locParts;
- /** Node to partition map from all nodes. */
+ /** Node to partition map. */
private GridDhtPartitionFullMap node2part;
/** */
@@ -305,7 +308,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
assert topVer != null;
- return !AffinityTopologyVersion.NONE.equals(topVer);
+ return topVer.initialized();
}
/** {@inheritDoc} */
@@ -446,7 +449,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
}
else
- createPartitions(affVer, affAssignment, updateSeq, exchFut);
+ createPartitions(affVer, affAssignment, updateSeq);
}
else {
// If preloader is disabled, then we simply clear out
@@ -496,10 +499,12 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
* @param affVer Affinity version.
* @param aff Affinity assignments.
* @param updateSeq Update sequence.
- * @param exchFut Exchange future for this version.
*/
- private void createPartitions(AffinityTopologyVersion affVer, List<List<ClusterNode>> aff, long updateSeq,
- GridDhtPartitionsExchangeFuture exchFut) {
+ private void createPartitions(
+ AffinityTopologyVersion affVer,
+ List<List<ClusterNode>> aff,
+ long updateSeq
+ ) {
if (!grp.affinityNode())
return;
@@ -573,6 +578,33 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
removeNode(evt.eventNode().id());
}
}
+ else if (affReady && grpStarted && exchFut.exchangeType() == NONE) {
+ assert !exchFut.context().mergeExchanges() : exchFut;
+ assert node2part != null && node2part.valid() : exchFut;
+
+ // Initialize node maps if group was started from joining client.
+ final List<ClusterNode> nodes = exchFut.firstEventCache().cacheGroupAffinityNodes(grp.groupId());
+
+ for (ClusterNode node : nodes) {
+ if (!node2part.containsKey(node.id()) && ctx.discovery().alive(node)) {
+ final GridDhtPartitionMap partMap = new GridDhtPartitionMap(node.id(),
+ 1L,
+ exchFut.initialVersion(),
+ new GridPartitionStateMap(),
+ false);
+
+ final AffinityAssignment aff = grp.affinity().cachedAffinity(exchFut.initialVersion());
+
+ for (Integer p0 : aff.primaryPartitions(node.id()))
+ partMap.put(p0, OWNING);
+
+ for (Integer p0 : aff.backupPartitions(node.id()))
+ partMap.put(p0, OWNING);
+
+ node2part.put(node.id(), partMap);
+ }
+ }
+ }
if (grp.affinityNode()) {
if (grpStarted ||
@@ -885,6 +917,10 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (updCntr != 0)
loc.updateCounter(updCntr);
+ // Create a partition in lost state.
+ if (lostParts != null && lostParts.contains(p))
+ loc.markLost();
+
if (ctx.pageStore() != null) {
try {
ctx.pageStore().onPartitionCreated(grp.groupId(), p);
@@ -1171,7 +1207,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
try {
assert node2part != null && node2part.valid() : "Invalid node-to-partitions map [topVer1=" + topVer +
- ", topVer2=" + this.readyTopVer +
+ ", topVer2=" + readyTopVer +
", node=" + ctx.igniteInstanceName() +
", grp=" + grp.cacheOrGroupName() +
", node2part=" + node2part + ']';
@@ -1225,12 +1261,8 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
Collection<UUID> affIds = affAssignment.getIds(p);
for (UUID nodeId : diffIds) {
- if (affIds.contains(nodeId)) {
- U.warn(log, "Node from diff is affinity node, skipping it [grp=" + grp.cacheOrGroupName() +
- ", node=" + nodeId + ']');
-
+ if (affIds.contains(nodeId))
continue;
- }
if (hasState(p, nodeId, OWNING, MOVING, RENTING)) {
ClusterNode n = ctx.discovery().node(nodeId);
@@ -1415,7 +1447,9 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
Set<Integer> partsToReload,
@Nullable Map<Integer, Long> partSizes,
@Nullable AffinityTopologyVersion msgTopVer,
- @Nullable GridDhtPartitionsExchangeFuture exchFut) {
+ @Nullable GridDhtPartitionsExchangeFuture exchFut,
+ @Nullable Set<Integer> lostParts
+ ) {
if (log.isDebugEnabled()) {
log.debug("Updating full partition map " +
"[grp=" + grp.cacheOrGroupName() + ", exchVer=" + exchangeVer + ", fullMap=" + fullMapString() + ']');
@@ -1470,7 +1504,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
// TODO FIXME https://issues.apache.org/jira/browse/IGNITE-11800
if (exchangeVer != null) {
// Ignore if exchange already finished or new exchange started.
- if (readyTopVer.compareTo(exchangeVer) > 0 || lastTopChangeVer.compareTo(exchangeVer) > 0) {
+ if (readyTopVer.after(exchangeVer) || lastTopChangeVer.after(exchangeVer)) {
U.warn(log, "Stale exchange id for full partition map update (will ignore) [" +
"grp=" + grp.cacheOrGroupName() +
", lastTopChange=" + lastTopChangeVer +
@@ -1481,9 +1515,10 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
}
- boolean fullMapUpdated = (node2part == null);
+ boolean fullMapUpdated = node2part == null;
if (node2part != null) {
+ // Merge maps.
for (GridDhtPartitionMap part : node2part.values()) {
GridDhtPartitionMap newPart = partMap.get(part.nodeId());
@@ -1525,16 +1560,24 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
fullMapUpdated = !node2part.containsKey(part.nodeId());
}
- // Remove entry if node left.
- for (Iterator<UUID> it = partMap.keySet().iterator(); it.hasNext(); ) {
- UUID nodeId = it.next();
+ GridDhtPartitionsExchangeFuture topFut =
+ exchFut == null ? ctx.exchange().lastFinishedFuture() : exchFut;
+
+ // topFut can be null if lastFinishedFuture has completed with error.
+ if (topFut != null) {
+ for (Iterator<UUID> it = partMap.keySet().iterator(); it.hasNext(); ) {
+ UUID nodeId = it.next();
- if (!ctx.discovery().alive(nodeId)) {
- if (log.isTraceEnabled())
- log.trace("Removing left node from full map update [grp=" + grp.cacheOrGroupName() +
- ", nodeId=" + nodeId + ", partMap=" + partMap + ']');
+ final ClusterNode node = topFut.events().discoveryCache().node(nodeId);
- it.remove();
+ if (node == null) {
+ if (log.isTraceEnabled())
+ log.trace("Removing left node from full map update [grp=" + grp.cacheOrGroupName() +
+ ", exchTopVer=" + exchangeVer + ", futVer=" + topFut.initialVersion() +
+ ", nodeId=" + nodeId + ", partMap=" + partMap + ']');
+
+ it.remove();
+ }
}
}
}
@@ -1562,10 +1605,36 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
assert exchangeVer.compareTo(readyTopVer) >= 0 && exchangeVer.compareTo(lastTopChangeVer) >= 0;
lastTopChangeVer = readyTopVer = exchangeVer;
+
+ // Apply lost partitions from full message.
+ if (lostParts != null) {
+ this.lostParts = new HashSet<>(lostParts);
+
+ for (Integer part : lostParts) {
+ GridDhtLocalPartition locPart = localPartition(part);
+
+ // EVICTED partitions should not be marked directly as LOST, or
+ // part.clearFuture lifecycle will be broken after resetting.
+ // New partition should be created instead.
+ if (locPart != null && locPart.state() != EVICTED) {
+ locPart.markLost();
+
+ GridDhtPartitionMap locMap = partMap.get(ctx.localNodeId());
+
+ locMap.put(part, LOST);
+ }
+ }
+ }
}
node2part = partMap;
+ if (log.isDebugEnabled()) {
+ log.debug("Partition map after processFullMessage [grp=" + grp.cacheOrGroupName() +
+ ", exchId=" + (exchFut == null ? null : exchFut.exchangeId()) +
+ ", fullMap=" + fullMapString() + ']');
+ }
+
if (exchangeVer == null && !grp.isReplicated() &&
(readyTopVer.initialized() && readyTopVer.compareTo(diffFromAffinityVer) >= 0)) {
AffinityAssignment affAssignment = grp.affinity().readyAffinity(readyTopVer);
@@ -1748,7 +1817,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
try {
GridDhtPartitionState state = part.state();
- if (!reserve || state == EVICTED || state == RENTING)
+ if (!reserve || state == EVICTED || state == RENTING || state == LOST)
continue;
long updCntr = cntrMap.updateCounter(part.id());
@@ -1987,6 +2056,10 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (updateRebalanceVer)
updateRebalanceVersion(assignment.topologyVersion(), assignment.assignment());
+
+ // Own orphan moving partitions (having no suppliers).
+ if (fut != null && (fut.events().hasServerJoin() || fut.changedBaseline()))
+ ownOrphans();
}
finally {
lock.writeLock().unlock();
@@ -1994,6 +2067,38 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
/**
+ * Check if some local moving partitions have no suitable supplier and own them.
+ * This can happen if a partition has been created by affinity assignment on new node and no supplier exists.
+ * For example, if a node joins topology being a first data node for cache with node filter configured.
+ */
+ private void ownOrphans() {
+ for (int p = 0; p < grp.affinity().partitions(); p++) {
+ boolean hasOwner = false;
+
+ for (GridDhtPartitionMap map : node2part.values()) {
+ if (map.get(p) == OWNING) {
+ hasOwner = true;
+
+ break;
+ }
+ }
+
+ if (!hasOwner) {
+ GridDhtLocalPartition locPart = localPartition(p);
+
+ if (locPart != null && locPart.state() != EVICTED && locPart.state() != LOST) {
+ locPart.own();
+
+ for (GridDhtPartitionMap map : node2part.values()) {
+ if (map.get(p) != null)
+ map.put(p, OWNING);
+ }
+ }
+ }
+ }
+ }
+
+ /**
* @param aff Affinity.
*/
private void createMovingPartitions(AffinityAssignment aff) {
@@ -2078,7 +2183,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
- @Override public boolean detectLostPartitions(AffinityTopologyVersion resTopVer, @Nullable DiscoveryEvent discoEvt) {
+ @Override public boolean detectLostPartitions(AffinityTopologyVersion resTopVer, GridDhtPartitionsExchangeFuture fut) {
ctx.database().checkpointReadLock();
try {
@@ -2088,9 +2193,22 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (node2part == null)
return false;
- PartitionLossPolicy plc = grp.config().getPartitionLossPolicy();
+ // Do not trigger lost partition events on activation.
+ DiscoveryEvent discoEvt = fut.activateCluster() ? null : fut.firstEvent();
+
+ final GridClusterStateProcessor state = grp.shared().kernalContext().state();
- assert plc != null;
+ boolean isInMemoryCluster = CU.isInMemoryCluster(
+ grp.shared().kernalContext().discovery().allNodes(),
+ grp.shared().kernalContext().marshallerContext().jdkMarshaller(),
+ U.resolveClassLoader(grp.shared().kernalContext().config())
+ );
+
+ boolean compatibleWithIgnorePlc = isInMemoryCluster
+ && state.isBaselineAutoAdjustEnabled() && state.baselineAutoAdjustTimeout() == 0L;
+
+ // Calculate how data loss is handled.
+ boolean safe = grp.config().getPartitionLossPolicy() != IGNORE || !compatibleWithIgnorePlc;
int parts = grp.affinity().partitions();
@@ -2104,9 +2222,11 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (!lost) {
boolean hasOwner = false;
+ // Detect if all owners are left.
for (GridDhtPartitionMap partMap : node2part.values()) {
if (partMap.get(part) == OWNING) {
hasOwner = true;
+
break;
}
}
@@ -2114,66 +2234,73 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (!hasOwner) {
lost = true;
- if (lostParts == null)
- lostParts = new TreeSet<>();
+ // Do not detect and record lost partition in IGNORE mode.
+ if (safe) {
+ if (lostParts == null)
+ lostParts = new TreeSet<>();
- lostParts.add(part);
+ lostParts.add(part);
- if (discoEvt != null) {
- if (recentlyLost == null)
- recentlyLost = new HashSet<>();
+ if (discoEvt != null) {
+ if (recentlyLost == null)
+ recentlyLost = new HashSet<>();
- recentlyLost.add(part);
+ recentlyLost.add(part);
- if (grp.eventRecordable(EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST)) {
- grp.addRebalanceEvent(part,
- EVT_CACHE_REBALANCE_PART_DATA_LOST,
- discoEvt.eventNode(),
- discoEvt.type(),
- discoEvt.timestamp());
+ if (grp.eventRecordable(EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST)) {
+ grp.addRebalanceEvent(part,
+ EVT_CACHE_REBALANCE_PART_DATA_LOST,
+ discoEvt.eventNode(),
+ discoEvt.type(),
+ discoEvt.timestamp());
+ }
}
}
}
}
if (lost) {
- long updSeq = updateSeq.incrementAndGet();
-
GridDhtLocalPartition locPart = localPartition(part, resTopVer, false, true);
if (locPart != null) {
if (locPart.state() == LOST)
continue;
- boolean marked = plc == PartitionLossPolicy.IGNORE ? locPart.own() : locPart.markLost();
+ final GridDhtPartitionState prevState = locPart.state();
+
+ changed = safe ? locPart.markLost() : locPart.own();
+
+ if (changed) {
+ long updSeq = updateSeq.incrementAndGet();
- if (marked)
updateLocal(locPart.id(), locPart.state(), updSeq, resTopVer);
- changed |= marked;
- }
- // Update map for remote node.
- else if (plc != PartitionLossPolicy.IGNORE) {
- for (Map.Entry<UUID, GridDhtPartitionMap> e : node2part.entrySet()) {
- if (e.getKey().equals(ctx.localNodeId()))
- continue;
-
- if (e.getValue().get(part) != EVICTED)
- e.getValue().put(part, LOST);
+ // If a partition was lost while rebalancing reset it's counter to force demander mode.
+ if (prevState == MOVING)
+ locPart.resetUpdateCounter();
}
}
+
+ // Update remote maps according to policy.
+ for (Map.Entry<UUID, GridDhtPartitionMap> entry : node2part.entrySet()) {
+ if (entry.getKey().equals(ctx.localNodeId()))
+ continue;
+
+ GridDhtPartitionState p0 = entry.getValue().get(part);
+
+ if (p0 != null && p0 != EVICTED)
+ entry.getValue().put(part, safe ? LOST : OWNING);
+ }
}
}
if (recentlyLost != null) {
- U.warn(log, "Detected lost partitions [grp=" + grp.cacheOrGroupName()
+ U.warn(log, "Detected lost partitions" + (!safe ? " (will ignore)" : "")
+ + " [grp=" + grp.cacheOrGroupName()
+ ", parts=" + S.compact(recentlyLost)
- + ", plc=" + plc + ", topVer=" + resTopVer + "]");
+ + ", topVer=" + resTopVer + "]");
}
- if (lostParts != null && plc != PartitionLossPolicy.IGNORE)
- grp.needsRecovery(true);
-
return changed;
}
finally {
@@ -2211,17 +2338,13 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
updateLocal(locPart.id(), locPart.state(), updSeq, resTopVer);
// Reset counters to zero for triggering full rebalance.
- locPart.resetUpdateCounter();
+ locPart.resetInitialUpdateCounter();
}
}
}
}
- checkEvictions(updSeq, grp.affinity().readyAffinity(resTopVer));
-
lostParts = null;
-
- grp.needsRecovery(false);
}
finally {
lock.writeLock().unlock();
@@ -2233,10 +2356,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
- @Override public Collection<Integer> lostPartitions() {
- if (grp.config().getPartitionLossPolicy() == PartitionLossPolicy.IGNORE)
- return Collections.emptySet();
-
+ @Override public Set<Integer> lostPartitions() {
lock.readLock().lock();
try {
@@ -2291,7 +2411,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
}
- // Then process remote partitions.
+ // Then process node maps.
for (Map.Entry<Integer, Set<UUID>> entry : ownersByUpdCounters.entrySet()) {
int part = entry.getKey();
Set<UUID> maxCounterPartOwners = entry.getValue();
@@ -2338,6 +2458,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
U.warn(log, "Partitions have been scheduled for rebalancing due to outdated update counter "
+ "[grp=" + grp.cacheOrGroupName()
+ + ", topVer=" + exchFut.initialVersion()
+ ", nodeId=" + nodeId
+ ", partsFull=" + S.compact(rebalancedParts)
+ ", partsHistorical=" + S.compact(historical) + "]");
@@ -2466,7 +2587,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
int ownerCnt = nodeIds.size();
int affCnt = affNodes.size();
- if (ownerCnt > affCnt) { //TODO !!! we could loss all owners in such case. Should be fixed by GG-13223
+ if (ownerCnt > affCnt) {
// Sort by node orders in ascending order.
Collections.sort(nodes, CU.nodeComparator(true));
@@ -2595,21 +2716,24 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
node2part.put(locNodeId, map);
}
-
- map.updateSequence(updateSeq, affVer);
+ else
+ map.updateSequence(updateSeq, affVer);
map.put(p, state);
if (!grp.isReplicated() && (state == MOVING || state == OWNING || state == RENTING)) {
- AffinityAssignment assignment = grp.affinity().cachedAffinity(diffFromAffinityVer);
+ // The conditition below can happen during partition recovery from persistent store.
+ if (diffFromAffinityVer != AffinityTopologyVersion.NONE) {
+ AffinityAssignment assignment = grp.affinity().cachedAffinity(diffFromAffinityVer);
- if (!assignment.getIds(p).contains(ctx.localNodeId())) {
- Set<UUID> diffIds = diffFromAffinity.get(p);
+ if (!assignment.getIds(p).contains(ctx.localNodeId())) {
+ Set<UUID> diffIds = diffFromAffinity.get(p);
- if (diffIds == null)
- diffFromAffinity.put(p, diffIds = U.newHashSet(3));
+ if (diffIds == null)
+ diffFromAffinity.put(p, diffIds = U.newHashSet(3));
- diffIds.add(ctx.localNodeId());
+ diffIds.add(ctx.localNodeId());
+ }
}
}
}
@@ -2696,7 +2820,8 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
", lastAffChangeVer=" + lastAffChangeVer + "]");
}
- grp.preloader().forceRebalance();
+ if (!((GridDhtPreloader)grp.preloader()).disableRebalancingCancellationOptimization())
+ grp.preloader().forceRebalance();
return;
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java
index e2d28c2..aae7559 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java
@@ -447,6 +447,13 @@ public class PartitionsEvictManager extends GridCacheSharedManagerAdapter {
}
/**
+ * Awaits all futures.
+ */
+ public void awaitFinishAll() {
+ evictionGroupsMap.values().forEach(GroupEvictionContext::awaitFinishAll);
+ }
+
+ /**
* Task for self-scheduled partition eviction / clearing.
*/
class PartitionEvictionTask implements Runnable {
@@ -508,7 +515,9 @@ public class PartitionsEvictManager extends GridCacheSharedManagerAdapter {
finishFut.onDone(ex);
if (cctx.kernalContext().isStopping()) {
- LT.warn(log, ex, "Partition eviction failed (current node is stopping).",
+ LT.warn(log, ex, "Partition eviction failed (current node is stopping) " +
+ "[grp=" + grpEvictionCtx.grp.cacheOrGroupName() +
+ ", readyVer=" + grpEvictionCtx.grp.topology().readyTopologyVersion() + ']',
false,
true);
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 792b62d..498cbeea5 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -211,6 +211,8 @@ import static org.apache.ignite.internal.pagemem.PageIdUtils.partId;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MASTER_KEY_CHANGE_RECORD;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.METASTORE_DATA_RECORD;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.fromOrdinal;
import static org.apache.ignite.internal.processors.cache.persistence.CheckpointState.FINISHED;
import static org.apache.ignite.internal.processors.cache.persistence.CheckpointState.LOCK_RELEASED;
@@ -1853,7 +1855,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
continue;
for (GridDhtLocalPartition locPart : grp.topology().currentLocalPartitions()) {
- if (locPart.state() == GridDhtPartitionState.OWNING && locPart.fullSize() > walRebalanceThreshold)
+ if (locPart.state() == OWNING && locPart.fullSize() > walRebalanceThreshold)
res.computeIfAbsent(grp.groupId(), k -> new HashSet<>()).add(locPart.id());
}
}
@@ -4189,11 +4191,16 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
CacheState state = new CacheState(parts.size());
for (GridDhtLocalPartition part : parts) {
+ GridDhtPartitionState partState = part.state();
+
+ if (partState == LOST)
+ partState = OWNING;
+
state.addPartitionState(
part.id(),
part.dataStore().fullSize(),
part.updateCounter(),
- (byte)part.state().ordinal()
+ (byte)partState.ordinal()
);
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
index ee43a60..d3a961d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
@@ -30,11 +30,11 @@ import java.util.NoSuchElementException;
import java.util.Queue;
import java.util.Set;
import java.util.UUID;
+import javax.cache.CacheException;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.PartitionLossPolicy;
import org.apache.ignite.cache.query.Query;
import org.apache.ignite.cluster.ClusterGroup;
import org.apache.ignite.cluster.ClusterNode;
@@ -44,7 +44,9 @@ import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.cluster.ClusterGroupEmptyCheckedException;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtUnreservedPartitionException;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
@@ -558,6 +560,20 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
@Override public GridCloseableIterator executeScanQuery() throws IgniteCheckedException {
assert type == SCAN : "Wrong processing of query: " + type;
+ if (!cctx.isLocal()) {
+ GridDhtCacheAdapter<?, ?> cacheAdapter = cctx.isNear() ? cctx.near().dht() : cctx.dht();
+
+ Set<Integer> lostParts = cacheAdapter.topology().lostPartitions();
+
+ if (!lostParts.isEmpty()) {
+ if (part == null || lostParts.contains(part)) {
+ throw new CacheException(new CacheInvalidStateException("Failed to execute query because cache partition " +
+ "has been lostParts [cacheName=" + cctx.name() +
+ ", part=" + (part == null ? lostParts.iterator().next() : part) + ']'));
+ }
+ }
+ }
+
// Affinity nodes snapshot.
Collection<ClusterNode> nodes = new ArrayList<>(nodes());
@@ -567,12 +583,7 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
if (part != null) {
if (forceLocal) {
throw new IgniteCheckedException("No queryable nodes for partition " + part
- + " [forced local query=" + this + "]");
- }
-
- if (isSafeLossPolicy()) {
- throw new IgniteCheckedException("Failed to execute scan query because cache partition has been " +
- "lost [cacheName=" + cctx.name() + ", part=" + part + "]");
+ + " [forced local query=" + this + "]");
}
}
@@ -623,16 +634,6 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
}
/**
- * @return true if current PartitionLossPolicy corresponds to *_SAFE values.
- */
- private boolean isSafeLossPolicy() {
- PartitionLossPolicy lossPlc = cctx.cache().configuration().getPartitionLossPolicy();
-
- return lossPlc == PartitionLossPolicy.READ_ONLY_SAFE ||
- lossPlc == PartitionLossPolicy.READ_WRITE_SAFE;
- }
-
- /**
* @return Nodes to execute on.
*/
private Collection<ClusterNode> nodes() throws IgniteCheckedException {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 2c498a0..784f705 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -63,6 +63,7 @@ import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
import org.apache.ignite.internal.processors.cache.CacheMetricsImpl;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.CacheObjectContext;
@@ -133,6 +134,7 @@ import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_OBJECT_READ;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.internal.GridClosureCallMode.BROADCAST;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SCAN;
import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SPI;
@@ -798,11 +800,10 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
final InternalScanFilter<K, V> intFilter = keyValFilter != null ? new InternalScanFilter<>(keyValFilter) : null;
try {
- if (keyValFilter instanceof PlatformCacheEntryFilter) {
+ if (keyValFilter instanceof PlatformCacheEntryFilter)
((PlatformCacheEntryFilter)keyValFilter).cacheContext(cctx);
- } else {
+ else
injectResources(keyValFilter);
- }
Integer part = cctx.isLocal() ? null : qry.partition();
@@ -832,9 +833,13 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
GridDhtLocalPartition locPart0 = dht.topology().localPartition(part, topVer, false);
- if (locPart0 == null || locPart0.state() != OWNING || !locPart0.reserve())
- throw new GridDhtUnreservedPartitionException(part, cctx.affinity().affinityTopologyVersion(),
- "Partition can not be reserved");
+ if (locPart0 == null || locPart0.state() != OWNING || !locPart0.reserve()) {
+ throw locPart0 != null && locPart0.state() == LOST ?
+ new CacheInvalidStateException("Failed to execute scan query because cache partition has been " +
+ "lost [cacheName=" + cctx.name() + ", part=" + part + "]") :
+ new GridDhtUnreservedPartitionException(part, cctx.affinity().affinityTopologyVersion(),
+ "Partition can not be reserved");
+ }
locPart = locPart0;
@@ -844,6 +849,17 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
else {
locPart = null;
+ if (!cctx.isLocal()) {
+ final GridDhtCacheAdapter dht = cctx.isNear() ? cctx.near().dht() : cctx.dht();
+
+ Set<Integer> lostParts = dht.topology().lostPartitions();
+
+ if (!lostParts.isEmpty()) {
+ throw new CacheInvalidStateException("Failed to execute scan query because cache partition " +
+ "has been lost [cacheName=" + cctx.name() + ", part=" + lostParts.iterator().next() + "]");
+ }
+ }
+
it = cctx.offheap().cacheIterator(cctx.cacheId(), true, backups, topVer,
qry.mvccSnapshot(), qry.isDataPageScanEnabled());
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 57c210b..a2bd1e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -110,7 +110,6 @@ import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRA
import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isNearEnabled;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
import static org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx.FinalizationStatus.USER_FINISH;
-import static org.apache.ignite.internal.util.lang.GridFunc.isEmpty;
import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
import static org.apache.ignite.transactions.TransactionState.PREPARED;
@@ -628,26 +627,10 @@ public class IgniteTxHandler {
return;
}
ctx.kernalContext().closure().runLocalWithThreadPolicy(thread, () -> {
- IgniteInternalFuture<GridNearTxPrepareResponse> fut = null;
-
- Throwable err = null;
-
try {
- for (IgniteTxEntry itm : F.concat(false, req.writes(), req.reads())) {
- err = topFut.validateCache(itm.context(), req.recovery(), isEmpty(req.writes()),
- null, null);
-
- if (err != null)
- break;
- }
-
- if (err == null)
- fut = processNearTxPrepareRequest0(node, req);
+ processNearTxPrepareRequest0(node, req);
}
finally {
- if (fut == null || fut.error() != null || err != null)
- sendResponseOnTimeoutOrError(e, topFut, node, req);
-
ctx.io().onMessageProcessed(req);
}
});
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java
index d9d2099..9f4080c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java
@@ -30,6 +30,7 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.cache.CacheInterceptor;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
import org.apache.ignite.internal.processors.cache.CacheStoppedException;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
@@ -149,8 +150,6 @@ public class IgniteTxStateImpl extends IgniteTxLocalStateAdapter {
set.add(key.key());
}
- StringBuilder invalidCaches = null;
-
for (Map.Entry<Integer, Set<KeyCacheObject>> e : keysByCacheId.entrySet()) {
int cacheId = e.getKey();
@@ -158,21 +157,10 @@ public class IgniteTxStateImpl extends IgniteTxLocalStateAdapter {
assert ctx != null : cacheId;
- Throwable err = topFut.validateCache(ctx, recovery(), read, null, e.getValue());
-
- if (err != null) {
- if (invalidCaches != null)
- invalidCaches.append(", ");
- else
- invalidCaches = new StringBuilder();
-
- invalidCaches.append(U.maskName(ctx.name()));
- }
- }
+ CacheInvalidStateException err = topFut.validateCache(ctx, recovery(), read, null, e.getValue());
- if (invalidCaches != null) {
- return new IgniteCheckedException("Failed to perform cache operation (cache topology is not valid): " +
- invalidCaches);
+ if (err != null)
+ return err;
}
for (int i = 0; i < activeCacheIds.size(); i++) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCachePartitionWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCachePartitionWorker.java
index be8eba5..529d93f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCachePartitionWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCachePartitionWorker.java
@@ -42,6 +42,7 @@ import static java.util.Objects.nonNull;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXTRA_INDEX_REBUILD_LOGGING;
import static org.apache.ignite.IgniteSystemProperties.getBoolean;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
@@ -157,7 +158,7 @@ public class SchemaIndexCachePartitionWorker extends GridWorker {
GridDhtPartitionState partState = locPart.state();
if (partState != EVICTED)
- reserved = (partState == OWNING || partState == RENTING || partState == MOVING) && locPart.reserve();
+ reserved = (partState == OWNING || partState == MOVING || partState == LOST) && locPart.reserve();
if (!reserved)
return;
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/ClientCreateCacheGroupOnJoinNodeMapsTest.java b/modules/core/src/test/java/org/apache/ignite/cache/ClientCreateCacheGroupOnJoinNodeMapsTest.java
new file mode 100644
index 0000000..389fd93
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/ClientCreateCacheGroupOnJoinNodeMapsTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache;
+
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+/**
+ * Tests if node maps are correct for caches created on client node join.
+ */
+public class ClientCreateCacheGroupOnJoinNodeMapsTest extends GridCommonAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ final IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setActiveOnStart(false);
+
+ boolean client = "client".equals(igniteInstanceName);
+
+ cfg.setClientMode(client);
+
+ if (client) {
+ cfg.setCacheConfiguration(defaultCacheConfiguration().setNearConfiguration(null).
+ setAffinity(new RendezvousAffinityFunction(false, 32)));
+ }
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** */
+ @Test
+ public void testNodeMapsVolatile() throws Exception {
+ final IgniteEx srv = startGrids(2);
+
+ srv.cluster().active(true);
+
+ awaitPartitionMapExchange();
+
+ final IgniteEx client = startGrid("client");
+
+ final GridDhtPartitionTopology top0 = grid(0).cachex(DEFAULT_CACHE_NAME).context().topology();
+ final GridDhtPartitionTopology top1 = grid(1).cachex(DEFAULT_CACHE_NAME).context().topology();
+
+ GridDhtPartitionFullMap map0 = U.field(top0, "node2part");
+ GridDhtPartitionFullMap map1 = U.field(top1, "node2part");
+
+ assertEquals(2, map0.size());
+ assertEquals(2, map1.size());
+
+ srv.cluster().active(false);
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/RebalanceAfterResettingLostPartitionTest.java b/modules/core/src/test/java/org/apache/ignite/cache/RebalanceAfterResettingLostPartitionTest.java
new file mode 100644
index 0000000..2cbc289
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/RebalanceAfterResettingLostPartitionTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache;
+
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class RebalanceAfterResettingLostPartitionTest extends GridCommonAbstractTest {
+ /** Cache name. */
+ private static final String CACHE_NAME = "cache" + UUID.randomUUID().toString();
+
+ /** Cache size */
+ public static final int CACHE_SIZE = 10_000;
+
+ /** Stop all grids and cleanup persistence directory. */
+ @Before
+ public void before() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** Stop all grids and cleanup persistence directory. */
+ @After
+ public void after() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+ cfg.setRebalanceBatchSize(100);
+
+ cfg.setConsistentId(igniteInstanceName);
+
+ DataStorageConfiguration storageCfg = new DataStorageConfiguration().setWalSegmentSize(4 * 1024 * 1024);
+
+ storageCfg.getDefaultDataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setMaxSize(500L * 1024 * 1024);
+
+ cfg.setDataStorageConfiguration(storageCfg);
+
+ cfg.setCacheConfiguration(new CacheConfiguration()
+ .setName(CACHE_NAME)
+ .setCacheMode(CacheMode.PARTITIONED)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setAffinity(new RendezvousAffinityFunction(false, 32))
+ .setBackups(1));
+
+ return cfg;
+ }
+
+ /**
+ * Test to restore lost partitions and rebalance data on working grid with two nodes.
+ *
+ * @throws Exception if fail.
+ */
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testRebalanceAfterPartitionsWereLost() throws Exception {
+ startGrids(2);
+
+ grid(0).cluster().active(true);
+
+ for (int j = 0; j < CACHE_SIZE; j++)
+ grid(0).cache(CACHE_NAME).put(j, "Value" + j);
+
+ String g1Name = grid(1).name();
+
+ // Stopping the the second node.
+ stopGrid(1);
+
+ // Cleaning the persistence for second node.
+ cleanPersistenceDir(g1Name);
+
+ AtomicInteger msgCntr = new AtomicInteger();
+
+ TestRecordingCommunicationSpi.spi(ignite(0)).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+ @Override public boolean apply(ClusterNode clusterNode, Message msg) {
+ if (msg instanceof GridDhtPartitionSupplyMessage &&
+ ((GridCacheGroupIdMessage)msg).groupId() == CU.cacheId(CACHE_NAME)) {
+ if (msgCntr.get() > 3)
+ return true;
+ else
+ msgCntr.incrementAndGet();
+ }
+
+ return false;
+ }
+ });
+
+ // Starting second node again(with the same consistent id).
+ startGrid(1);
+
+ // Waitting for rebalance.
+ TestRecordingCommunicationSpi.spi(ignite(0)).waitForBlocked();
+
+ // Killing the first node at the moment of rebalancing.
+ stopGrid(0);
+
+ // Returning first node to the cluster.
+ IgniteEx g0 = startGrid(0);
+
+ assertTrue(Objects.requireNonNull(
+ grid(0).cachex(CACHE_NAME)).context().topology().localPartitions().stream().allMatch(
+ p -> p.state() == GridDhtPartitionState.LOST));
+
+ // Verify that partition loss is detected.
+ assertTrue(Objects.requireNonNull(
+ grid(1).cachex(CACHE_NAME)).context().topology().localPartitions().stream().allMatch(
+ p -> p.state() == GridDhtPartitionState.LOST));
+
+ // Reset lost partitions and wait for PME.
+ grid(1).resetLostPartitions(Arrays.asList(CACHE_NAME, "ignite-sys-cache"));
+
+ awaitPartitionMapExchange();
+
+ assertTrue(Objects.requireNonNull(
+ grid(0).cachex(CACHE_NAME)).context().topology().localPartitions().stream().allMatch(
+ p -> p.state() == GridDhtPartitionState.OWNING));
+
+ // Verify that partitions are in owning state.
+ assertTrue(Objects.requireNonNull(
+ grid(1).cachex(CACHE_NAME)).context().topology().localPartitions().stream().allMatch(
+ p -> p.state() == GridDhtPartitionState.OWNING));
+
+ // Verify that data was successfully rebalanced.
+ for (int i = 0; i < CACHE_SIZE; i++)
+ assertEquals("Value" + i, grid(0).cache(CACHE_NAME).get(i));
+
+ for (int i = 0; i < CACHE_SIZE; i++)
+ assertEquals("Value" + i, grid(1).cache(CACHE_NAME).get(i));
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/ResetLostPartitionTest.java b/modules/core/src/test/java/org/apache/ignite/cache/ResetLostPartitionTest.java
index 6a99c57..87ef18d 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/ResetLostPartitionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/ResetLostPartitionTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.cache;
import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.ignite.Ignite;
@@ -31,13 +32,12 @@ import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl;
+import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Test;
import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
/**
*
@@ -75,7 +75,7 @@ public class ResetLostPartitionTest extends GridCommonAbstractTest {
DataStorageConfiguration storageCfg = new DataStorageConfiguration();
- storageCfg.setPageSize(1024).setWalMode(LOG_ONLY).setWalSegmentSize(8 * 1024 * 1024);
+ storageCfg.setPageSize(1024).setWalMode(LOG_ONLY).setWalSegmentSize(4 * 1024 * 1024);
storageCfg.getDefaultDataRegionConfiguration()
.setPersistenceEnabled(true)
@@ -164,41 +164,40 @@ public class ResetLostPartitionTest extends GridCommonAbstractTest {
//Start node 3.
startGrid(2);
- //Loss data expected because rebalance to node 1 have not finished and node 2 was stopped.
+ // Data loss is expected because rebalance to node 1 have not finished and node 2 was stopped.
assertTrue(CACHE_NAMES.length * CACHE_SIZE > averageSizeAroundAllNodes());
+ // Check all nodes report same lost partitions.
for (String cacheName : CACHE_NAMES) {
- //Node 1 will have only OWNING partitions.
- assertTrue(getPartitionsStates(0, cacheName).stream().allMatch(state -> state == OWNING));
+ Collection<Integer> lost = null;
- //Node 3 will have only OWNING partitions.
- assertTrue(getPartitionsStates(2, cacheName).stream().allMatch(state -> state == OWNING));
- }
+ for (Ignite grid : G.allGrids()) {
+ if (lost == null)
+ lost = grid.cache(cacheName).lostPartitions();
+ else
+ assertEquals(lost, grid.cache(cacheName).lostPartitions());
+ }
- boolean hasLost = false;
- for (String cacheName : CACHE_NAMES) {
- //Node 2 will have OWNING and LOST partitions.
- hasLost |= getPartitionsStates(1, cacheName).stream().anyMatch(state -> state == LOST);
+ assertTrue(lost != null && !lost.isEmpty());
}
- assertTrue(hasLost);
-
if (reactivateGridBeforeResetPart) {
grid(0).cluster().active(false);
grid(0).cluster().active(true);
}
- //Try to reset lost partitions.
+ // Try to reset lost partitions.
grid(2).resetLostPartitions(Arrays.asList(CACHE_NAMES));
awaitPartitionMapExchange();
+ // Check all nodes report same lost partitions.
for (String cacheName : CACHE_NAMES) {
- //Node 2 will have only OWNING partitions.
- assertTrue(getPartitionsStates(1, cacheName).stream().allMatch(state -> state == OWNING));
+ for (Ignite grid : G.allGrids())
+ assertTrue(grid.cache(cacheName).lostPartitions().isEmpty());
}
- //All data was back.
+ // All data was back.
assertEquals(CACHE_NAMES.length * CACHE_SIZE, averageSizeAroundAllNodes());
//Stop node 2 for checking rebalance correctness from this node.
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java b/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java
index df1ddb8..8a84af8 100644
--- a/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java
@@ -43,6 +43,9 @@ public class SystemWorkersBlockingTest extends GridCommonAbstractTest {
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+ // Set small value for the test.
+ cfg.setSystemWorkerBlockedTimeout(1_000);
+
AbstractFailureHandler failureHnd = new AbstractFailureHandler() {
@Override protected boolean handle(Ignite ignite, FailureContext failureCtx) {
if (failureCtx.type() == FailureType.SYSTEM_WORKER_BLOCKED)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestDelayingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestDelayingCommunicationSpi.java
index 4068c6a..de6808f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/TestDelayingCommunicationSpi.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/TestDelayingCommunicationSpi.java
@@ -25,12 +25,11 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.spi.IgniteSpiException;
-import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
/**
*
*/
-public abstract class TestDelayingCommunicationSpi extends TcpCommunicationSpi {
+public abstract class TestDelayingCommunicationSpi extends TestRecordingCommunicationSpi {
/** {@inheritDoc} */
@Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC)
throws IgniteSpiException {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/encryption/MasterKeyChangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/encryption/MasterKeyChangeTest.java
index 1b9c6c4..9b52bfc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/encryption/MasterKeyChangeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/encryption/MasterKeyChangeTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.encryption;
import java.io.Serializable;
+import java.util.Collections;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.ignite.IgniteCache;
@@ -107,6 +108,8 @@ public class MasterKeyChangeTest extends AbstractEncryptionTest {
assertTrue(checkMasterKeyName(MASTER_KEY_NAME_2));
+ grid(GRID_0).resetLostPartitions(Collections.singleton(ENCRYPTED_CACHE));
+
checkEncryptedCaches(grids.get1(), ignite);
stopGrid(GRID_1);
@@ -119,6 +122,8 @@ public class MasterKeyChangeTest extends AbstractEncryptionTest {
assertTrue(checkMasterKeyName(MASTER_KEY_NAME_2));
+ grid(GRID_0).resetLostPartitions(Collections.singleton(ENCRYPTED_CACHE));
+
checkEncryptedCaches(grids.get1(), grid1);
}
@@ -333,6 +338,8 @@ public class MasterKeyChangeTest extends AbstractEncryptionTest {
IgniteEx grid = startGrid(GRID_0);
+ grid(GRID_1).resetLostPartitions(Collections.singleton(ENCRYPTED_CACHE));
+
// 8. Check that restarted node recoveries keys from WAL. Check data.
assertEquals(MASTER_KEY_NAME_2, grid(GRID_0).encryption().getMasterKeyName());
assertEquals(MASTER_KEY_NAME_2, grid(GRID_1).encryption().getMasterKeyName());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsManageTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsManageTest.java
index 314f5db..271768a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsManageTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsManageTest.java
@@ -408,6 +408,9 @@ public class CacheMetricsManageTest extends GridCommonAbstractTest {
// Start node 1 again.
startGrid(1);
+ if (persistence)
+ ig2.resetLostPartitions(Arrays.asList(CACHE1, CACHE2));
+
assertCachesStatisticsMode(true, true);
int gridIdx = 0;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
index 956cd11..2c527a5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
@@ -28,6 +28,7 @@ import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.TopologyValidator;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Test;
@@ -46,12 +47,14 @@ public class CacheValidatorMetricsTest extends GridCommonAbstractTest implements
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+ cfg.setActiveOnStart(false);
+
CacheConfiguration cCfg1 = new CacheConfiguration()
.setName(CACHE_NAME_1)
.setCacheMode(CacheMode.PARTITIONED)
.setBackups(0)
.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
- .setPartitionLossPolicy(PartitionLossPolicy.READ_ONLY_ALL);
+ .setPartitionLossPolicy(PartitionLossPolicy.READ_ONLY_SAFE);
CacheConfiguration cCfg2 = new CacheConfiguration()
.setName(CACHE_NAME_2)
@@ -96,12 +99,15 @@ public class CacheValidatorMetricsTest extends GridCommonAbstractTest implements
*/
@Test
public void testCacheValidatorMetrics() throws Exception {
- startGrid(1);
+ final IgniteEx crd = startGrid(1);
+ crd.cluster().active(true);
+ crd.cluster().baselineAutoAdjustEnabled(false);
assertCacheStatus(CACHE_NAME_1, true, true);
assertCacheStatus(CACHE_NAME_2, true, false);
startGrid(2);
+ resetBaselineTopology();
awaitPartitionMapExchange();
@@ -112,10 +118,10 @@ public class CacheValidatorMetricsTest extends GridCommonAbstractTest implements
awaitPartitionMapExchange();
- // Invalid for writing due to invalid topology.
- assertCacheStatus(CACHE_NAME_1, true, false);
-
// Invalid for writing due to partitions loss.
+ assertCacheStatus(CACHE_NAME_1, false, false);
+
+ // Invalid for writing due to invalid topology.
assertCacheStatus(CACHE_NAME_2, true, false);
}
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheWithDifferentDataRegionConfigurationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheWithDifferentDataRegionConfigurationTest.java
index f45511c..e85d94c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheWithDifferentDataRegionConfigurationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheWithDifferentDataRegionConfigurationTest.java
@@ -174,8 +174,6 @@ public class CacheWithDifferentDataRegionConfigurationTest extends GridCommonAbs
.andExclusiveCache(CACHE_2, REGION_2)
.start();
- node1.cluster().active(true);
-
populateCache(node1, CACHE_1, 1000);
populateCache(node2, CACHE_2, 350);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePartitionMapUpdateSafeLossPolicyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePartitionMapUpdateSafeLossPolicyTest.java
new file mode 100644
index 0000000..5e7a4e7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePartitionMapUpdateSafeLossPolicyTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cache.PartitionLossPolicy;
+import org.apache.ignite.cluster.ClusterTopologyException;
+import org.apache.ignite.internal.util.typedef.G;
+
+/**
+ *
+ */
+public class IgniteCachePartitionMapUpdateSafeLossPolicyTest extends IgniteCachePartitionMapUpdateTest {
+ /** {@inheritDoc} */
+ @Override protected PartitionLossPolicy policy() {
+ return PartitionLossPolicy.READ_WRITE_SAFE;
+ }
+
+ /** Lost partitions counter. */
+ private AtomicInteger lostCnt = new AtomicInteger();
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ lostCnt.set(0);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ assertTrue(lostCnt.get() > 0);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void stopGrid(int idx) {
+ List<Ignite> grids0 = G.allGrids();
+
+ super.stopGrid(idx);
+
+ if (grids0.size() == 1)
+ return;
+
+ Stream.of(CACHE1, CACHE2).forEach(new Consumer<String>() {
+ @Override public void accept(String cache) {
+ Ignite testGrid = G.allGrids().get(0);
+
+ lostCnt.addAndGet(testGrid.cache(cache).lostPartitions().size());
+
+ try {
+ testGrid.resetLostPartitions(Collections.singleton(cache));
+ } catch (ClusterTopologyException ignored) {
+ // No-op.
+ }
+ }
+ });
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePartitionMapUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePartitionMapUpdateTest.java
index d38d908..7c52d53 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePartitionMapUpdateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePartitionMapUpdateTest.java
@@ -22,11 +22,13 @@ import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ThreadLocalRandom;
+import java.util.Random;
+import org.apache.ignite.cache.PartitionLossPolicy;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Test;
@@ -44,10 +46,10 @@ public class IgniteCachePartitionMapUpdateTest extends GridCommonAbstractTest {
private static final String CACHE2_ATTR = "cache2";
/** */
- private static final String CACHE1 = "cache1";
+ protected static final String CACHE1 = "cache1";
/** */
- private static final String CACHE2 = "cache2";
+ protected static final String CACHE2 = "cache2";
/** */
private boolean startClientCache;
@@ -62,17 +64,17 @@ public class IgniteCachePartitionMapUpdateTest extends GridCommonAbstractTest {
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
- CacheConfiguration ccfg1 = new CacheConfiguration(DEFAULT_CACHE_NAME);
+ CacheConfiguration ccfg1 = new CacheConfiguration(CACHE1);
- ccfg1.setName(CACHE1);
ccfg1.setCacheMode(PARTITIONED);
ccfg1.setBackups(1);
+ ccfg1.setPartitionLossPolicy(policy());
ccfg1.setNodeFilter(new AttributeFilter(CACHE1_ATTR));
- CacheConfiguration ccfg2 = new CacheConfiguration(DEFAULT_CACHE_NAME);
+ CacheConfiguration ccfg2 = new CacheConfiguration(CACHE2);
- ccfg2.setName(CACHE2);
ccfg2.setCacheMode(PARTITIONED);
+ ccfg2.setPartitionLossPolicy(policy());
ccfg2.setNodeFilter(new AttributeFilter(CACHE2_ATTR));
List<CacheConfiguration> ccfgs = new ArrayList<>();
@@ -162,7 +164,7 @@ public class IgniteCachePartitionMapUpdateTest extends GridCommonAbstractTest {
*/
@Test
public void testRandom() throws Exception {
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
+ Random rnd = new Random();
final int NODE_CNT = 10;
@@ -183,7 +185,9 @@ public class IgniteCachePartitionMapUpdateTest extends GridCommonAbstractTest {
LinkedHashSet<Integer> stopSeq = new LinkedHashSet<>();
while (stopSeq.size() != NODE_CNT)
- stopSeq.add(rnd.nextInt(0, NODE_CNT));
+ stopSeq.add(rnd.nextInt(NODE_CNT));
+
+ log.info("Stop sequence: " + stopSeq + ", seed=" + U.field(rnd, "seed"));
for (Integer idx : stopSeq) {
log.info("Stop node: " + idx);
@@ -206,6 +210,54 @@ public class IgniteCachePartitionMapUpdateTest extends GridCommonAbstractTest {
}
/**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testNodeLeft() throws Exception {
+ cache1 = false;
+ cache2 = false;
+
+ startGrid(0);
+
+ awaitPartitionMapExchange();
+
+ cache1 = true;
+ cache2 = true;
+
+ startGrid(1);
+
+ awaitPartitionMapExchange();
+
+ cache1 = true;
+ cache2 = true;
+
+ startGrid(2);
+
+ awaitPartitionMapExchange();
+
+ stopGrid(1);
+
+ awaitPartitionMapExchange();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testNodeLeft2() throws Exception {
+ startClientCache = true;
+
+ testNodeLeft();
+ }
+
+ /**
+ * @return Policy.
+ */
+ protected PartitionLossPolicy policy() {
+ return PartitionLossPolicy.IGNORE;
+ }
+
+ /**
*
*/
static class AttributeFilter implements IgnitePredicate<ClusterNode> {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 4819936..be0b976 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -1536,6 +1536,8 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
ignite(0).cluster().state(initialState);
}
+ ignite(0).resetLostPartitions(Arrays.asList(CACHE_NAME_PREFIX + "0", CACHE_NAME_PREFIX + "1"));
+
checkCaches(nodesCnt);
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
index 2308399..f696e09 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
@@ -60,6 +60,7 @@ import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents;
import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsAbstractMessage;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
@@ -774,7 +775,7 @@ public class CacheExchangeMergeTest extends GridCommonAbstractTest {
*/
@Test
public void testMergeServersFail1_1() throws Exception {
- mergeServersFail1(false);
+ mergeServersFail1(false, false, 8);
}
/**
@@ -782,35 +783,117 @@ public class CacheExchangeMergeTest extends GridCommonAbstractTest {
*/
@Test
public void testMergeServersFail1_2() throws Exception {
- mergeServersFail1(true);
+ mergeServersFail1(true, false, 8);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testMergeServersFail1_3() throws Exception {
+ mergeServersFail1(false, true, 8);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testMergeServersFail1_4() throws Exception {
+ mergeServersFail1(true, true, 8);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testMergeServersFail1_5() throws Exception {
+ mergeServersFail1(false, false, 7);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testMergeServersFail1_6() throws Exception {
+ mergeServersFail1(true, false, 7);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testMergeServersFail1_7() throws Exception {
+ mergeServersFail1(false, true, 7);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testMergeServersFail1_8() throws Exception {
+ mergeServersFail1(true, true, 7);
}
/**
* @param waitRebalance Wait for rebalance end before start tested topology change.
+ * @param delayRebalance Delay rebalancing before checking caches.
+ * @param mergeTopVer Merge topology version (7 or 8).
* @throws Exception If failed.
*/
- private void mergeServersFail1(boolean waitRebalance) throws Exception {
+ private void mergeServersFail1(boolean waitRebalance, boolean delayRebalance, int mergeTopVer) throws Exception {
+ testSpi = true;
+
final Ignite srv0 = startGrids(5);
if (waitRebalance)
awaitPartitionMapExchange();
+ if (delayRebalance) {
+ for (Ignite allGrid : G.allGrids()) {
+ TestRecordingCommunicationSpi.spi(allGrid).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+ @Override public boolean apply(ClusterNode clusterNode, Message msg) {
+ return msg instanceof GridDhtPartitionDemandMessage;
+ }
+ });
+ }
+ }
+
final List<DiscoveryEvent> mergedEvts = new ArrayList<>();
- mergeExchangeWaitVersion(srv0, 8, mergedEvts);
+ mergeExchangeWaitVersion(srv0, mergeTopVer, mergedEvts);
UUID grid3Id = grid(3).localNode().id();
UUID grid2Id = grid(2).localNode().id();
stopGrid(getTestIgniteInstanceName(4), true, false);
stopGrid(getTestIgniteInstanceName(3), true, false);
+
+ if (mergeTopVer == 7) {
+ waitForReadyTopology(grid(0).cachex(cacheNames[0]).context().topology(),
+ new AffinityTopologyVersion(7, 0));
+ }
+
stopGrid(getTestIgniteInstanceName(2), true, false);
- checkCaches();
+ checkAffinity();
+
+ checkCaches0();
+
+ checkAffinity();
+
+ if (delayRebalance) {
+ for (Ignite allGrid : G.allGrids())
+ TestRecordingCommunicationSpi.spi(allGrid).stopBlock();
+ }
awaitPartitionMapExchange();
- assertTrue("Unexpected number of merged disco events: " + mergedEvts.size(), mergedEvts.size() == 2);
+ checkTopologiesConsistency();
+
+ checkCaches0();
+
+ assertTrue("Unexpected number of merged disco events: " + mergedEvts.size(),
+ mergedEvts.size() == mergeTopVer - 6);
for (DiscoveryEvent discoEvt : mergedEvts) {
ClusterNode evtNode = discoEvt.eventNode();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossDetectionOnNodeLeftTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossDetectionOnNodeLeftTest.java
deleted file mode 100644
index 88792ec..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossDetectionOnNodeLeftTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.distributed;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.events.CacheRebalancingEvent;
-import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.junit.Test;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
-import static org.apache.ignite.cache.PartitionLossPolicy.IGNORE;
-import static org.apache.ignite.testframework.GridTestUtils.mergeExchangeWaitVersion;
-
-/**
- * Tests if lost partitions are same on left nodes after other owners removal.
- */
-public class CachePartitionLossDetectionOnNodeLeftTest extends GridCommonAbstractTest {
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
-
- cfg.setConsistentId(igniteInstanceName);
-
- cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME).
- setAtomicityMode(TRANSACTIONAL).
- setCacheMode(PARTITIONED).
- // If default will ever change...
- setPartitionLossPolicy(IGNORE).
- setBackups(0));
-
- cfg.setIncludeEventTypes(EventType.EVTS_ALL);
-
- return cfg;
- }
-
- /**
- *
- */
- @Test
- public void testPartitionLossDetectionOnNodeLeft() throws Exception {
- try {
- final Ignite srv0 = startGrids(5);
-
- List<Integer> lost0 = Collections.synchronizedList(new ArrayList<>());
- List<Integer> lost1 = Collections.synchronizedList(new ArrayList<>());
-
- grid(0).events().localListen(evt -> {
- lost0.add(((CacheRebalancingEvent)evt).partition());
-
- return true;
- }, EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST);
-
- grid(1).events().localListen(evt -> {
- lost1.add(((CacheRebalancingEvent)evt).partition());
-
- return true;
- }, EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST);
-
- awaitPartitionMapExchange();
-
- mergeExchangeWaitVersion(srv0, 8, null);
-
- int[] p2 = srv0.affinity(DEFAULT_CACHE_NAME).primaryPartitions(grid(2).localNode());
- int[] p3 = srv0.affinity(DEFAULT_CACHE_NAME).primaryPartitions(grid(3).localNode());
- int[] p4 = srv0.affinity(DEFAULT_CACHE_NAME).primaryPartitions(grid(4).localNode());
-
- List<Integer> expLostParts = new ArrayList<>();
-
- for (int i = 0; i < p2.length; i++)
- expLostParts.add(p2[i]);
- for (int i = 0; i < p3.length; i++)
- expLostParts.add(p3[i]);
- for (int i = 0; i < p4.length; i++)
- expLostParts.add(p4[i]);
-
- Collections.sort(expLostParts);
-
- stopGrid(getTestIgniteInstanceName(4), true, false);
- stopGrid(getTestIgniteInstanceName(3), true, false);
- stopGrid(getTestIgniteInstanceName(2), true, false);
-
- waitForReadyTopology(internalCache(1, DEFAULT_CACHE_NAME).context().topology(), new AffinityTopologyVersion(8, 0));
-
- assertEquals("Node0", S.compact(expLostParts), S.compact(lost0));
- assertEquals("Node1", S.compact(expLostParts), S.compact(lost1));
- }
- finally {
- stopAllGrids();
- }
- }
-}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossWithPersistenceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossWithPersistenceTest.java
new file mode 100644
index 0000000..7df1db6
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossWithPersistenceTest.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.stream.IntStream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.PartitionLossPolicy;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.apache.ignite.cache.PartitionLossPolicy.READ_WRITE_SAFE;
+
+/**
+ *
+ */
+public class CachePartitionLossWithPersistenceTest extends GridCommonAbstractTest {
+ /** */
+ public static final int WAIT = 2_000;
+
+ /** */
+ private static final int PARTS_CNT = 32;
+
+ /** */
+ private PartitionLossPolicy lossPlc;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+ cfg.setConsistentId(igniteInstanceName);
+ cfg.setClientMode(igniteInstanceName.startsWith("client"));
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ .setWalSegmentSize(4 * 1024 * 1024)
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setMaxSize(100L * 1024 * 1024))
+ );
+
+ cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME).
+ setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL).
+ setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC).
+ setPartitionLossPolicy(lossPlc).
+ setBackups(1).
+ setAffinity(new RendezvousAffinityFunction(false, PARTS_CNT)));
+
+ return cfg;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * Tests activation on partial baseline with lost partitions.
+ */
+ @Test
+ public void testResetOnLesserTopologyAfterRestart() throws Exception {
+ IgniteEx crd = startGrids(5);
+ crd.cluster().active(true);
+
+ stopAllGrids();
+
+ crd = startGrids(2);
+ crd.cluster().active(true);
+
+ resetBaselineTopology();
+
+ assertFalse(grid(0).cache(DEFAULT_CACHE_NAME).lostPartitions().isEmpty());
+ assertFalse(grid(1).cache(DEFAULT_CACHE_NAME).lostPartitions().isEmpty());
+
+ crd.resetLostPartitions(Collections.singleton(DEFAULT_CACHE_NAME));
+
+ awaitPartitionMapExchange();
+ }
+
+ /**
+ *
+ */
+ @Test
+ @Ignore("https://ggsystems.atlassian.net/browse/GG-28521")
+ public void testConsistencyAfterResettingLostPartitions_1() throws Exception {
+ doTestConsistencyAfterResettingLostPartitions(0, false);
+ }
+
+ /**
+ * Lagging node cannot be rebalanced from joining node.
+ */
+ @Test
+ public void testConsistencyAfterResettingLostPartitions_2() throws Exception {
+ doTestConsistencyAfterResettingLostPartitions(1, true);
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void testConsistencyAfterResettingLostPartitions_3() throws Exception {
+ doTestConsistencyAfterResettingLostPartitions(2, false);
+ }
+
+ /**
+ * Test scenario: two nodes are left causing data loss and later returned to topology in various order.
+ * <p>
+ * Expected result: after resetting lost state partitions are synced and no data loss occured. No assertions happened
+ * due to updates going to moving partitions.
+ *
+ * @param partResetMode Reset mode:
+ * <ul>
+ * <li>0 - reset then only lagging node is returned.</li>
+ * <li>1 - reset then both nodes are returned.</li>
+ * <li>2 - reset then both nodes are returned and new node is added to baseline causing partition movement.</li>
+ * </ul>
+ * @param testPutToMovingPart {@code True} to try updating moving partition.
+ */
+ private void doTestConsistencyAfterResettingLostPartitions(int partResetMode, boolean testPutToMovingPart) throws Exception {
+ lossPlc = READ_WRITE_SAFE;
+
+ IgniteEx crd = startGrids(2);
+ crd.cluster().active(true);
+
+ startGrid(2);
+ resetBaselineTopology();
+ awaitPartitionMapExchange();
+
+ // Find a lost partition which is primary for g1.
+ int part = IntStream.range(0, PARTS_CNT).boxed().filter(new Predicate<Integer>() {
+ @Override public boolean test(Integer p) {
+ final List<ClusterNode> nodes = new ArrayList<>(crd.affinity(DEFAULT_CACHE_NAME).mapPartitionToPrimaryAndBackups(p));
+
+ return nodes.get(0).equals(grid(1).localNode()) && nodes.get(1).equals(grid(2).localNode());
+ }
+ }).findFirst().orElseThrow(AssertionError::new);
+
+ stopGrid(1);
+
+ final IgniteInternalCache<Object, Object> cachex = crd.cachex(DEFAULT_CACHE_NAME);
+
+ for (int p = 0; p < PARTS_CNT; p++)
+ cachex.put(p, 0);
+
+ stopGrid(2); // g1 now lags behind g2.
+
+ final Collection<Integer> lostParts = crd.cache(DEFAULT_CACHE_NAME).lostPartitions();
+
+ assertEquals(PARTS_CNT, cachex.context().topology().localPartitions().size() + lostParts.size());
+
+ assertTrue(lostParts.contains(part));
+
+ // Start lagging node first.
+ final IgniteEx g1 = startGrid(1);
+
+ final Collection<Integer> g1LostParts = g1.cache(DEFAULT_CACHE_NAME).lostPartitions();
+
+ assertEquals(lostParts, g1LostParts);
+
+ if (testPutToMovingPart) {
+ // Block rebalancing from g2 to g1 to ensure a primary partition is in moving state.
+ TestRecordingCommunicationSpi.spi(g1).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+ @Override public boolean apply(ClusterNode node, Message msg) {
+ return msg instanceof GridDhtPartitionDemandMessage;
+ }
+ });
+ }
+
+ if (partResetMode == 0)
+ crd.resetLostPartitions(Collections.singleton(DEFAULT_CACHE_NAME));
+
+ final IgniteEx g2 = startGrid(2);
+
+ final Collection<Integer> g2LostParts = g2.cache(DEFAULT_CACHE_NAME).lostPartitions();
+
+ if (partResetMode != 0)
+ assertEquals(lostParts, g2LostParts);
+
+ if (partResetMode == 1)
+ crd.resetLostPartitions(Collections.singleton(DEFAULT_CACHE_NAME));
+
+ if (testPutToMovingPart) {
+ TestRecordingCommunicationSpi.spi(g1).waitForBlocked();
+
+ /** Try put to moving partition. Due to forced reassignment g2 should be a primary for the partition. */
+ cachex.put(part, 0);
+
+ TestRecordingCommunicationSpi.spi(g1).stopBlock();
+ }
+
+ if (partResetMode == 2) {
+ final IgniteEx g3 = startGrid(3);
+
+ resetBaselineTopology();
+
+ crd.resetLostPartitions(Collections.singleton(DEFAULT_CACHE_NAME));
+ }
+
+ awaitPartitionMapExchange();
+
+ assertPartitionsSame(idleVerify(crd, DEFAULT_CACHE_NAME));
+
+ // Read validation.
+ for (int p = 0; p < PARTS_CNT; p++) {
+ for (Ignite ignite : G.allGrids())
+ assertEquals("Partition " + p, 0, ignite.cache(DEFAULT_CACHE_NAME).get(p));
+ }
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossWithRestartsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossWithRestartsTest.java
new file mode 100644
index 0000000..abf40e0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossWithRestartsTest.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.util.AttributeNodeFilter;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ *
+ */
+@RunWith(Parameterized.class)
+public class CachePartitionLossWithRestartsTest extends GridCommonAbstractTest {
+ /** */
+ private static final int PARTS_CNT = 32;
+
+ /** */
+ private static final String START_CACHE_ATTR = "has_cache";
+
+ /** Possible values: -1, 0, 2 */
+ @Parameterized.Parameter(value = 0)
+ public int nonAffIdx;
+
+ /** Possible values: true, false */
+ @Parameterized.Parameter(value = 1)
+ public boolean startClientCache;
+
+ /** Possible values: true, false */
+ @Parameterized.Parameter(value = 2)
+ public boolean dfltRegionPersistence;
+
+ /** Possible values: 3, -1 */
+ @Parameterized.Parameter(value = 3)
+ public int clientIdx;
+
+ /** */
+ @Parameterized.Parameters(name = "{0} {1} {2} {3}")
+ public static List<Object[]> parameters() {
+ ArrayList<Object[]> params = new ArrayList<>();
+
+ for (boolean persistent : new boolean[]{false, true}) {
+ params.add(new Object[]{-1, false, persistent, 3});
+ params.add(new Object[]{0, false, persistent, 3});
+ params.add(new Object[]{2, false, persistent, 3});
+
+ params.add(new Object[]{-1, false, persistent, -1});
+ params.add(new Object[]{0, false, persistent, -1});
+ params.add(new Object[]{2, false, persistent, -1});
+
+ params.add(new Object[]{-1, true, persistent, 3});
+ params.add(new Object[]{0, true, persistent, 3});
+ params.add(new Object[]{2, true, persistent, 3});
+
+ params.add(new Object[]{-1, true, persistent, -1});
+ params.add(new Object[]{0, true, persistent, -1});
+ params.add(new Object[]{2, true, persistent, -1});
+ }
+
+ return params;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setActiveOnStart(false);
+
+ cfg.setConsistentId(igniteInstanceName);
+
+ if (getTestIgniteInstanceIndex(igniteInstanceName) == clientIdx)
+ cfg.setClientMode(true);
+
+ cfg.setIncludeEventTypes(EventType.EVTS_ALL);
+
+ DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+ dsCfg.setWalSegmentSize(4 * 1024 * 1024);
+ dsCfg.setWalMode(WALMode.LOG_ONLY);
+
+ final int size = 50 * 1024 * 1024;
+
+ DataRegionConfiguration dfltRegCfg = new DataRegionConfiguration();
+ dfltRegCfg.setName(DEFAULT_CACHE_NAME).setInitialSize(size).setMaxSize(size).setPersistenceEnabled(dfltRegionPersistence);
+
+ dsCfg.setDefaultDataRegionConfiguration(dfltRegCfg);
+
+ cfg.setDataStorageConfiguration(dsCfg);
+
+ // Do not start cache on non-affinity node.
+ CacheConfiguration ccfg = defaultCacheConfiguration().setNearConfiguration(null).
+ setNodeFilter(new AttributeNodeFilter(START_CACHE_ATTR, Boolean.TRUE)).
+ setBackups(0).
+ setAffinity(new RendezvousAffinityFunction(false, PARTS_CNT));
+
+ if (startClientCache)
+ cfg.setCacheConfiguration(ccfg);
+
+ if (getTestIgniteInstanceIndex(igniteInstanceName) != nonAffIdx) {
+ cfg.setUserAttributes(F.asMap(START_CACHE_ATTR, Boolean.TRUE));
+
+ if (!startClientCache)
+ cfg.setCacheConfiguration(ccfg);
+ }
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void testPartitionLossDetectionOnClientTopology() throws Exception {
+ final IgniteEx crd = startGrids(3);
+ crd.cluster().baselineAutoAdjustEnabled(false);
+ crd.cluster().active(true);
+
+ assertTrue(grid(1).cache(DEFAULT_CACHE_NAME).lostPartitions().isEmpty());
+ assertTrue(grid(2).cache(DEFAULT_CACHE_NAME).lostPartitions().isEmpty());
+
+ final IgniteEx g3 = startGrid(3);
+
+ awaitPartitionMapExchange();
+
+ stopGrid(1);
+
+ // Loss detection is done just before exchange future completion.
+ // Will wait for it's finishing.
+ AffinityTopologyVersion topVer = new AffinityTopologyVersion(5, 0);
+
+ GridDhtPartitionTopology top0 = waitForDetection(crd, topVer);
+ GridDhtPartitionTopology top1 = waitForDetection(grid(2), topVer);
+ GridDhtPartitionTopology top2 = waitForDetection(g3, topVer);
+
+ final Set<Integer> lost1 = new HashSet<>(top0.lostPartitions());
+ final Set<Integer> lost2 = new HashSet<>(top1.lostPartitions());
+ final Set<Integer> lost3 = new HashSet<>(top2.lostPartitions());
+
+ assertFalse(lost1.isEmpty());
+
+ assertEquals(lost1, lost2);
+ assertEquals(lost1, lost3);
+
+ GridDhtPartitionTopology top = startGrid(1).cachex(DEFAULT_CACHE_NAME).context().topology();
+ assertEquals(lost1, top.lostPartitions());
+
+ // TODO https://issues.apache.org/jira/browse/IGNITE-13053
+ grid(1).resetLostPartitions(Collections.singleton(DEFAULT_CACHE_NAME));
+
+ awaitPartitionMapExchange();
+ }
+
+ /**
+ * @param node Node.
+ * @param topVer Topology version.
+ */
+ private GridDhtPartitionTopology waitForDetection(IgniteEx node, AffinityTopologyVersion topVer) throws Exception {
+ GridCacheSharedContext<Object, Object> cctx = node.context().cache().context();
+
+ CacheGroupDescriptor desc = cctx.affinity().cacheGroups().get(CU.cacheId(DEFAULT_CACHE_NAME));
+
+ CacheGroupContext grp = cctx.cache().cacheGroup(desc.groupId());
+
+ GridDhtPartitionTopology top = grp != null ? grp.topology() :
+ cctx.exchange().clientTopology(desc.groupId(), null);
+
+ cctx.exchange().affinityReadyFuture(topVer).get();
+ cctx.exchange().lastTopologyFuture().get();
+
+ return top;
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLostAfterSupplierHasLeftTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLostAfterSupplierHasLeftTest.java
new file mode 100644
index 0000000..f28991e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLostAfterSupplierHasLeftTest.java
@@ -0,0 +1,409 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.PartitionLossPolicy;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ * Test scenario: last supplier has left while a partition on demander is cleared before sending first demand request.
+ */
+public class CachePartitionLostAfterSupplierHasLeftTest extends GridCommonAbstractTest {
+ /** */
+ private static final int PARTS_CNT = 64;
+
+ /** */
+ private PartitionLossPolicy lossPlc;
+
+ /** */
+ private boolean persistence;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+ cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+ cfg.setActiveOnStart(false);
+
+ cfg.setConsistentId(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ .setWalSegmentSize(4 * 1024 * 1024)
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(persistence)
+ .setMaxSize(100L * 1024 * 1024))
+ );
+
+ cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME).
+ setAtomicityMode(TRANSACTIONAL).
+ setCacheMode(PARTITIONED).
+ setBackups(1).
+ setPartitionLossPolicy(lossPlc).
+ setAffinity(new RendezvousAffinityFunction(false, PARTS_CNT)));
+
+ return cfg;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPartitionLostWhileClearing_FailOnCrd() throws Exception {
+ lossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
+ persistence = true;
+
+ doTestPartitionLostWhileClearing(2, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPartitionLostWhileClearing_FailOnCrd_Volatile() throws Exception {
+ lossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
+ persistence = false;
+
+ doTestPartitionLostWhileClearing(2, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPartitionLostWhileClearing_FailOnFullMessage() throws Exception {
+ lossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
+ persistence = true;
+
+ doTestPartitionLostWhileClearing(3, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPartitionLostWhileClearing_FailOnFullMessage_Volatile() throws Exception {
+ lossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
+ persistence = false;
+
+ doTestPartitionLostWhileClearing(3, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPartitionLostWhileClearing_FailOnCrd_Unsafe() throws Exception {
+ lossPlc = PartitionLossPolicy.IGNORE; // READ_WRITE_SAFE is used instead of IGNORE.
+ persistence = true;
+
+ doTestPartitionLostWhileClearing(2, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPartitionLostWhileClearing_FailOnCrd_Unsafe_Volatile() throws Exception {
+ lossPlc = PartitionLossPolicy.IGNORE; // READ_WRITE_SAFE is used instead of IGNORE.
+ persistence = false;
+
+ doTestPartitionLostWhileClearing(2, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPartitionLostWhileClearing_FailOnFullMessage_Unsafe() throws Exception {
+ lossPlc = PartitionLossPolicy.IGNORE; // READ_WRITE_SAFE is used instead of IGNORE.
+ persistence = true;
+
+ doTestPartitionLostWhileClearing(3, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPartitionLostWhileClearing_FailOnFullMessage_Unsafe_Volatile() throws Exception {
+ lossPlc = PartitionLossPolicy.IGNORE; // READ_WRITE_SAFE is used instead of IGNORE.
+ persistence = false;
+
+ doTestPartitionLostWhileClearing(3, 0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPartitionLostWhileClearing_Restart() throws Exception {
+ lossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
+ persistence = true;
+
+ doTestPartitionLostWhileClearing(2, 1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ @Ignore("https://issues.apache.org/jira/browse/IGNITE-13054")
+ public void testPartitionLostWhileClearing_Restart_2() throws Exception {
+ lossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
+ persistence = true;
+
+ doTestPartitionLostWhileClearing(2, 2);
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void testPartitionConsistencyOnSupplierRestart() throws Exception {
+ lossPlc = PartitionLossPolicy.READ_ONLY_SAFE;
+ persistence = true;
+
+ int entryCnt = PARTS_CNT * 200;
+
+ IgniteEx crd = (IgniteEx)startGridsMultiThreaded(2);
+
+ crd.cluster().active(true);
+
+ IgniteCache<Integer, String> cache0 = crd.cache(DEFAULT_CACHE_NAME);
+
+ for (int i = 0; i < entryCnt / 2; i++)
+ cache0.put(i, String.valueOf(i));
+
+ forceCheckpoint();
+
+ stopGrid(1);
+
+ for (int i = entryCnt / 2; i < entryCnt; i++)
+ cache0.put(i, String.valueOf(i));
+
+ final IgniteConfiguration cfg = getConfiguration(getTestIgniteInstanceName(1));
+
+ final TestRecordingCommunicationSpi spi1 = (TestRecordingCommunicationSpi)cfg.getCommunicationSpi();
+
+ spi1.blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+ @Override public boolean apply(ClusterNode node, Message msg) {
+ if (msg instanceof GridDhtPartitionDemandMessage) {
+ GridDhtPartitionDemandMessage msg0 = (GridDhtPartitionDemandMessage)msg;
+
+ return msg0.groupId() == CU.cacheId(DEFAULT_CACHE_NAME);
+ }
+
+ return false;
+ }
+ });
+
+ startGrid(cfg);
+
+ spi1.waitForBlocked(1);
+
+ // Will cause a cancellation of rebalancing because a supplier has left.
+ stopGrid(0);
+
+ awaitPartitionMapExchange();
+
+ final Collection<Integer> lostParts = grid(1).cache(DEFAULT_CACHE_NAME).lostPartitions();
+
+ assertEquals(PARTS_CNT, lostParts.size());
+
+ final IgniteEx g0 = startGrid(0);
+
+ final Collection<Integer> lostParts2 = g0.cache(DEFAULT_CACHE_NAME).lostPartitions();
+
+ assertEquals(PARTS_CNT, lostParts2.size());
+
+ spi1.stopBlock();
+
+ g0.resetLostPartitions(Collections.singletonList(DEFAULT_CACHE_NAME));
+
+ awaitPartitionMapExchange();
+
+ assertPartitionsSame(idleVerify(grid(0), DEFAULT_CACHE_NAME));
+ }
+
+ /**
+ * Test scenario: last supplier has left while a partition on demander is cleared before sending first demand request.
+ *
+ * @param cnt Nodes count.
+ * @param mode Test mode: 0 - reset while clearing, 1 - restart while clearing and activate all, 2 - restart while
+ * clearing and activate in wrong order.
+ * @throws Exception If failed.
+ */
+ private void doTestPartitionLostWhileClearing(int cnt, int mode) throws Exception {
+ IgniteEx crd = startGrids(cnt);
+ crd.cluster().baselineAutoAdjustEnabled(false);
+ crd.cluster().active(true);
+
+ int partId = -1;
+ int idx0 = 0;
+ int idx1 = 1;
+
+ for (int p = 0; p < PARTS_CNT; p++) {
+ List<ClusterNode> nodes = new ArrayList<>(crd.affinity(DEFAULT_CACHE_NAME).mapPartitionToPrimaryAndBackups(p));
+
+ if (grid(nodes.get(0)) == grid(idx0) && grid(nodes.get(1)) == grid(idx1)) {
+ partId = p;
+
+ break;
+ }
+ }
+
+ assertTrue(partId >= 0);
+
+ final int keysCnt = 10_010;
+
+ List<Integer> keys = partitionKeys(grid(idx0).cache(DEFAULT_CACHE_NAME), partId, keysCnt, 0);
+
+ load(grid(idx0), DEFAULT_CACHE_NAME, keys.subList(0, keysCnt - 10));
+
+ stopGrid(idx1);
+
+ load(grid(idx0), DEFAULT_CACHE_NAME, keys.subList(keysCnt - 10, keysCnt));
+
+ TestRecordingCommunicationSpi.spi(grid(0)).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+ @Override public boolean apply(ClusterNode clusterNode, Message msg) {
+ if (msg instanceof GridDhtPartitionSupplyMessage) {
+ GridDhtPartitionSupplyMessage msg0 = (GridDhtPartitionSupplyMessage) msg;
+
+ return msg0.groupId() == CU.cacheId(DEFAULT_CACHE_NAME);
+ }
+
+ return false;
+ }
+ });
+
+ IgniteEx g1 = startGrid(idx1);
+
+ stopGrid(idx0); // Stop supplier in the middle of rebalancing.
+
+ final GridDhtLocalPartition part = g1.cachex(DEFAULT_CACHE_NAME).context().topology().localPartition(partId);
+
+ assertEquals(GridDhtPartitionState.LOST, part.state());
+ assertTrue(g1.cachex(DEFAULT_CACHE_NAME).lostPartitions().contains(partId));
+
+ if (mode != 0) {
+ stopAllGrids();
+
+ if (mode == 1) {
+ crd = startGrids(cnt);
+ crd.cluster().active(true);
+ }
+ else if (mode == 2) {
+ crd = startGrid(idx1);
+ crd.cluster().active(true);
+
+ startGrid(idx0);
+ }
+ else
+ fail("Mode: " + mode);
+
+ awaitPartitionMapExchange();
+
+ assertPartitionsSame(idleVerify(crd, DEFAULT_CACHE_NAME));
+ }
+ else {
+ // Will own a clearing partition.
+ g1.resetLostPartitions(Collections.singletonList(DEFAULT_CACHE_NAME));
+
+ awaitPartitionMapExchange();
+
+ // Expecting partition in OWNING state.
+ final PartitionUpdateCounter cntr = counter(partId, DEFAULT_CACHE_NAME, g1.name());
+ assertNotNull(cntr);
+
+ // Counter must be reset.
+ assertEquals(0, cntr.get());
+
+ // Puts done concurrently with clearing after reset should not be lost.
+ g1.cache(DEFAULT_CACHE_NAME).putAll(keys.stream().collect(Collectors.toMap(k -> k, v -> -1)));
+
+ g1.context().cache().context().evict().awaitFinishAll();
+
+ for (Integer key : keys)
+ assertEquals("key=" + key.toString(), -1, g1.cache(DEFAULT_CACHE_NAME).get(key));
+ }
+ }
+
+ /**
+ * @param ignite Ignite.
+ * @param cache Cache.
+ * @param keys Keys.
+ */
+ private void load(IgniteEx ignite, String cache, List<Integer> keys) {
+ try (IgniteDataStreamer<Object, Object> s = ignite.dataStreamer(cache)) {
+ for (Integer key : keys)
+ s.addData(key, key);
+ }
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLostWhileClearingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLostWhileClearingTest.java
deleted file mode 100644
index d5a4594..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLostWhileClearingTest.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.distributed;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.junit.Test;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
-
-/**
- * Test scenario: supplier is left during rebalancing leaving partition in OWNING state (but actually LOST)
- * because only one owner left.
- * <p>
- * Expected result: no assertions are triggered.
- */
-public class CachePartitionLostWhileClearingTest extends GridCommonAbstractTest {
- /** */
- private static final int PARTS_CNT = 64;
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
-
- cfg.setConsistentId(igniteInstanceName);
-
- cfg.setDataStorageConfiguration(
- new DataStorageConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- .setWalSegmentSize(4 * 1024 * 1024)
- .setDefaultDataRegionConfiguration(
- new DataRegionConfiguration()
- .setPersistenceEnabled(true)
- .setMaxSize(100L * 1024 * 1024))
- );
-
- cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME).
- setAtomicityMode(TRANSACTIONAL).
- setCacheMode(PARTITIONED).
- setBackups(1).
- setAffinity(new RendezvousAffinityFunction(false, PARTS_CNT)));
-
- return cfg;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override protected void beforeTest() throws Exception {
- super.beforeTest();
-
- cleanPersistenceDir();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- cleanPersistenceDir();
- }
-
- /**
- * @throws Exception If failed.
- */
- @Test
- public void testPartitionLostWhileClearing_FailOnCrd() throws Exception {
- doTestPartitionLostWhileClearing(2);
- }
-
- /**
- * @throws Exception If failed.
- */
- @Test
- public void testPartitionLostWhileClearing_FailOnFullMessage() throws Exception {
- doTestPartitionLostWhileClearing(3);
- }
-
- /**
- * @param cnt Nodes count.
- *
- * @throws Exception If failed.
- */
- private void doTestPartitionLostWhileClearing(int cnt) throws Exception {
- try {
- IgniteEx crd = startGrids(cnt);
-
- crd.cluster().active(true);
-
- int partId = -1;
- int idx0 = 0;
- int idx1 = 1;
-
- for (int p = 0; p < PARTS_CNT; p++) {
- List<ClusterNode> nodes = new ArrayList<>(crd.affinity(DEFAULT_CACHE_NAME).mapPartitionToPrimaryAndBackups(p));
-
- if (grid(nodes.get(0)) == grid(idx0) && grid(nodes.get(1)) == grid(idx1)) {
- partId = p;
-
- break;
- }
- }
-
- assertTrue(partId >= 0);
-
- load(grid(idx0), DEFAULT_CACHE_NAME, partId, 10_000, 0);
-
- stopGrid(idx1);
-
- load(grid(idx0), DEFAULT_CACHE_NAME, partId, 10, 10_000);
-
- IgniteEx g1 = startGrid(idx1);
-
- stopGrid(idx0); // Restart supplier in the middle of clearing.
-
- awaitPartitionMapExchange();
-
- // Expecting partition in OWNING state.
- assertNotNull(counter(partId, DEFAULT_CACHE_NAME, g1.name()));
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @param ignite Ignite.
- * @param cache Cache.
- * @param partId Partition id.
- * @param cnt Count.
- * @param skip Skip.
- */
- private void load(IgniteEx ignite, String cache, int partId, int cnt, int skip) {
- List<Integer> keys = partitionKeys(ignite.cache(cache), partId, cnt, skip);
-
- try (IgniteDataStreamer<Object, Object> s = ignite.dataStreamer(cache)) {
- for (Integer key : keys)
- s.addData(key, key);
- }
- }
-}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheResultIsNotNullOnPartitionLossTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheResultIsNotNullOnPartitionLossTest.java
index 2e6a495..8872465 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheResultIsNotNullOnPartitionLossTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheResultIsNotNullOnPartitionLossTest.java
@@ -65,6 +65,7 @@ public class CacheResultIsNotNullOnPartitionLossTest extends GridCommonAbstractT
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+ cfg.setActiveOnStart(false);
cfg.setIncludeEventTypes(EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST);
cfg.setCacheConfiguration(
@@ -92,6 +93,9 @@ public class CacheResultIsNotNullOnPartitionLossTest extends GridCommonAbstractT
for (Integer i : list)
startGrid(i);
+ grid(0).cluster().active(true);
+ grid(0).cluster().baselineAutoAdjustEnabled(false);
+
client = startClientGrid(CLIENT_IDX);
try (IgniteDataStreamer<Integer, Integer> dataStreamer = client.dataStreamer(DEFAULT_CACHE_NAME)) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java
index afe8e1e..4bbece7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java
@@ -29,18 +29,20 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.CacheRebalancingEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.GridTestUtils.SF;
import org.apache.ignite.testframework.MvccFeatureChecker;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.util.TestTcpCommunicationSpi;
-import org.junit.Ignore;
import org.junit.Test;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
@@ -61,6 +63,7 @@ public class GridCachePartitionNotLoadedEventSelfTest extends GridCommonAbstract
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+ cfg.setActiveOnStart(false);
if (igniteInstanceName.matches(".*\\d")) {
String idStr = UUID.randomUUID().toString();
@@ -98,12 +101,14 @@ public class GridCachePartitionNotLoadedEventSelfTest extends GridCommonAbstract
/**
* @throws Exception If failed.
*/
- @Ignore("https://issues.apache.org/jira/browse/IGNITE-5968")
@Test
public void testPrimaryAndBackupDead() throws Exception {
backupCnt = 1;
- startGridsMultiThreaded(4);
+ IgniteEx crd = (IgniteEx) startGridsMultiThreaded(4);
+
+ crd.cluster().baselineAutoAdjustEnabled(false);
+ crd.cluster().active(true);
awaitPartitionMapExchange();
@@ -138,7 +143,14 @@ public class GridCachePartitionNotLoadedEventSelfTest extends GridCommonAbstract
awaitPartitionMapExchange();
- assert !cache.containsKey(key);
+ try {
+ cache.containsKey(key);
+
+ fail("Cache ops should be forbidden");
+ }
+ catch (Exception e) {
+ assertTrue(X.hasCause(e, CacheInvalidStateException.class));
+ }
final long awaitingTimeoutMs = SF.apply(5 * 60 * 1000);
@@ -160,10 +172,11 @@ public class GridCachePartitionNotLoadedEventSelfTest extends GridCommonAbstract
*/
@Test
public void testPrimaryDead() throws Exception {
- startGrid(0);
+ IgniteEx crd = startGrid(0);
startGrid(1);
- awaitPartitionMapExchange();
+ crd.cluster().baselineAutoAdjustEnabled(false);
+ crd.cluster().active(true);
final PartitionNotFullyLoadedListener lsnr = new PartitionNotFullyLoadedListener();
@@ -181,8 +194,6 @@ public class GridCachePartitionNotLoadedEventSelfTest extends GridCommonAbstract
awaitPartitionMapExchange();
- assert !jcache(1).containsKey(key);
-
GridTestUtils.waitForCondition(new GridAbsPredicate() {
@Override public boolean apply() {
return !lsnr.lostParts.isEmpty();
@@ -197,12 +208,15 @@ public class GridCachePartitionNotLoadedEventSelfTest extends GridCommonAbstract
public void testStableTopology() throws Exception {
backupCnt = 1;
- startGrid(1);
+ IgniteEx crd = startGrid(1);
- awaitPartitionMapExchange();
+ crd.cluster().baselineAutoAdjustEnabled(false);
+ crd.cluster().active(true);
startGrid(0);
+ resetBaselineTopology();
+
final PartitionNotFullyLoadedListener lsnr = new PartitionNotFullyLoadedListener();
grid(1).events().localListen(lsnr, EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST);
@@ -221,8 +235,6 @@ public class GridCachePartitionNotLoadedEventSelfTest extends GridCommonAbstract
awaitPartitionMapExchange();
- assert jcache(1).containsKey(key);
-
assert lsnr.lostParts.isEmpty();
}
@@ -234,10 +246,13 @@ public class GridCachePartitionNotLoadedEventSelfTest extends GridCommonAbstract
public void testMapPartitioned() throws Exception {
backupCnt = 0;
- startGrid(0);
+ IgniteEx crd = startGrid(0);
startGrid(1);
+ crd.cluster().baselineAutoAdjustEnabled(false);
+ crd.cluster().active(true);
+
final PartitionNotFullyLoadedListener lsnr = new PartitionNotFullyLoadedListener();
grid(1).events().localListen(lsnr, EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheGroupsPartitionLossPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheGroupsPartitionLossPolicySelfTest.java
index a0040b7..ad2f30f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheGroupsPartitionLossPolicySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheGroupsPartitionLossPolicySelfTest.java
@@ -17,330 +17,26 @@
package org.apache.ignite.internal.processors.cache.distributed;
-import java.util.Collection;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.TimeUnit;
-import javax.cache.CacheException;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.PartitionLossPolicy;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.events.CacheRebalancingEvent;
-import org.apache.ignite.events.Event;
-import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
-import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
/**
- *
+ * Tests partition loss policies working for in-memory groups with multiple caches.
*/
-public class IgniteCacheGroupsPartitionLossPolicySelfTest extends GridCommonAbstractTest {
- /** */
- private PartitionLossPolicy partLossPlc;
-
+public class IgniteCacheGroupsPartitionLossPolicySelfTest extends IgniteCachePartitionLossPolicySelfTest {
/** */
private static final String GROUP_NAME = "group";
- /** */
- private static final String CACHE_1 = "cache1";
-
- /** */
- private static final String CACHE_2 = "cache2";
-
/** {@inheritDoc} */
- @SuppressWarnings("unchecked")
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
- CacheConfiguration ccfg1 = new CacheConfiguration(CACHE_1)
- .setGroupName(GROUP_NAME)
- .setCacheMode(PARTITIONED)
- .setBackups(0)
- .setWriteSynchronizationMode(FULL_SYNC)
- .setPartitionLossPolicy(partLossPlc)
- .setAffinity(new RendezvousAffinityFunction(false, 32));
-
- CacheConfiguration ccfg2 = new CacheConfiguration(ccfg1)
- .setName(CACHE_2);
-
- cfg.setCacheConfiguration(ccfg1, ccfg2);
+ for (CacheConfiguration ccfg : cfg.getCacheConfiguration()) {
+ assertNull(ccfg.getGroupName());
- cfg.setIncludeEventTypes(EventType.EVTS_ALL);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadOnlySafe() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_ONLY_SAFE;
-
- checkLostPartition(false, true);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadOnlyAll() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_ONLY_ALL;
-
- checkLostPartition(false, false);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafe() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- checkLostPartition(true, true);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteAll() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_WRITE_ALL;
-
- checkLostPartition(true, false);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Ignore("https://issues.apache.org/jira/browse/IGNITE-5078")
- @Test
- public void testIgnore() throws Exception {
- prepareTopology();
-
- String cacheName = ThreadLocalRandom.current().nextBoolean() ? CACHE_1 : CACHE_2;
-
- for (Ignite ig : G.allGrids()) {
- IgniteCache<Integer, Integer> cache = ig.cache(cacheName);
-
- Collection<Integer> lost = cache.lostPartitions();
-
- assertTrue("[grid=" + ig.name() + ", lost=" + lost.toString() + ']', lost.isEmpty());
-
- int parts = ig.affinity(cacheName).partitions();
-
- for (int i = 0; i < parts; i++) {
- cache.get(i);
-
- cache.put(i, i);
- }
- }
- }
-
- /**
- * @param canWrite {@code True} if writes are allowed.
- * @param safe {@code True} if lost partition should trigger exception.
- * @throws Exception if failed.
- */
- private void checkLostPartition(boolean canWrite, boolean safe) throws Exception {
- assert partLossPlc != null;
-
- String cacheName = ThreadLocalRandom.current().nextBoolean() ? CACHE_1 : CACHE_2;
-
- int part = prepareTopology();
-
- for (Ignite ig : G.allGrids()) {
- info("Checking node: " + ig.cluster().localNode().id());
-
- verifyCacheOps(cacheName, canWrite, safe, part, ig);
-
- IgniteCache<Integer, Integer> cache = ig.cache(cacheName);
-
- // Check we can read and write to lost partition in recovery mode.
- IgniteCache<Integer, Integer> recoverCache = cache.withPartitionRecover();
-
- for (int lostPart : recoverCache.lostPartitions()) {
- recoverCache.get(lostPart);
- recoverCache.put(lostPart, lostPart);
- }
-
- // Check that writing in recover mode does not clear partition state.
- verifyCacheOps(cacheName, canWrite, safe, part, ig);
+ ccfg.setGroupName(GROUP_NAME);
}
- // Check that partition state does not change after we start a new node.
- IgniteEx grd = startGrid(3);
-
- info("Newly started node: " + grd.cluster().localNode().id());
-
- for (Ignite ig : G.allGrids())
- verifyCacheOps(cacheName, canWrite, safe, part, ig);
-
- ignite(0).resetLostPartitions(F.asList(CACHE_1, CACHE_2));
-
- awaitPartitionMapExchange(true, true, null);
-
- for (Ignite ig : G.allGrids()) {
- IgniteCache<Integer, Integer> cache = ig.cache(cacheName);
-
- assertTrue(cache.lostPartitions().isEmpty());
-
- int parts = ig.affinity(cacheName).partitions();
-
- for (int i = 0; i < parts; i++) {
- cache.get(i);
-
- cache.put(i, i);
- }
- }
- }
-
- /**
- *
- * @param canWrite {@code True} if writes are allowed.
- * @param safe {@code True} if lost partition should trigger exception.
- * @param part Lost partition ID.
- * @param ig Ignite instance.
- */
- private void verifyCacheOps(String cacheName, boolean canWrite, boolean safe, int part, Ignite ig) {
- IgniteCache<Integer, Integer> cache = ig.cache(cacheName);
-
- Collection<Integer> lost = cache.lostPartitions();
-
- assertTrue("Failed to find expected lost partition [exp=" + part + ", lost=" + lost + ']',
- lost.contains(part));
-
- int parts = ig.affinity(cacheName).partitions();
-
- // Check read.
- for (int i = 0; i < parts; i++) {
- try {
- Integer actual = cache.get(i);
-
- if (cache.lostPartitions().contains(i)) {
- if (safe)
- fail("Reading from a lost partition should have failed: " + i);
- // else we could have read anything.
- }
- else
- assertEquals((Integer)i, actual);
- }
- catch (CacheException e) {
- assertTrue("Read exception should only be triggered in safe mode: " + e, safe);
- assertTrue("Read exception should only be triggered for a lost partition " +
- "[ex=" + e + ", part=" + i + ']', cache.lostPartitions().contains(i));
- }
- }
-
- // Check write.
- for (int i = 0; i < parts; i++) {
- try {
- cache.put(i, i);
-
- assertTrue("Write in read-only mode should be forbidden: " + i, canWrite);
-
- if (cache.lostPartitions().contains(i))
- assertFalse("Writing to a lost partition should have failed: " + i, safe);
- }
- catch (CacheException e) {
- if (canWrite) {
- assertTrue("Write exception should only be triggered in safe mode: " + e, safe);
- assertTrue("Write exception should only be triggered for a lost partition: " + e,
- cache.lostPartitions().contains(i));
- }
- // else expected exception regardless of partition.
- }
- }
- }
-
- /**
- * @return Lost partition ID.
- * @throws Exception If failed.
- */
- private int prepareTopology() throws Exception {
- startGrids(4);
-
- final String cacheName = ThreadLocalRandom.current().nextBoolean() ? CACHE_1 : CACHE_2;
-
- Affinity<Object> aff = ignite(0).affinity(cacheName);
-
- for (int i = 0; i < aff.partitions(); i++) {
- ignite(0).cache(CACHE_1).put(i, i);
- ignite(0).cache(CACHE_2).put(i, i);
- }
-
- startClientGrid(4);
-
- for (int i = 0; i < 5; i++)
- info(">>> Node [idx=" + i + ", nodeId=" + ignite(i).cluster().localNode().id() + ']');
-
- awaitPartitionMapExchange();
-
- ClusterNode killNode = ignite(3).cluster().localNode();
-
- int part = -1;
-
- for (int i = 0; i < aff.partitions(); i++) {
- if (aff.isPrimary(killNode, i)) {
- part = i;
-
- break;
- }
- }
-
- if (part == -1)
- throw new IllegalStateException("No partition on node: " + killNode);
-
- final CountDownLatch[] partLost = new CountDownLatch[3];
-
- // Check events.
- for (int i = 0; i < 3; i++) {
- final CountDownLatch latch = new CountDownLatch(1);
- partLost[i] = latch;
-
- final int part0 = part;
-
- grid(i).events().localListen(new P1<Event>() {
- @Override public boolean apply(Event evt) {
- assert evt.type() == EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
-
- CacheRebalancingEvent cacheEvt = (CacheRebalancingEvent)evt;
-
- if (cacheEvt.partition() == part0 && F.eq(cacheName, cacheEvt.cacheName())) {
- latch.countDown();
-
- // Auto-unsubscribe.
- return false;
- }
-
- return true;
- }
- }, EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST);
- }
-
- ignite(3).close();
-
- for (CountDownLatch latch : partLost)
- assertTrue("Failed to wait for partition LOST event", latch.await(10, TimeUnit.SECONDS));
-
- return part;
+ return cfg;
}
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
index 4352c85..05cc971 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
@@ -18,59 +18,58 @@
package org.apache.ignite.internal.processors.cache.distributed;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import javax.cache.CacheException;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.PartitionLossPolicy;
import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.events.CacheRebalancingEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.events.EventType;
import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.TestDelayingCommunicationSpi;
-import org.apache.ignite.internal.managers.communication.GridIoMessage;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsAbstractMessage;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
-import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiPredicate;
import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.MvccFeatureChecker;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import static java.util.Arrays.asList;
-import static java.util.Collections.singletonList;
import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
-import static org.junit.Assume.assumeFalse;
+import static org.apache.ignite.cache.PartitionLossPolicy.IGNORE;
+import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_ALL;
+import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_SAFE;
+import static org.apache.ignite.cache.PartitionLossPolicy.READ_WRITE_SAFE;
/**
*
@@ -78,920 +77,461 @@ import static org.junit.Assume.assumeFalse;
@RunWith(Parameterized.class)
public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTest {
/** */
- @Parameterized.Parameters(name = "{0}")
- public static List<Object[]> parameters() {
- ArrayList<Object[]> params = new ArrayList<>();
+ private static final int PARTS_CNT = 32;
- params.add(new Object[]{TRANSACTIONAL});
+ /** */
+ private boolean client;
- if (!MvccFeatureChecker.forcedMvcc())
- params.add(new Object[]{ATOMIC});
+ /** */
+ @Parameterized.Parameter(value = 0)
+ public CacheAtomicityMode atomicityMode;
- return params;
- }
+ /** */
+ @Parameterized.Parameter(value = 1)
+ public PartitionLossPolicy partLossPlc;
+
+ /** */
+ @Parameterized.Parameter(value = 2)
+ public int backups;
/** */
- private static PartitionLossPolicy partLossPlc;
+ @Parameterized.Parameter(value = 3)
+ public boolean autoAdjust;
/** */
- private static int backups;
+ @Parameterized.Parameter(value = 4)
+ public int nodes;
/** */
- private static final AtomicBoolean delayPartExchange = new AtomicBoolean(false);
+ @Parameterized.Parameter(value = 5)
+ public int[] stopNodes;
/** */
- private final TopologyChanger killSingleNode = new TopologyChanger(
- false, singletonList(3), asList(0, 1, 2, 4), 0);
+ @Parameterized.Parameter(value = 6)
+ public boolean persistence;
/** */
- private static boolean isPersistenceEnabled;
+ private static final String[] CACHES = new String[]{"cache1", "cache2"};
/** */
- @Parameterized.Parameter
- public CacheAtomicityMode atomicity;
+ @Parameterized.Parameters(name = "{0} {1} {2} {3} {4} {6}")
+ public static List<Object[]> parameters() {
+ ArrayList<Object[]> params = new ArrayList<>();
+
+ Random r = new Random();
+ System.out.println("Seed: " + U.field(r, "seed"));
+
+ for (CacheAtomicityMode mode : Arrays.asList(TRANSACTIONAL, ATOMIC)) {
+ // Test always scenarios.
+ params.add(new Object[]{mode, IGNORE, 0, false, 3, new int[]{2}, false});
+ params.add(new Object[]{mode, IGNORE, 0, false, 3, new int[]{2}, true});
+ params.add(new Object[]{mode, READ_ONLY_SAFE, 1, true, 4, new int[]{2, 0}, false});
+ params.add(new Object[]{mode, IGNORE, 1, false, 4, new int[]{0, 2}, false});
+ params.add(new Object[]{mode, READ_WRITE_SAFE, 2, true, 5, new int[]{1, 0, 2}, false});
+
+ // Random scenarios.
+ for (Integer backups : Arrays.asList(0, 1, 2)) {
+ int nodes = backups + 3;
+ int[] stopIdxs = new int[backups + 1];
+
+ List<Integer> tmp = IntStream.range(0, nodes).boxed().collect(Collectors.toList());
+ Collections.shuffle(tmp, r);
+
+ for (int i = 0; i < stopIdxs.length; i++)
+ stopIdxs[i] = tmp.get(i);
+
+ params.add(new Object[]{mode, READ_WRITE_SAFE, backups, false, nodes, stopIdxs, false});
+ params.add(new Object[]{mode, IGNORE, backups, false, nodes, stopIdxs, false});
+ params.add(new Object[]{mode, READ_ONLY_SAFE, backups, false, nodes, stopIdxs, false});
+ params.add(new Object[]{mode, READ_ONLY_ALL, backups, false, nodes, stopIdxs, false});
+ params.add(new Object[]{mode, READ_WRITE_SAFE, backups, true, nodes, stopIdxs, false});
+ params.add(new Object[]{mode, IGNORE, backups, true, nodes, stopIdxs, false});
+ params.add(new Object[]{mode, READ_ONLY_SAFE, backups, true, nodes, stopIdxs, false});
+ params.add(new Object[]{mode, READ_ONLY_ALL, backups, true, nodes, stopIdxs, false});
+
+ boolean ignored = false; // Autoadjust is currently ignored for persistent mode.
+
+ params.add(new Object[]{mode, READ_WRITE_SAFE, backups, ignored, nodes, stopIdxs, true});
+ params.add(new Object[]{mode, IGNORE, backups, ignored, nodes, stopIdxs, true});
+ params.add(new Object[]{mode, READ_ONLY_SAFE, backups, ignored, nodes, stopIdxs, true});
+ params.add(new Object[]{mode, READ_ONLY_ALL, backups, ignored, nodes, stopIdxs, true});
+ params.add(new Object[]{mode, READ_WRITE_SAFE, backups, ignored, nodes, stopIdxs, true});
+ params.add(new Object[]{mode, IGNORE, backups, ignored, nodes, stopIdxs, true});
+ params.add(new Object[]{mode, READ_ONLY_SAFE, backups, ignored, nodes, stopIdxs, true});
+ params.add(new Object[]{mode, READ_ONLY_ALL, backups, ignored, nodes, stopIdxs, true});
+ }
+ }
+
+ return params;
+ }
/** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
- cfg.setCommunicationSpi(new TestDelayingCommunicationSpi() {
- /** {@inheritDoc} */
- @Override protected boolean delayMessage(Message msg, GridIoMessage ioMsg) {
- return delayPartExchange.get() &&
- (msg instanceof GridDhtPartitionsFullMessage || msg instanceof GridDhtPartitionsAbstractMessage);
- }
+ cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
- });
-
- cfg.setCacheConfiguration(cacheConfiguration());
cfg.setConsistentId(gridName);
+
+ cfg.setClientMode(client);
+
cfg.setDataStorageConfiguration(
new DataStorageConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ .setWalSegmentSize(4 * 1024 * 1024)
.setDefaultDataRegionConfiguration(
new DataRegionConfiguration()
- .setPersistenceEnabled(isPersistenceEnabled)
- ));
+ .setPersistenceEnabled(persistence)
+ .setMaxSize(100L * 1024 * 1024))
+ );
+
+ CacheConfiguration[] ccfgs = new CacheConfiguration[CACHES.length];
+
+ for (int i = 0; i < ccfgs.length; i++) {
+ ccfgs[i] = new CacheConfiguration(CACHES[i])
+ .setAtomicityMode(atomicityMode)
+ .setCacheMode(PARTITIONED)
+ .setBackups(backups)
+ .setWriteSynchronizationMode(FULL_SYNC)
+ .setPartitionLossPolicy(partLossPlc)
+ .setAffinity(new RendezvousAffinityFunction(false, PARTS_CNT));
+ }
+
+ cfg.setCacheConfiguration(ccfgs);
cfg.setIncludeEventTypes(EventType.EVTS_ALL);
return cfg;
}
- /**
- * @return Cache configuration.
- */
- protected CacheConfiguration<Integer, Integer> cacheConfiguration() {
- CacheConfiguration<Integer, Integer> cacheCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
-
- cacheCfg.setCacheMode(PARTITIONED);
- cacheCfg.setBackups(backups);
- cacheCfg.setWriteSynchronizationMode(FULL_SYNC);
- cacheCfg.setPartitionLossPolicy(partLossPlc);
- cacheCfg.setAffinity(new RendezvousAffinityFunction(false, 32));
- cacheCfg.setAtomicityMode(atomicity);
-
- return cacheCfg;
- }
-
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
super.beforeTest();
- delayPartExchange.set(false);
-
- partLossPlc = PartitionLossPolicy.IGNORE;
-
- backups = 0;
-
- isPersistenceEnabled = false;
+ cleanPersistenceDir();
}
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- cleanPersistenceDir();
-
super.afterTest();
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadOnlySafe() throws Exception {
- assumeFalse("https://issues.apache.org/jira/browse/IGNITE-11107", MvccFeatureChecker.forcedMvcc());
-
- partLossPlc = PartitionLossPolicy.READ_ONLY_SAFE;
-
- checkLostPartition(false, true, killSingleNode);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadOnlySafeWithPersistence() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_ONLY_SAFE;
-
- isPersistenceEnabled = true;
-
- checkLostPartition(false, true, killSingleNode);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadOnlyAll() throws Exception {
- assumeFalse("https://issues.apache.org/jira/browse/IGNITE-11107", MvccFeatureChecker.forcedMvcc());
-
- partLossPlc = PartitionLossPolicy.READ_ONLY_ALL;
-
- checkLostPartition(false, false, killSingleNode);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Ignore("https://issues.apache.org/jira/browse/IGNITE-10041")
- @Test
- public void testReadOnlyAllWithPersistence() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_ONLY_ALL;
-
- isPersistenceEnabled = true;
-
- checkLostPartition(false, false, killSingleNode);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafe() throws Exception {
- assumeFalse("https://issues.apache.org/jira/browse/IGNITE-11107", MvccFeatureChecker.forcedMvcc());
-
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- checkLostPartition(true, true, killSingleNode);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeWithPersistence() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- isPersistenceEnabled = true;
-
- checkLostPartition(true, true, killSingleNode);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteAll() throws Exception {
- assumeFalse("https://issues.apache.org/jira/browse/IGNITE-11107", MvccFeatureChecker.forcedMvcc());
-
- partLossPlc = PartitionLossPolicy.READ_WRITE_ALL;
-
- checkLostPartition(true, false, killSingleNode);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Ignore("https://issues.apache.org/jira/browse/IGNITE-10041")
- @Test
- public void testReadWriteAllWithPersistence() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_WRITE_ALL;
-
- isPersistenceEnabled = true;
-
- checkLostPartition(true, false, killSingleNode);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeAfterKillTwoNodes() throws Exception {
- assumeFalse("https://issues.apache.org/jira/browse/IGNITE-11107", MvccFeatureChecker.forcedMvcc());
-
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- checkLostPartition(true, true, new TopologyChanger(false, asList(3, 2), asList(0, 1, 4), 0));
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeAfterKillTwoNodesWithPersistence() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- isPersistenceEnabled = true;
-
- checkLostPartition(true, true, new TopologyChanger(false, asList(3, 2), asList(0, 1, 4), 0));
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeAfterKillTwoNodesWithDelay() throws Exception {
- assumeFalse("https://issues.apache.org/jira/browse/IGNITE-11107", MvccFeatureChecker.forcedMvcc());
-
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- checkLostPartition(true, true, new TopologyChanger(false, asList(3, 2), asList(0, 1, 4), 20));
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeAfterKillTwoNodesWithDelayWithPersistence() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- isPersistenceEnabled = true;
-
- checkLostPartition(true, true, new TopologyChanger(false, asList(3, 2), asList(0, 1, 4), 20));
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeWithBackupsAfterKillThreeNodes() throws Exception {
- assumeFalse("https://issues.apache.org/jira/browse/IGNITE-11107", MvccFeatureChecker.forcedMvcc());
-
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- backups = 1;
-
- checkLostPartition(true, true, new TopologyChanger(true, asList(3, 2, 1), asList(0, 4), 0));
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeWithBackupsAfterKillThreeNodesWithPersistence() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- backups = 1;
-
- isPersistenceEnabled = true;
-
- checkLostPartition(true, true, new TopologyChanger(true, asList(3, 2, 1), asList(0, 4), 0));
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeAfterKillCrd() throws Exception {
- assumeFalse("https://issues.apache.org/jira/browse/IGNITE-11107", MvccFeatureChecker.forcedMvcc());
-
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- checkLostPartition(true, true, new TopologyChanger(true, asList(3, 0), asList(1, 2, 4), 0));
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeAfterKillCrdWithPersistence() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- isPersistenceEnabled = true;
-
- checkLostPartition(true, true, new TopologyChanger(true, asList(3, 0), asList(1, 2, 4), 0));
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeWithBackups() throws Exception {
- assumeFalse("https://issues.apache.org/jira/browse/IGNITE-11107", MvccFeatureChecker.forcedMvcc());
-
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- backups = 1;
-
- checkLostPartition(true, true, new TopologyChanger(true, asList(3, 2), asList(0, 1, 4), 0));
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeWithBackupsWithPersistence() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- backups = 1;
-
- isPersistenceEnabled = true;
- checkLostPartition(true, true, new TopologyChanger(true, asList(3, 2), asList(0, 1, 4), 0));
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeWithBackupsAfterKillCrd() throws Exception {
- assumeFalse("https://issues.apache.org/jira/browse/IGNITE-11107", MvccFeatureChecker.forcedMvcc());
-
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- backups = 1;
-
- checkLostPartition(true, true, new TopologyChanger(true, asList(3, 0), asList(1, 2, 4), 0));
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testReadWriteSafeWithBackupsAfterKillCrdWithPersistence() throws Exception {
- partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE;
-
- backups = 1;
-
- isPersistenceEnabled = true;
-
- checkLostPartition(true, true, new TopologyChanger(true, asList(3, 0), asList(1, 2, 4), 0));
- }
-
- /**
- * @throws Exception if failed.
- */
- @Ignore("https://issues.apache.org/jira/browse/IGNITE-5078")
- @Test
- public void testIgnore() throws Exception {
- partLossPlc = PartitionLossPolicy.IGNORE;
-
- checkIgnore(killSingleNode);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Ignore("https://issues.apache.org/jira/browse/IGNITE-5078,https://issues.apache.org/jira/browse/IGNITE-10041")
- @Test
- public void testIgnoreWithPersistence() throws Exception {
- partLossPlc = PartitionLossPolicy.IGNORE;
-
- isPersistenceEnabled = true;
-
- checkIgnore(killSingleNode);
- }
-
- /**
- * @throws Exception if failed.
- */
- @Test
- public void testIgnoreKillThreeNodes() throws Exception {
- partLossPlc = PartitionLossPolicy.IGNORE;
-
- // TODO aliveNodes should include node 4, but it fails due to https://issues.apache.org/jira/browse/IGNITE-5078.
- // TODO need to add 4 to the aliveNodes after IGNITE-5078 is fixed.
- // TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, Arrays.asList(1, 2, 3), Arrays.asList(0, 4), 0);
- TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, asList(1, 2, 3), singletonList(0), 0);
+ stopAllGrids();
- checkIgnore(onlyCrdIsAlive);
+ cleanPersistenceDir();
}
/**
* @throws Exception if failed.
*/
- @Ignore("https://issues.apache.org/jira/browse/IGNITE-10041")
@Test
- public void testIgnoreKillThreeNodesWithPersistence() throws Exception {
- partLossPlc = PartitionLossPolicy.IGNORE;
-
- isPersistenceEnabled = true;
-
- // TODO aliveNodes should include node 4, but it fails due to https://issues.apache.org/jira/browse/IGNITE-5078.
- // TODO need to add 4 to the aliveNodes after IGNITE-5078 is fixed.
- // TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, Arrays.asList(1, 2, 3), Arrays.asList(0, 4), 0);
- TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, asList(1, 2, 3), singletonList(0), 0);
-
- checkIgnore(onlyCrdIsAlive);
- }
+ public void checkLostPartition() throws Exception {
+ log.info("Stop sequence: " + IntStream.of(stopNodes).boxed().collect(Collectors.toList()));
- /**
- * @param topChanger topology changer.
- * @throws Exception if failed.
- */
- private void checkIgnore(TopologyChanger topChanger) throws Exception {
- topChanger.changeTopology();
+ boolean safe = persistence || !(partLossPlc == IGNORE && autoAdjust);
- for (Ignite ig : G.allGrids()) {
- IgniteCache<Integer, Integer> cache = ig.cache(DEFAULT_CACHE_NAME);
+ String cacheName = CACHES[ThreadLocalRandom.current().nextInt(CACHES.length)];
- Collection<Integer> lost = cache.lostPartitions();
+ Map<UUID, Set<Integer>> lostMap = new ConcurrentHashMap<>();
- assertTrue("[grid=" + ig.name() + ", lost=" + lost.toString() + ']', lost.isEmpty());
+ Set<Integer> expLostParts = prepareTopology(nodes, autoAdjust, new P1<Event>() {
+ @Override public boolean apply(Event evt) {
+ assert evt.type() == EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
- int parts = ig.affinity(DEFAULT_CACHE_NAME).partitions();
+ CacheRebalancingEvent cacheEvt = (CacheRebalancingEvent)evt;
- for (int i = 0; i < parts; i++) {
- cache.get(i);
+ lostMap.computeIfAbsent(evt.node().id(), k -> Collections.synchronizedSet(new HashSet<>())).add(cacheEvt.partition());
- cache.put(i, i);
+ return true;
}
- }
- }
+ }, stopNodes);
- /**
- * @param canWrite {@code True} if writes are allowed.
- * @param safe {@code True} if lost partition should trigger exception.
- * @param topChanger topology changer.
- * @throws Exception if failed.
- */
- private void checkLostPartition(boolean canWrite, boolean safe, TopologyChanger topChanger) throws Exception {
- assert partLossPlc != null;
-
- List<Integer> lostParts = topChanger.changeTopology();
-
- // Wait for all grids (servers and client) have same topology version
- // to make sure that all nodes received map with lost partition.
- boolean success = GridTestUtils.waitForCondition(() -> {
- AffinityTopologyVersion last = null;
- for (Ignite ig : G.allGrids()) {
- AffinityTopologyVersion ver = ((IgniteEx)ig).context().cache().context().exchange().readyAffinityVersion();
-
- if (last != null && !last.equals(ver))
- return false;
-
- last = ver;
- }
-
- return true;
- }, 10000);
-
- assertTrue("Failed to wait for new topology", success);
+ int[] stopNodesSorted = Arrays.copyOf(stopNodes, stopNodes.length);
+ Arrays.sort(stopNodesSorted);
for (Ignite ig : G.allGrids()) {
- info("Checking node: " + ig.cluster().localNode().id());
-
- IgniteCache<Integer, Integer> cache = ig.cache(DEFAULT_CACHE_NAME);
-
- verifyLostPartitions(ig, lostParts);
-
- verifyCacheOps(canWrite, safe, ig);
-
- validateQuery(safe, ig);
-
- // TODO withPartitionRecover doesn't work with BLT - https://issues.apache.org/jira/browse/IGNITE-10041.
- if (!isPersistenceEnabled) {
- // Check we can read and write to lost partition in recovery mode.
- IgniteCache<Integer, Integer> recoverCache = cache.withPartitionRecover();
-
- for (int lostPart : recoverCache.lostPartitions()) {
- recoverCache.get(lostPart);
- recoverCache.put(lostPart, lostPart);
- }
-
- // Check that writing in recover mode does not clear partition state.
- verifyLostPartitions(ig, lostParts);
-
- verifyCacheOps(canWrite, safe, ig);
+ if (Arrays.binarySearch(stopNodesSorted, getTestIgniteInstanceIndex(ig.name())) >= 0)
+ continue;
- validateQuery(safe, ig);
- }
+ verifyCacheOps(cacheName, expLostParts, ig, safe);
}
- checkNewNode(true, canWrite, safe);
- checkNewNode(false, canWrite, safe);
+ // Check that partition state does not change after we return nodes.
+ for (int i = 0; i < stopNodes.length; i++) {
+ int node = stopNodes[i];
- // Bring all nodes back.
- for (int i : topChanger.killNodes) {
- IgniteEx grd = startGrid(i);
+ IgniteEx grd = startGrid(node);
info("Newly started node: " + grd.cluster().localNode().id());
-
- // Check that partition state does not change after we start each node.
- // TODO With persistence enabled LOST partitions become OWNING after a node joins back - https://issues.apache.org/jira/browse/IGNITE-10044.
- if (!isPersistenceEnabled) {
- for (Ignite ig : G.allGrids()) {
- verifyCacheOps(canWrite, safe, ig);
-
- // TODO Query effectively waits for rebalance due to https://issues.apache.org/jira/browse/IGNITE-10057
- // TODO and after resetLostPartition there is another OWNING copy in the cluster due to https://issues.apache.org/jira/browse/IGNITE-10058.
- // TODO Uncomment after https://issues.apache.org/jira/browse/IGNITE-10058 is fixed.
-// validateQuery(safe, ig);
- }
- }
}
- ignite(4).resetLostPartitions(singletonList(DEFAULT_CACHE_NAME));
+ for (int i = 0; i < nodes + 1; i++)
+ verifyCacheOps(cacheName, expLostParts, grid(i), safe);
+
+ if (safe)
+ ignite(0).resetLostPartitions(Arrays.asList(CACHES));
awaitPartitionMapExchange(true, true, null);
for (Ignite ig : G.allGrids()) {
- IgniteCache<Integer, Integer> cache = ig.cache(DEFAULT_CACHE_NAME);
+ IgniteCache<Integer, Integer> cache = ig.cache(cacheName);
assertTrue(cache.lostPartitions().isEmpty());
- int parts = ig.affinity(DEFAULT_CACHE_NAME).partitions();
+ int parts = ig.affinity(cacheName).partitions();
for (int i = 0; i < parts; i++) {
cache.get(i);
cache.put(i, i);
}
-
- for (int i = 0; i < parts; i++) {
- checkQueryPasses(ig, false, i);
-
- if (shouldExecuteLocalQuery(ig, i))
- checkQueryPasses(ig, true, i);
-
- }
-
- checkQueryPasses(ig, false);
}
- }
-
- /**
- * @param client Client flag.
- * @param canWrite Can write flag.
- * @param safe Safe flag.
- * @throws Exception If failed to start a new node.
- */
- private void checkNewNode(
- boolean client,
- boolean canWrite,
- boolean safe
- ) throws Exception {
- try {
- IgniteEx cl = client ? startClientGrid("newNode") : startGrid("newNode");
-
- CacheGroupContext grpCtx = cl.context().cache().cacheGroup(CU.cacheId(DEFAULT_CACHE_NAME));
- assertTrue(grpCtx.needsRecovery());
+ if (safe) {
+ for (Ignite ig : G.allGrids()) {
+ if (Arrays.binarySearch(stopNodesSorted, getTestIgniteInstanceIndex(ig.name())) >= 0)
+ continue;
- verifyCacheOps(canWrite, safe, cl);
+ Set<Integer> lostParts = lostMap.get(ig.cluster().localNode().id());
- validateQuery(safe, cl);
- }
- finally {
- stopGrid("newNode", false);
+ assertEquals(expLostParts, lostParts);
+ }
}
}
/**
- * @param node Node.
- * @param lostParts Lost partition IDs.
+ * @param cacheName Cache name.
+ * @param expLostParts Expected lost parts.
+ * @param ig Ignite.
+ * @param safe Safe.
*/
- private void verifyLostPartitions(Ignite node, List<Integer> lostParts) {
- IgniteCache<Integer, Integer> cache = node.cache(DEFAULT_CACHE_NAME);
+ private void verifyCacheOps(String cacheName, Set<Integer> expLostParts, Ignite ig, boolean safe) {
+ boolean readOnly = partLossPlc == READ_ONLY_SAFE || partLossPlc == READ_ONLY_ALL;
- Set<Integer> actualSortedLostParts = new TreeSet<>(cache.lostPartitions());
- Set<Integer> expSortedLostParts = new TreeSet<>(lostParts);
+ IgniteCache<Integer, Integer> cache = ig.cache(cacheName);
- assertEqualsCollections(expSortedLostParts, actualSortedLostParts);
- }
-
- /**
- * @param canWrite {@code True} if writes are allowed.
- * @param safe {@code True} if lost partition should trigger exception.
- * @param ig Ignite instance.
- */
- private void verifyCacheOps(boolean canWrite, boolean safe, Ignite ig) {
- IgniteCache<Integer, Integer> cache = ig.cache(DEFAULT_CACHE_NAME);
+ int parts = ig.affinity(cacheName).partitions();
- int parts = ig.affinity(DEFAULT_CACHE_NAME).partitions();
+ if (!safe)
+ assertTrue(cache.lostPartitions().isEmpty());
- // Check read.
- for (int i = 0; i < parts; i++) {
+ // Check single reads.
+ for (int p = 0; p < parts; p++) {
try {
- Integer actual = cache.get(i);
+ Integer actual = cache.get(p);
- if (cache.lostPartitions().contains(i)) {
- if (safe)
- fail("Reading from a lost partition should have failed: " + i + " " + ig.name());
- // else we could have read anything.
+ if (safe) {
+ assertTrue("Reading from a lost partition should have failed [part=" + p + ']',
+ !cache.lostPartitions().contains(p));
+
+ assertEquals(p, actual.intValue());
}
else
- assertEquals((Integer)i, actual);
+ assertEquals(expLostParts.contains(p) ? null : p, actual);
}
catch (CacheException e) {
- assertTrue("Read exception should only be triggered in safe mode: " + e, safe);
+ assertTrue(X.getFullStackTrace(e), X.hasCause(e, CacheInvalidStateException.class));
+
assertTrue("Read exception should only be triggered for a lost partition " +
- "[ex=" + e + ", part=" + i + ']', cache.lostPartitions().contains(i));
+ "[ex=" + X.getFullStackTrace(e) + ", part=" + p + ']', cache.lostPartitions().contains(p));
}
}
- // Check write.
- for (int i = 0; i < parts; i++) {
+ // Check single writes.
+ for (int p = 0; p < parts; p++) {
try {
- cache.put(i, i);
-
- assertTrue("Write in read-only mode should be forbidden: " + i, canWrite);
-
- if (cache.lostPartitions().contains(i))
- assertFalse("Writing to a lost partition should have failed: " + i, safe);
- }
- catch (CacheException e) {
- if (canWrite) {
- assertTrue("Write exception should only be triggered in safe mode: " + e, safe);
- assertTrue("Write exception should only be triggered for a lost partition: " + e,
- cache.lostPartitions().contains(i));
- }
- // else expected exception regardless of partition.
- }
- }
- }
+ cache.put(p, p);
- /**
- * @param nodes List of nodes to find partition.
- * @return List of partitions that aren't primary or backup for specified nodes.
- */
- private List<Integer> noPrimaryOrBackupPartition(List<Integer> nodes) {
- Affinity<Object> aff = ignite(4).affinity(DEFAULT_CACHE_NAME);
-
- List<Integer> parts = new ArrayList<>();
+ if (!safe && expLostParts.contains(p))
+ cache.remove(p);
- Integer part;
+ if (readOnly) {
+ assertTrue(!cache.lostPartitions().contains(p));
- for (int i = 0; i < aff.partitions(); i++) {
- part = i;
-
- for (Integer id : nodes) {
- if (aff.isPrimaryOrBackup(grid(id).cluster().localNode(), i)) {
- part = null;
+ fail("Writing to a cache containing lost partitions should have failed [part=" + p + ']');
+ }
- break;
+ if (safe) {
+ assertTrue("Writing to a lost partition should have failed [part=" + p + ']',
+ !cache.lostPartitions().contains(p));
}
}
+ catch (CacheException e) {
+ assertTrue(X.getFullStackTrace(e), X.hasCause(e, CacheInvalidStateException.class));
- if (part != null)
- parts.add(i);
+ assertTrue("Write exception should only be triggered for a lost partition or in read-only mode " +
+ "[ex=" + X.getFullStackTrace(e) + ", part=" + p + ']', readOnly || cache.lostPartitions().contains(p));
+ }
}
- return parts;
- }
+ Set<Integer> notLost = IntStream.range(0, parts).boxed().filter(p -> !expLostParts.contains(p)).collect(Collectors.toSet());
- /**
- * Validate query execution on a node.
- *
- * @param safe Safe flag.
- * @param node Node.
- */
- private void validateQuery(boolean safe, Ignite node) {
- // Get node lost and remaining partitions.
- IgniteCache<?, ?> cache = node.cache(DEFAULT_CACHE_NAME);
+ try {
+ Map<Integer, Integer> res = cache.getAll(expLostParts);
- Collection<Integer> lostParts = cache.lostPartitions();
+ assertFalse("Reads from lost partitions should have been allowed only in non-safe mode", safe);
+ }
+ catch (CacheException e) {
+ assertTrue(X.getFullStackTrace(e), X.hasCause(e, CacheInvalidStateException.class));
+ }
- int part = cache.lostPartitions().stream().findFirst().orElseThrow(AssertionError::new);
+ try {
+ Map<Integer, Integer> res = cache.getAll(notLost);
+ }
+ catch (Exception e) {
+ fail("Reads from non lost partitions should have been always allowed");
+ }
- Integer remainingPart = null;
+ try {
+ cache.putAll(expLostParts.stream().collect(Collectors.toMap(k -> k, v -> v)));
- for (int i = 0; i < node.affinity(DEFAULT_CACHE_NAME).partitions(); i++) {
- if (lostParts.contains(i))
- continue;
+ assertFalse("Writes to lost partitions should have been allowed only in non-safe mode", safe);
- remainingPart = i;
+ cache.removeAll(expLostParts);
+ }
+ catch (CacheException e) {
+ assertTrue(X.getFullStackTrace(e), X.hasCause(e, CacheInvalidStateException.class));
+ }
+
+ try {
+ cache.putAll(notLost.stream().collect(Collectors.toMap(k -> k, v -> v)));
- break;
+ assertTrue("Writes to non-lost partitions should have been allowed only in read-write or non-safe mode",
+ !safe || !readOnly);
+ }
+ catch (CacheException e) {
+ assertTrue(X.getFullStackTrace(e), X.hasCause(e, CacheInvalidStateException.class));
}
- assertNotNull("Failed to find a partition that isn't lost", remainingPart);
+ // Check queries.
+ for (int p = 0; p < parts; p++) {
+ boolean loc = ig.affinity(cacheName).isPrimary(ig.cluster().localNode(), p);
- // 1. Check query against all partitions.
- validateQuery0(safe, node);
+ List<?> objects;
- // 2. Check query against LOST partition.
- validateQuery0(safe, node, part);
+ try {
+ objects = runQuery(ig, cacheName, false, p);
- // 3. Check query on remaining partition.
- checkQueryPasses(node, false, remainingPart);
+ assertTrue("Query over lost partition should have failed: safe=" + safe +
+ ", expLost=" + expLostParts + ", p=" + p, !safe || !expLostParts.contains(p));
- if (shouldExecuteLocalQuery(node, remainingPart))
- checkQueryPasses(node, true, remainingPart);
+ if (safe)
+ assertEquals(1, objects.size());
+ } catch (Exception e) {
+ assertTrue(X.getFullStackTrace(e), X.hasCause(e, CacheInvalidStateException.class));
+ }
- // 4. Check query over two partitions - normal and LOST.
- validateQuery0(safe, node, part, remainingPart);
- }
+ try {
+ runQuery(ig, cacheName, false, -1);
- /**
- * Query validation routine.
- *
- * @param safe Safe flag.
- * @param node Node.
- * @param parts Partitions.
- */
- private void validateQuery0(boolean safe, Ignite node, int... parts) {
- if (safe)
- checkQueryFails(node, false, parts);
- else
- checkQueryPasses(node, false, parts);
-
- if (shouldExecuteLocalQuery(node, parts)) {
- if (safe)
- checkQueryFails(node, true, parts);
- else
- checkQueryPasses(node, true, parts);
- }
- }
+ assertFalse("Query should have failed in safe mode with lost partitions", safe);
+ } catch (Exception e) {
+ assertTrue("Query must always work in unsafe mode", safe);
- /**
- * @return true if the given node is primary for all given partitions.
- */
- private boolean shouldExecuteLocalQuery(Ignite node, int... parts) {
- if (parts == null || parts.length == 0)
- return false;
+ assertTrue(X.getFullStackTrace(e), X.hasCause(e, CacheInvalidStateException.class));
+ }
+
+ if (loc) {
+ try {
+ objects = runQuery(ig, cacheName, true, p);
- int numOfPrimaryParts = 0;
+ assertTrue("Query over lost partition should have failed: safe=" + safe +
+ ", expLost=" + expLostParts + ", p=" + p, !safe || !expLostParts.contains(p));
- for (int nodePrimaryPart : node.affinity(DEFAULT_CACHE_NAME).primaryPartitions(node.cluster().localNode())) {
- for (int part : parts) {
- if (part == nodePrimaryPart)
- numOfPrimaryParts++;
+ if (safe)
+ assertEquals(1, objects.size());
+ } catch (Exception e) {
+ assertTrue(X.getFullStackTrace(e), X.hasCause(e, CacheInvalidStateException.class));
+ }
}
}
-
- return numOfPrimaryParts == parts.length;
}
/**
- * @param node Node.
- * @param loc Local flag.
- * @param parts Partitions.
+ * @param ig Ignite.
+ * @param cacheName Cache name.
+ * @param loc Local.
+ * @param part Partition.
*/
- protected void checkQueryPasses(Ignite node, boolean loc, int... parts) {
- // Scan queries don't support multiple partitions.
- if (parts != null && parts.length > 1)
- return;
-
- // TODO Local scan queries fail in non-safe modes - https://issues.apache.org/jira/browse/IGNITE-10059.
- if (loc)
- return;
-
- IgniteCache cache = node.cache(DEFAULT_CACHE_NAME);
+ protected List<?> runQuery(Ignite ig, String cacheName, boolean loc, int part) {
+ IgniteCache cache = ig.cache(cacheName);
ScanQuery qry = new ScanQuery();
- if (parts != null && parts.length > 0)
- qry.setPartition(parts[0]);
+ if (part != -1)
+ qry.setPartition(part);
if (loc)
qry.setLocal(true);
- cache.query(qry).getAll();
+ return cache.query(qry).getAll();
}
/**
- * @param node Node.
- * @param loc Local flag.
- * @param parts Partitions.
+ * @param nodes Nodes.
+ * @param autoAdjust Auto adjust.
+ * @param lsnr Listener.
+ * @param stopNodes Stop nodes.
*/
- protected void checkQueryFails(Ignite node, boolean loc, int... parts) {
- if (parts.length == 0)
- return;
-
- IgniteCache cache = node.cache(DEFAULT_CACHE_NAME);
-
- String msg = loc ? "forced local query" : "partition has been lost";
- GridTestUtils.assertThrows(log, () -> {
- List res = null;
- for (int partition : parts) {
- ScanQuery qry = new ScanQuery();
- qry.setPartition(partition);
-
- if (loc)
- qry.setLocal(true);
+ private Set<Integer> prepareTopology(int nodes, boolean autoAdjust, P1<Event> lsnr, int... stopNodes) throws Exception {
+ final IgniteEx crd = startGrids(nodes);
+ crd.cluster().baselineAutoAdjustEnabled(autoAdjust);
+ crd.cluster().active(true);
- res = cache.query(qry).getAll();
- }
-
- return res;
- }, IgniteCheckedException.class, msg);
- }
+ Affinity<Object> aff = ignite(0).affinity(CACHES[0]);
- /** */
- private class TopologyChanger {
- /** Flag to delay partition exchange */
- private boolean delayExchange;
-
- /** List of nodes to kill */
- private List<Integer> killNodes;
-
- /** List of nodes to be alive */
- private List<Integer> aliveNodes;
-
- /** Delay between node stops */
- private long stopDelay;
-
- /**
- * @param delayExchange Flag for delay partition exchange.
- * @param killNodes List of nodes to kill.
- * @param aliveNodes List of nodes to be alive.
- * @param stopDelay Delay between stopping nodes.
- */
- private TopologyChanger(boolean delayExchange, List<Integer> killNodes, List<Integer> aliveNodes,
- long stopDelay) {
- this.delayExchange = delayExchange;
- this.killNodes = killNodes;
- this.aliveNodes = aliveNodes;
- this.stopDelay = stopDelay;
+ for (int i = 0; i < aff.partitions(); i++) {
+ for (String cacheName0 : CACHES)
+ ignite(0).cache(cacheName0).put(i, i);
}
- /**
- * @return Lost partition ID.
- * @throws Exception If failed.
- */
- private List<Integer> changeTopology() throws Exception {
- startGrids(4);
-
- if (isPersistenceEnabled)
- grid(0).cluster().active(true);
-
- Affinity<Object> aff = ignite(0).affinity(DEFAULT_CACHE_NAME);
-
- for (int i = 0; i < aff.partitions(); i++)
- ignite(0).cache(DEFAULT_CACHE_NAME).put(i, i);
-
- startClientGrid(4);
-
- for (int i = 0; i < 5; i++)
- info(">>> Node [idx=" + i + ", nodeId=" + ignite(i).cluster().localNode().id() + ']');
-
- awaitPartitionMapExchange();
-
- final List<Integer> parts = noPrimaryOrBackupPartition(aliveNodes);
+ client = true;
- if (parts.isEmpty())
- throw new IllegalStateException("No partition on nodes: " + killNodes);
+ startGrid(nodes);
- final List<Map<Integer, Semaphore>> lostMap = new ArrayList<>();
+ client = false;
- for (int i : aliveNodes) {
- HashMap<Integer, Semaphore> semaphoreMap = new HashMap<>();
+ for (int i = 0; i < nodes; i++)
+ info(">>> Node [idx=" + i + ", nodeId=" + ignite(i).cluster().localNode().id() + ']');
- for (Integer part : parts)
- semaphoreMap.put(part, new Semaphore(0));
+ awaitPartitionMapExchange();
- lostMap.add(semaphoreMap);
+ Set<Integer> expLostParts = new HashSet<>();
- grid(i).events().localListen(new P1<Event>() {
- @Override public boolean apply(Event evt) {
- assert evt.type() == EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
+ int[] stopNodesSorted = Arrays.copyOf(stopNodes, stopNodes.length);
+ Arrays.sort(stopNodesSorted);
- CacheRebalancingEvent cacheEvt = (CacheRebalancingEvent)evt;
+ // Find partitions not owned by any remaining node.
+ for (int i = 0; i < PARTS_CNT; i++) {
+ int c = 0;
- if (F.eq(DEFAULT_CACHE_NAME, cacheEvt.cacheName())) {
- if (semaphoreMap.containsKey(cacheEvt.partition()))
- semaphoreMap.get(cacheEvt.partition()).release();
- }
-
- return true;
- }
- }, EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST);
+ for (int idx = 0; idx < nodes; idx++) {
+ if (Arrays.binarySearch(stopNodesSorted, idx) < 0 && !aff.isPrimary(grid(idx).localNode(), i) && !aff.isBackup(grid(idx).localNode(), i))
+ c++;
}
- if (delayExchange)
- delayPartExchange.set(true);
+ if (c == nodes - stopNodes.length)
+ expLostParts.add(i);
+ }
- ExecutorService executor = Executors.newFixedThreadPool(killNodes.size());
+ assertFalse("Expecting lost partitions for the test scneario", expLostParts.isEmpty());
- for (Integer node : killNodes) {
- executor.submit(new Runnable() {
- @Override public void run() {
- grid(node).close();
+ for (Ignite ignite : G.allGrids()) {
+ // Prevent rebalancing to bring partitions in owning state.
+ if (backups > 0) {
+ TestRecordingCommunicationSpi.spi(ignite).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+ @Override public boolean apply(ClusterNode clusterNode, Message msg) {
+ return msg instanceof GridDhtPartitionDemandMessage;
}
});
-
- Thread.sleep(stopDelay);
}
- executor.shutdown();
-
- delayPartExchange.set(false);
-
- Thread.sleep(5_000L);
+ ignite.events().localListen(lsnr, EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST);
+ }
- for (Map<Integer, Semaphore> map : lostMap) {
- for (Map.Entry<Integer, Semaphore> entry : map.entrySet())
- assertTrue("Failed to wait for partition LOST event for partition: " + entry.getKey(), entry.getValue().tryAcquire(1));
- }
+ for (int i = 0; i < stopNodes.length; i++)
+ stopGrid(stopNodes[i], true);
- for (Map<Integer, Semaphore> map : lostMap) {
- for (Map.Entry<Integer, Semaphore> entry : map.entrySet())
- assertFalse("Partition LOST event raised twice for partition: " + entry.getKey(), entry.getValue().tryAcquire(1));
- }
-
- return parts;
- }
+ return expLostParts;
}
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePrimarySyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePrimarySyncTest.java
index 7937f85..a46903a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePrimarySyncTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePrimarySyncTest.java
@@ -62,16 +62,19 @@ public class IgniteCachePrimarySyncTest extends GridCommonAbstractTest {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
CacheConfiguration<Object, Object> ccfg1 = new CacheConfiguration<>(ATOMIC_CACHE)
+ .setReadFromBackup(false)
.setAtomicityMode(ATOMIC)
.setBackups(2)
.setWriteSynchronizationMode(PRIMARY_SYNC);
CacheConfiguration<Object, Object> ccfg2 = new CacheConfiguration<>(TX_CACHE)
+ .setReadFromBackup(false)
.setAtomicityMode(TRANSACTIONAL)
.setBackups(2)
.setWriteSynchronizationMode(PRIMARY_SYNC);
CacheConfiguration<Object, Object> ccfg3 = new CacheConfiguration<>(MVCC_CACHE)
+ .setReadFromBackup(false)
.setAtomicityMode(TRANSACTIONAL_SNAPSHOT)
.setBackups(2)
.setWriteSynchronizationMode(PRIMARY_SYNC);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTxFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTxFailoverTest.java
index 5636b19..8878c6f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTxFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTxFailoverTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache.mvcc;
+import java.util.Collections;
import java.util.concurrent.CyclicBarrier;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteTransactions;
@@ -29,10 +30,12 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
import org.apache.ignite.internal.processors.cache.WalStateManager;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
+import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
@@ -227,12 +230,12 @@ public class CacheMvccTxFailoverTest extends GridCommonAbstractTest {
barrier.await();
- startGrid(1);
+ IgniteEx g1 = startGrid(1);
+ g1.resetLostPartitions(Collections.singleton(DEFAULT_CACHE_NAME));
barrier.await();
}
catch (Exception e) {
-
barrier.reset();
}
}
@@ -259,6 +262,9 @@ public class CacheMvccTxFailoverTest extends GridCommonAbstractTest {
tx.rollback();
}
+ catch (Exception e) {
+ assertTrue(X.hasCause(e, CacheInvalidStateException.class));
+ }
barrier.await();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteLostPartitionsOnLeaveBaselineSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteLostPartitionsOnLeaveBaselineSelfTest.java
new file mode 100644
index 0000000..ac2ed34
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteLostPartitionsOnLeaveBaselineSelfTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.PartitionLossPolicy;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+import java.util.Arrays;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ * Test check that on leave baseline node's partitions marked as lost on mixed region grid.
+ */
+public class IgniteLostPartitionsOnLeaveBaselineSelfTest extends GridCommonAbstractTest {
+ /**
+ *
+ */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void afterTestsStopped() throws Exception {
+ cleanPersistenceDir();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setConsistentId(gridName);
+
+ DataStorageConfiguration memCfg = new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setMaxSize(10 * 1024 * 1024)
+ .setPersistenceEnabled(true)
+ .setMetricsEnabled(true)
+ .setName("dflt-plc"))
+ .setDataRegionConfigurations(new DataRegionConfiguration()
+ .setMaxSize(10 * 1024 * 1024)
+ .setPersistenceEnabled(false)
+ .setMetricsEnabled(true)
+ .setName("no-persistence"))
+ .setWalMode(WALMode.LOG_ONLY)
+ .setPageSize(4 * 1024)
+ .setMetricsEnabled(true);
+
+ cfg.setDataStorageConfiguration(memCfg);
+
+ ((TcpDiscoverySpi) cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+ return cfg;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+
+ super.afterTest();
+ }
+
+ /**
+ * @param name Cache name.
+ * @param cacheMode Cache mode.
+ * @param atomicityMode Atomicity mode.
+ * @return Cache configuration.
+ */
+ private CacheConfiguration cacheConfiguration(
+ String name,
+ CacheMode cacheMode,
+ CacheAtomicityMode atomicityMode,
+ String dataRegName
+ ) {
+ CacheConfiguration ccfg = new CacheConfiguration();
+
+ ccfg.setName(name);
+ ccfg.setAtomicityMode(atomicityMode);
+ ccfg.setBackups(0);
+ ccfg.setCacheMode(cacheMode);
+ ccfg.setPartitionLossPolicy(PartitionLossPolicy.READ_WRITE_SAFE);
+ ccfg.setDataRegionName(dataRegName);
+
+ return ccfg;
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ @Test
+ public void testLostPartitionsOnLeaveBaseline() throws Exception {
+ try {
+ final IgniteEx gridFirst = startGrid(0);
+ startGrid(1);
+
+ gridFirst.cluster().active(true);
+
+ gridFirst.getOrCreateCaches(Arrays.asList(
+ cacheConfiguration("cache-no-persistence", PARTITIONED, ATOMIC, "no-persistence"),
+ cacheConfiguration("cache-persistence", PARTITIONED, ATOMIC, null)
+ ));
+
+ IgniteInternalCache<Object, Object> cacheNoPersistence = gridFirst.cachex("cache-no-persistence");
+ IgniteInternalCache<Object, Object> cachePersistence = gridFirst.cachex("cache-persistence");
+
+ for (int i = 0; i < 10; i++) {
+ cacheNoPersistence.put(i, i);
+ cachePersistence.put(i, i);
+ }
+
+ stopGrid(1);
+
+ resetBaselineTopology();
+
+ assertTrue("List of lost partitions for cache without persistence should not be empty.",
+ !cacheNoPersistence.context().topology().lostPartitions().isEmpty());
+
+ assertTrue("List of lost partitions for cache with persistence should not be empty.",
+ !cachePersistence.context().topology().lostPartitions().isEmpty());
+ } finally {
+ stopAllGrids();
+ }
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
index 1903c65..fde3ee8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
@@ -115,6 +115,7 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
CacheConfiguration ccfg3 = cacheConfiguration(INDEXED_CACHE_IN_MEMORY)
+ .setBackups(2)
.setDataRegionName(IN_MEMORY_REGION);
QueryEntity qryEntity = new QueryEntity(Integer.class.getName(), TestValue.class.getName());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java
index 6e66410..89ab8b4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java
@@ -179,8 +179,14 @@ public class IgnitePdsCorruptedIndexTest extends GridCommonAbstractTest {
corruptedNode.cluster().active(true);
+ // Not all owners have been returned, data loss is expected.
+ assertFalse(grid(0).cache(CACHE).lostPartitions().isEmpty());
+ assertFalse(grid(corruptedNodeName).cache(CACHE).lostPartitions().isEmpty());
+
resetBaselineTopology();
+ grid(0).resetLostPartitions(Collections.singleton(CACHE));
+
// If index was corrupted, rebalance or one of the following queries should be failed.
awaitPartitionMapExchange();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
index 77ebd3b..0b7fd23 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.file.OpenOption;
+import java.util.Collection;
+import java.util.Collections;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.CountDownLatch;
@@ -99,7 +101,7 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr
.setInitialSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
.setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
)
- // Test verifies checkpoint count, so it is essencial that no checkpoint is triggered by timeout
+ // Test verifies checkpoint count, so it is essential that no checkpoint is triggered by timeout
.setCheckpointFrequency(999_999_999_999L)
.setWalMode(WALMode.LOG_ONLY)
.setFileIOFactory(new TestFileIOFactory(new DataStorageConfiguration().getFileIOFactory()))
@@ -108,7 +110,7 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr
cfg.setCacheConfiguration(
new CacheConfiguration(DEFAULT_CACHE_NAME)
.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
- // Test checks internal state before and after rebalance, so it is configured to be triggered manually
+ // Test checks internal state before and after rebalance, so it is configured to be triggered manually
.setRebalanceDelay(-1)
.setBackups(dfltCacheBackupCnt),
@@ -459,7 +461,7 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr
for (Ignite g : G.allGrids())
g.cache(REPL_CACHE).rebalance();
- awaitPartitionMapExchange();
+ awaitPartitionMapExchange(false, false, null, false, Collections.singleton(REPL_CACHE));
for (int nodeIdx = 2; nodeIdx < nodeCnt; nodeIdx++) {
CacheGroupContext grpCtx = grid(nodeIdx).cachex(REPL_CACHE).context().group();
@@ -580,10 +582,20 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr
for (int k = 0; k < keysCnt; k++)
assertFalse("k=" + k + ", v=" + cache.get(k), cache.containsKey(k));
+ Collection<Integer> lostParts = cache.lostPartitions();
+
Set<Integer> keys = new TreeSet<>();
- for (int k = 0; k < keysCnt; k++)
+ for (int k = 0; k < keysCnt; k++) {
+ // Skip lost partitions.
+ if (lostParts.contains(newIgnite.affinity(DEFAULT_CACHE_NAME).partition(k)))
+ continue;
+
keys.add(k);
+ }
+
+ for (Integer k : keys)
+ assertFalse("k=" + k + ", v=" + cache.get(k), cache.containsKey(k));
assertFalse(cache.containsKeys(keys));
}
@@ -615,6 +627,8 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr
ignite.cluster().setBaselineTopology(5);
+ ignite.resetLostPartitions(Collections.singleton(DEFAULT_CACHE_NAME));
+
// Await fully exchange complete.
awaitExchange((IgniteEx)ignite);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
index c654550..be41114 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
@@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.LockSupport;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -45,7 +46,9 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteFuture;
@@ -55,6 +58,7 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
import org.apache.ignite.transactions.TransactionConcurrency;
import org.apache.ignite.transactions.TransactionIsolation;
+import org.apache.ignite.transactions.TransactionRollbackException;
import org.junit.Ignore;
import org.junit.Test;
@@ -110,6 +114,7 @@ public class ClientAffinityAssignmentWithBaselineTest extends GridCommonAbstract
if (!igniteInstanceName.startsWith(CLIENT_GRID_NAME)) {
cfg.setDataStorageConfiguration(
new DataStorageConfiguration()
+ .setWalSegmentSize(4 * 1024 * 1024)
.setDefaultDataRegionConfiguration(
new DataRegionConfiguration()
.setPersistenceEnabled(true)
@@ -911,12 +916,8 @@ public class ClientAffinityAssignmentWithBaselineTest extends GridCommonAbstract
threadProgressTracker.compute(Thread.currentThread().getId(),
(tId, ops) -> ops == null ? 1 : ops + 1);
}
- catch (CacheException e) {
- if (e.getCause() instanceof ClusterTopologyException)
- ((ClusterTopologyException)e.getCause()).retryReadyFuture().get();
- }
- catch (ClusterTopologyException e) {
- e.retryReadyFuture().get();
+ catch (CacheException | ClusterTopologyException | TransactionRollbackException e) {
+ awaitTopology(e);
}
}
}
@@ -930,6 +931,30 @@ public class ClientAffinityAssignmentWithBaselineTest extends GridCommonAbstract
}
/**
+ * Extract cause of type {@link ClusterTopologyException} (if exists) and awaits retry topology.
+ *
+ * @param e Original exception.
+ */
+ private void awaitTopology(Throwable e) throws IgniteCheckedException {
+ if (e instanceof TransactionRollbackException) {
+ TransactionRollbackException e0 = (TransactionRollbackException) e;
+
+ ClusterTopologyCheckedException e00 = X.cause(e0, ClusterTopologyCheckedException.class);
+ IgniteInternalFuture f;
+
+ if (e00 != null && (f = e00.retryReadyFuture()) != null)
+ f.get();
+ }
+
+ ClusterTopologyException ex = X.cause(e, ClusterTopologyException.class);
+ IgniteFuture f;
+
+ // For now in MVCC case the topology exception doesn't have a remap future.
+ if (ex != null && (f = ex.retryReadyFuture()) != null)
+ f.get();
+ }
+
+ /**
* @param waitMs Wait milliseconds.
* @param loadError Load error.
* @param threadProgressTracker Thread progress tracker.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAbsentEvictionNodeOutOfBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAbsentEvictionNodeOutOfBaselineTest.java
index 0f4ddd4..0db24aa 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAbsentEvictionNodeOutOfBaselineTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAbsentEvictionNodeOutOfBaselineTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache.persistence.baseline;
+import java.util.Collections;
import java.util.List;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
@@ -99,6 +100,8 @@ public class IgniteAbsentEvictionNodeOutOfBaselineTest extends GridCommonAbstrac
resetBaselineTopology();
+ ignite0.resetLostPartitions(Collections.singleton(TEST_CACHE_NAME));
+
awaitPartitionMapExchange();
for (int i = 0; i < 200; i++)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheRestoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheRestoreTest.java
index cf3791a..8c8a89d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheRestoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheRestoreTest.java
@@ -25,6 +25,7 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Test;
@@ -130,7 +131,9 @@ public class IgnitePdsCacheRestoreTest extends GridCommonAbstractTest {
ccfgs = configurations2();
}
- startGrid(2);
+ IgniteEx g2 = startGrid(2);
+
+ g2.resetLostPartitions(Arrays.asList("c1", "c2", "c3"));
cache1 = ignite(2).cache("c1");
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java
index 500539d..dd3cbfa 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.OpenOption;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -75,6 +76,12 @@ public class IgnitePdsCacheWalDisabledOnRebalancingTest extends GridCommonAbstra
private static final int CACHE_SIZE = 2_000;
/** */
+ private static final String CACHE1_NAME = "cache1";
+
+ /** */
+ private static final String CACHE2_NAME = "cache2";
+
+ /** */
private static final String CACHE3_NAME = "cache3";
/** Function to generate cache values. */
@@ -114,12 +121,12 @@ public class IgnitePdsCacheWalDisabledOnRebalancingTest extends GridCommonAbstra
// This is required because some tests do full clearing of persistence folder losing BLT info on next join.
cfg.setConsistentId(igniteInstanceName);
- CacheConfiguration ccfg1 = new CacheConfiguration("cache1")
+ CacheConfiguration ccfg1 = new CacheConfiguration(CACHE1_NAME)
.setAtomicityMode(CacheAtomicityMode.ATOMIC)
.setCacheMode(CacheMode.REPLICATED)
.setAffinity(new RendezvousAffinityFunction(false, CACHE1_PARTS_NUM));
- CacheConfiguration ccfg2 = new CacheConfiguration("cache2")
+ CacheConfiguration ccfg2 = new CacheConfiguration(CACHE2_NAME)
.setBackups(1)
.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
.setCacheMode(CacheMode.PARTITIONED)
@@ -369,8 +376,12 @@ public class IgnitePdsCacheWalDisabledOnRebalancingTest extends GridCommonAbstra
startClientGrid("client");
+ assertFalse(grid(1).cache(CACHE2_NAME).lostPartitions().isEmpty());
+
fileIoBlockingSemaphore.release(Integer.MAX_VALUE);
+ ig0.resetLostPartitions(Collections.singleton(CACHE2_NAME));
+
awaitPartitionMapExchange();
assertPartitionsSame(idleVerify(grid(0), CACHE3_NAME));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
index 5b9fd1d..a3e8e97 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
@@ -22,6 +22,7 @@ import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.file.Paths;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Set;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
@@ -261,6 +262,8 @@ public class WalCompactionTest extends GridCommonAbstractTest {
resetBaselineTopology();
+ ignite.resetLostPartitions(Collections.singleton(CACHE_NAME));
+
// This node will join to different blt.
startGrid(2);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AtomicPartitionCounterStateConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AtomicPartitionCounterStateConsistencyTest.java
index f3b9553..217fd8f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AtomicPartitionCounterStateConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AtomicPartitionCounterStateConsistencyTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.cache.transactions;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -29,6 +30,8 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.cluster.ClusterTopologyException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
import org.apache.ignite.internal.util.typedef.X;
import org.junit.Ignore;
@@ -106,11 +109,16 @@ public class AtomicPartitionCounterStateConsistencyTest extends TxPartitionCount
if (batch) {
cache.putAll(insertedKeys.stream().collect(toMap(k -> k, v -> v, (k, v) -> v, LinkedHashMap::new)));
+
+ Collections.sort(rmvKeys);
+
cache.removeAll(new LinkedHashSet<>(rmvKeys));
}
}
catch (Exception e) {
- assertTrue(X.getFullStackTrace(e), X.hasCause(e, ClusterTopologyException.class));
+ assertTrue(X.getFullStackTrace(e), X.hasCause(e, ClusterTopologyException.class) ||
+ X.hasCause(e, ClusterTopologyCheckedException.class) ||
+ X.hasCause(e, CacheInvalidStateException.class));
}
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPartitionCounterStateAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPartitionCounterStateAbstractTest.java
index 1dac326..d3745c2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPartitionCounterStateAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPartitionCounterStateAbstractTest.java
@@ -75,6 +75,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
import org.jetbrains.annotations.Nullable;
import static java.util.concurrent.TimeUnit.DAYS;
@@ -85,6 +86,7 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
import static org.apache.ignite.internal.TestRecordingCommunicationSpi.spi;
import static org.apache.ignite.testframework.GridTestUtils.runMultiThreadedAsync;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
/**
* Test framework for ordering transaction's prepares and commits by intercepting messages and releasing then
@@ -151,6 +153,23 @@ public abstract class TxPartitionCounterStateAbstractTest extends GridCommonAbst
return PARTS_CNT;
}
+ /**
+ * @return Default tx concurrency.
+ */
+ protected TransactionConcurrency concurrency() {
+ return PESSIMISTIC;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+ }
+
/** */
protected boolean persistenceEnabled() {
return true;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPartitionCounterStateConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPartitionCounterStateConsistencyTest.java
index e461c50..f54af84 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPartitionCounterStateConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPartitionCounterStateConsistencyTest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.transactions;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
@@ -43,8 +44,10 @@ import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.cluster.ClusterTopologyException;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
@@ -53,6 +56,7 @@ import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
import org.apache.ignite.internal.processors.cache.CacheEntryInfoCollection;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
@@ -78,7 +82,6 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
import static org.apache.ignite.internal.processors.cache.GridCacheOperation.CREATE;
import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE;
-import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
/**
@@ -545,7 +548,19 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
}
});
- fut.get();
+ try {
+ fut.get(10_000);
+ }
+ catch (IgniteFutureTimeoutCheckedException e) {
+ for (Ignite ignite : G.allGrids()) {
+ final PartitionUpdateCounter cntr = counter(primaryParts[0], ignite.name());
+ log.info("Node: " + ignite.name() + ", cntr=" + cntr);
+ }
+
+ assertPartitionsSame(idleVerify(crd, DEFAULT_CACHE_NAME));
+
+ fail("Rebalancing is expected");
+ }
awaitPartitionMapExchange();
@@ -811,13 +826,12 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
assertNotNull(rndAddrsField);
rndAddrsField.set(customDiscoSpi, true);
- Ignite crd = startGrid(0); // Start coordinator with custom discovery SPI.
-
- configureBaselineAutoAdjust();
-
+ IgniteEx crd = startGrid(0); // Start coordinator with custom discovery SPI.
IgniteEx g1 = startGrid(1);
startGrid(2);
+ crd.cluster().baselineAutoAdjustEnabled(false);
+
crd.cluster().active(true);
// Same name pattern as in test configuration.
@@ -836,7 +850,6 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
assertEquals(consistentId, g3.localNode().consistentId());
- crd.cluster().baselineAutoAdjustEnabled(false);
resetBaselineTopology();
awaitPartitionMapExchange();
@@ -952,6 +965,74 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
doTestConsistencyAfterBaselineNodeStopAndRemoval(2);
}
+ /** */
+ @Test
+ public void testPrimaryLeftUnderLoadToSwitchingPartitions_1() throws Exception {
+ doTestPrimaryLeftUnderLoadToSwitchingPartitions(0, 2);
+ }
+
+ /** */
+ @Test
+ public void testPrimaryLeftUnderLoadToSwitchingPartitions_2() throws Exception {
+ doTestPrimaryLeftUnderLoadToSwitchingPartitions(1, 2);
+ }
+
+ /** */
+ @Test
+ public void testPrimaryLeftUnderLoadToSwitchingPartitions_3() throws Exception {
+ doTestPrimaryLeftUnderLoadToSwitchingPartitions(0, 3);
+ }
+
+ /** */
+ @Test
+ public void testPrimaryLeftUnderLoadToSwitchingPartitions_4() throws Exception {
+ doTestPrimaryLeftUnderLoadToSwitchingPartitions(1, 3);
+ }
+
+ /**
+ * The scenario:
+ * <p>
+ * 1. Start updates only to primary partitions what will be switched when this node has left.
+ * 2. Stop primary.
+ * 3. Wait for switch.
+ * 4. Expect no assertions.
+ *
+ * Note: applicable only for BLT compatible caches (currently only persistent mode).
+ *
+ * @param backups Backups count.
+ * @param nodes Nodes count.
+ */
+ private void doTestPrimaryLeftUnderLoadToSwitchingPartitions(int backups, int nodes) throws Exception {
+ this.backups = backups;
+
+ final IgniteEx crd = startGrids(nodes);
+
+ crd.cluster().active(true);
+
+ List<Integer> primaryKeys = primaryKeys(crd.cache(DEFAULT_CACHE_NAME), 1024);
+
+ Random r = new Random();
+
+ AtomicBoolean stop = new AtomicBoolean();
+
+ final IgniteInternalFuture<?> fut =
+ doRandomUpdates(r, grid(1), primaryKeys, grid(1).cache(DEFAULT_CACHE_NAME), stop::get);
+
+ doSleep(3_000);
+
+ crd.close();
+
+ doSleep(2_000);
+
+ stop.set(true);
+
+ fut.get();
+
+ awaitPartitionMapExchange();
+
+ checkFutures();
+ }
+
/**
* Test a scenario when partition is evicted and owned again with non-zero initial and current counters.
* When rebalancing is finished no partition desync should happen.
@@ -963,6 +1044,8 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
IgniteEx prim = startGrids(srvNodes);
+ prim.cluster().baselineAutoAdjustEnabled(false);
+
prim.cluster().active(true);
for (int p = 0; p < partitions(); p++) {
@@ -976,8 +1059,7 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
awaitPartitionMapExchange();
- if (persistenceEnabled())
- resetBaselineTopology(); // topVer=5,1
+ resetBaselineTopology(); // topVer=5,1
awaitPartitionMapExchange();
@@ -987,8 +1069,7 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
awaitPartitionMapExchange();
- if (persistenceEnabled())
- resetBaselineTopology(); // topVer=6,1
+ resetBaselineTopology(); // topVer=6,1
awaitPartitionMapExchange(true, true, null);
@@ -1004,6 +1085,10 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
startGrid(2);
startGrid(3);
+
+ assertFalse(grid(0).cache(DEFAULT_CACHE_NAME).lostPartitions().isEmpty());
+
+ grid(0).resetLostPartitions(Collections.singleton(DEFAULT_CACHE_NAME));
}
prim.context().cache().context().exchange().rebalanceDelay(500);
@@ -1018,8 +1103,7 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
prim.cache(DEFAULT_CACHE_NAME),
stop::get);
- if (persistenceEnabled())
- resetBaselineTopology(); // topVer=7,1
+ resetBaselineTopology(); // topVer=7,1
awaitPartitionMapExchange();
@@ -1082,13 +1166,8 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
* @param stopClo A closure providing stop condition.
* @return Finish future.
*/
- protected IgniteInternalFuture<?> doRandomUpdates(
- Random r,
- Ignite near,
- List<Integer> primaryKeys,
- IgniteCache<Object, Object> cache,
- BooleanSupplier stopClo
- ) throws Exception {
+ protected IgniteInternalFuture<?> doRandomUpdates(Random r, Ignite near, List<Integer> primaryKeys,
+ IgniteCache<Object, Object> cache, BooleanSupplier stopClo) throws Exception {
LongAdder puts = new LongAdder();
LongAdder removes = new LongAdder();
@@ -1101,7 +1180,7 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
List<Integer> keys = primaryKeys.subList(rangeStart, rangeStart + range);
- try (Transaction tx = near.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 0, 0)) {
+ try (Transaction tx = near.transactions().txStart(concurrency(), REPEATABLE_READ, 0, 0)) {
List<Integer> insertedKeys = new ArrayList<>();
for (Integer key : keys) {
@@ -1122,11 +1201,16 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
}
}
- tx.commit();
+ if (r.nextFloat() < 0.1)
+ tx.rollback();
+ else
+ tx.commit();
}
catch (Exception e) {
assertTrue(X.getFullStackTrace(e), X.hasCause(e, ClusterTopologyException.class) ||
- X.hasCause(e, TransactionRollbackException.class));
+ X.hasCause(e, ClusterTopologyCheckedException.class) ||
+ X.hasCause(e, TransactionRollbackException.class) ||
+ X.hasCause(e, CacheInvalidStateException.class));
}
}
@@ -1223,4 +1307,14 @@ public class TxPartitionCounterStateConsistencyTest extends TxPartitionCounterSt
@Override protected long getPartitionMapExchangeTimeout() {
return getTestTimeout();
}
+
+ /**
+ * Some tests require determined affinity assignments.
+ * Derived classes can break required order and cause hanging of tests.
+ *
+ * @return Instance name.
+ */
+ @Override public String getTestIgniteInstanceName() {
+ return "transactions.TxPartitionCounterStateConsistencyTest";
+ }
}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 74e78a5..7eae72d 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -36,6 +36,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import javax.cache.Cache;
import javax.cache.CacheException;
import javax.cache.integration.CompletionListener;
@@ -91,8 +92,11 @@ import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
import org.apache.ignite.internal.processors.cache.local.GridLocalCache;
@@ -106,7 +110,6 @@ import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.PA;
-import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.T3;
import org.apache.ignite.internal.util.typedef.internal.LT;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -571,7 +574,7 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
boolean waitNode2PartUpdate,
@Nullable Collection<ClusterNode> nodes
) throws InterruptedException {
- awaitPartitionMapExchange(waitEvicts, waitNode2PartUpdate, nodes, false);
+ awaitPartitionMapExchange(waitEvicts, waitNode2PartUpdate, nodes, false, null);
}
/**
@@ -596,6 +599,26 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
@Nullable Collection<ClusterNode> nodes,
boolean printPartState
) throws InterruptedException {
+ awaitPartitionMapExchange(waitEvicts, waitNode2PartUpdate, nodes, printPartState, null);
+ }
+
+ /**
+ * @param waitEvicts If {@code true} will wait for evictions finished.
+ * @param waitNode2PartUpdate If {@code true} will wait for nodes node2part info update finished.
+ * @param nodes Optional nodes. If {@code null} method will wait for all nodes, for non null collection nodes will
+ * be filtered
+ * @param printPartState If {@code true} will print partition state if evictions not happened.
+ * @param cacheNames Wait for specific caches.
+ * @throws InterruptedException If interrupted.
+ */
+ @SuppressWarnings("BusyWait")
+ protected void awaitPartitionMapExchange(
+ boolean waitEvicts,
+ boolean waitNode2PartUpdate,
+ @Nullable Collection<ClusterNode> nodes,
+ boolean printPartState,
+ @Nullable Set<String> cacheNames
+ ) throws InterruptedException {
long timeout = getPartitionMapExchangeTimeout();
long startTime = -1;
@@ -658,7 +681,7 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
for (IgniteCacheProxy<?, ?> c : g0.context().cache().jcaches()) {
CacheConfiguration cfg = c.context().config();
- if (cfg == null)
+ if (cfg == null || cacheNames != null && !cacheNames.contains(cfg.getName()))
continue;
if (cfg.getCacheMode() != LOCAL &&
@@ -854,14 +877,30 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
* Print partitionState for cache.
*/
protected static void printPartitionState(String cacheName, int firstParts) {
+ printPartitionState(cacheName, firstParts, G.allGrids());
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @param firstParts Count partition for print (will be print first count partition).
+ * @param nodes Grid nodes.
+ *
+ * Print partitionState for cache.
+ */
+ protected static void printPartitionState(String cacheName, int firstParts, List<? extends Ignite> nodes) {
StringBuilder sb = new StringBuilder();
sb.append("----preload sync futures----\n");
- for (Ignite ig : G.allGrids()) {
+ for (Ignite ig : nodes) {
IgniteKernal k = ((IgniteKernal)ig);
- IgniteInternalFuture<?> syncFut = k.internalCache(cacheName)
+ GridCacheAdapter<Object, Object> adapter = k.internalCache(cacheName);
+
+ if (adapter == null)
+ continue;
+
+ IgniteInternalFuture<?> syncFut = adapter
.preloader()
.syncFuture();
@@ -876,10 +915,15 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
sb.append("----rebalance futures----\n");
- for (Ignite ig : G.allGrids()) {
+ for (Ignite ig : nodes) {
IgniteKernal k = ((IgniteKernal)ig);
- IgniteInternalFuture<?> f = k.internalCache(cacheName)
+ GridCacheAdapter<Object, Object> adapter = k.internalCache(cacheName);
+
+ if (adapter == null)
+ continue;
+
+ IgniteInternalFuture<?> f = adapter
.preloader()
.rebalanceFuture();
@@ -889,22 +933,14 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
.append(" res=").append(f.isDone() ? f.get() : "N/A")
.append(" topVer=")
.append((U.hasField(f, "topVer") ?
- String.valueOf(U.<Object>field(f, "topVer")) : "[unknown] may be it is finished future"))
+ String.valueOf(U.<Object>field(f, "topVer")) : "N/A"))
.append("\n");
- Map<UUID, T2<Long, Collection<Integer>>> remaining = U.field(f, "remaining");
-
- sb.append("remaining:");
-
- if (remaining.isEmpty())
- sb.append("empty\n");
- else
- for (Map.Entry<UUID, T2<Long, Collection<Integer>>> e : remaining.entrySet())
- sb.append("\nuuid=").append(e.getKey())
- .append(" startTime=").append(e.getValue().getKey())
- .append(" parts=").append(Arrays.toString(e.getValue().getValue().toArray()))
- .append("\n");
+ Map<UUID, IgniteDhtDemandedPartitionsMap> remaining = U.field(f, "remaining");
+ sb.append("remaining: ");
+ sb.append(remaining.toString());
+ sb.append("\n");
}
catch (Throwable e) {
log.error(e.getMessage());
@@ -913,14 +949,19 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
sb.append("----partition state----\n");
- for (Ignite g : G.allGrids()) {
+ for (Ignite g : nodes) {
IgniteKernal g0 = (IgniteKernal)g;
sb.append("localNodeId=").append(g0.localNode().id())
.append(" grid=").append(g0.name())
.append("\n");
- IgniteCacheProxy<?, ?> cache = g0.context().cache().jcache(cacheName);
+ IgniteCacheProxy<?, ?> cache = null;
+ try {
+ cache = g0.context().cache().jcache(cacheName);
+ } catch (IllegalArgumentException e) {
+ continue; // Client topology.
+ }
GridDhtCacheAdapter<?, ?> dht = dht(cache);
@@ -960,20 +1001,44 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
.append("\n");
for (UUID nodeId : F.nodeIds(g0.context().discovery().allNodes())) {
- if (!nodeId.equals(g0.localNode().id()))
- sb.append(" nodeId=")
- .append(nodeId)
- .append(" part=")
- .append(p)
- .append(" counters=")
- .append(part == null ? "NA" : part.dataStore().partUpdateCounter())
- .append(" fullSize=")
- .append(part == null ? "NA" : part.fullSize())
- .append(" state=")
- .append(top.partitionState(nodeId, p))
- .append(" isAffNode=")
- .append(affNodes.contains(nodeId))
- .append("\n");
+ if (!nodeId.equals(g0.localNode().id())) {
+ top.readLock();
+
+ // Peek to remote state directly to distinguish if a partition is EVICTED or yet not initialized.
+ GridDhtPartitionFullMap map = U.field(top, "node2part");
+
+ try {
+ final GridDhtPartitionMap nodeMap = map.get(nodeId);
+
+ if (nodeMap == null)
+ continue; // Skip client node.
+
+ final GridDhtPartitionState rmtState = nodeMap.get(p);
+
+ if (rmtState != null) {
+ sb.append(" nodeId=")
+ .append(nodeId)
+ .append(" part=")
+ .append(p)
+ .append(" state=")
+ .append(rmtState)
+ .append(" isAffNode=")
+ .append(affNodes.contains(nodeId))
+ .append("\n");
+ }
+ else {
+ sb.append(" nodeId=")
+ .append(nodeId)
+ .append(" part=")
+ .append(p)
+ .append(" is null")
+ .append("\n");
+ }
+ }
+ finally {
+ top.readUnlock();
+ }
+ }
}
}
@@ -2515,4 +2580,23 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
protected void enableCheckpoints(Ignite node, boolean enable) throws IgniteCheckedException {
enableCheckpoints(Collections.singletonList(node), enable);
}
+
+ /**
+ * Loads data into a cache using stream data source.
+ *
+ * @param ignite Ignite.
+ * @param cache Cache.
+ * @param stream Key-value source stream.
+ */
+ protected void load(IgniteEx ignite, String cache, Stream<Integer> stream) {
+ try (IgniteDataStreamer<Object, Object> s = ignite.dataStreamer(cache)) {
+ final Iterator<Integer> it = stream.iterator();
+
+ while (it.hasNext()) {
+ Integer key = it.next();
+
+ s.addData(key, key);
+ }
+ }
+ }
}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite5.java
index e27ddad..8514db0 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite5.java
@@ -47,6 +47,8 @@ import org.apache.ignite.internal.processors.cache.PartitionsExchangeOnDiscovery
import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest;
import org.apache.ignite.internal.processors.cache.distributed.GridExchangeFreeCellularSwitchComplexOperationsTest;
import org.apache.ignite.internal.processors.cache.distributed.GridExchangeFreeCellularSwitchIsolationTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheGroupsPartitionLossPolicySelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePartitionLossPolicySelfTest;
import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheTxIteratorSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.dht.NotMappedPartitionInTxTest;
import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.IgniteCacheAtomicProtocolTest;
@@ -110,6 +112,9 @@ public class IgniteCacheMvccTestSuite5 {
ignoredTests.add(GridExchangeFreeCellularSwitchIsolationTest.class);
ignoredTests.add(GridExchangeFreeCellularSwitchComplexOperationsTest.class);
+ ignoredTests.add(IgniteCachePartitionLossPolicySelfTest.class);
+ ignoredTests.add(IgniteCacheGroupsPartitionLossPolicySelfTest.class);
+
return IgniteCacheTestSuite5.suite(ignoredTests);
}
}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite6.java
index 949ff41..8694dfd 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite6.java
@@ -36,6 +36,7 @@ import org.apache.ignite.internal.processors.cache.datastructures.IgniteExchange
import org.apache.ignite.internal.processors.cache.datastructures.IgniteExchangeLatchManagerDiscoHistoryTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheExchangeMergeTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheParallelStartTest;
+import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLossWithRestartsTest;
import org.apache.ignite.internal.processors.cache.distributed.ExchangeMergeStaleServerNodesTest;
import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionEvictionDuringReadThroughSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.IgniteCache150ClientsTest;
@@ -102,6 +103,9 @@ public class IgniteCacheMvccTestSuite6 {
// Read-through is not allowed with MVCC and transactional cache.
ignoredTests.add(TxOptimisticReadThroughTest.class);
+ // TODO https://issues.apache.org/jira/browse/IGNITE-13051
+ ignoredTests.add(CachePartitionLossWithRestartsTest.class);
+
List<Class<?>> suite = new ArrayList<>((IgniteCacheTestSuite6.suite(ignoredTests)));
// Add mvcc versions for skipped tests.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite7.java
index 982521d..6f4270e 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite7.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite7.java
@@ -30,6 +30,7 @@ import org.apache.ignite.internal.processors.cache.CacheDataRegionConfigurationT
import org.apache.ignite.internal.processors.cache.CacheGroupMetricsTest;
import org.apache.ignite.internal.processors.cache.MvccCacheGroupMetricsTest;
import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
+import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLostAfterSupplierHasLeftTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersMvccTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingMvccTest;
@@ -69,6 +70,7 @@ public class IgniteCacheMvccTestSuite7 {
// TODO IGNITE-10175: refactor these tests (use assume) to support both mvcc and non-mvcc modes after moving to JUnit4/5.
ignoredTests.add(CacheGroupMetricsTest.class); // See MvccCacheGroupMetricsMBeanTest
ignoredTests.add(GridCacheRebalancingPartitionCountersTest.class); // See GridCacheRebalancingPartitionCountersMvccTest
+ ignoredTests.add(CachePartitionLostAfterSupplierHasLeftTest.class); // Data loss partially works with MVCC.
// Test logic is not compatible with MVCC style tx locking.
ignoredTests.add(TxCrossCacheMapOnInvalidTopologyTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
index ee42927..940165a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
@@ -48,6 +48,7 @@ import org.apache.ignite.internal.processors.cache.IgniteAtomicCacheEntryProcess
import org.apache.ignite.internal.processors.cache.IgniteCacheEntryProcessorNodeJoinTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheIncrementTxTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheNoSyncForGetTest;
+import org.apache.ignite.internal.processors.cache.IgniteCachePartitionMapUpdateSafeLossPolicyTest;
import org.apache.ignite.internal.processors.cache.IgniteCachePartitionMapUpdateTest;
import org.apache.ignite.internal.processors.cache.IgniteClientCacheStartFailoverTest;
import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheAndNodeStop;
@@ -297,6 +298,7 @@ public class IgniteCacheTestSuite2 {
GridTestUtils.addTestIfNeeded(suite, IgniteCacheClientNodePartitionsExchangeTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, IgniteCacheClientNodeChangingTopologyTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, IgniteCacheServerNodeConcurrentStart.class, ignoredTests);
+ GridTestUtils.addTestIfNeeded(suite, IgniteCachePartitionMapUpdateSafeLossPolicyTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, IgniteCacheEntryProcessorNodeJoinTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, IgniteAtomicCacheEntryProcessorNodeJoinTest.class, ignoredTests);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index a1d9902..02877e3 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -35,7 +35,7 @@ import org.apache.ignite.internal.processors.cache.datastructures.IgniteExchange
import org.apache.ignite.internal.processors.cache.datastructures.IgniteExchangeLatchManagerDiscoHistoryTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheExchangeMergeTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheParallelStartTest;
-import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLossDetectionOnNodeLeftTest;
+import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLossWithRestartsTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheTryLockMultithreadedTest;
import org.apache.ignite.internal.processors.cache.distributed.ExchangeMergeStaleServerNodesTest;
import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionEvictionDuringReadThroughSelfTest;
@@ -144,7 +144,7 @@ public class IgniteCacheTestSuite6 {
GridTestUtils.addTestIfNeeded(suite, CacheNoAffinityExchangeTest.class, ignoredTests);
- GridTestUtils.addTestIfNeeded(suite, CachePartitionLossDetectionOnNodeLeftTest.class, ignoredTests);
+ GridTestUtils.addTestIfNeeded(suite, CachePartitionLossWithRestartsTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, TxLocalDhtMixedCacheModesTest.class, ignoredTests);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
index 1cdd5fc..1824be5 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
@@ -39,13 +39,15 @@ import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAf
import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheDataLossOnPartitionMoveTest;
-import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLostWhileClearingTest;
+import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLossWithPersistenceTest;
+import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLostAfterSupplierHasLeftTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheRentingStateRepairTest;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheScheduleResendPartitionsAfterEvictionTest;
import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheStartWithLoadTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingTest;
import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMultinodeMixedRegionsTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteLostPartitionsOnLeaveBaselineSelfTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
import org.apache.ignite.internal.processors.cache.persistence.db.CheckpointBufferDeadlockTest;
import org.apache.ignite.internal.processors.cache.transactions.AtomicPartitionCounterStateConsistencyHistoryRebalanceTest;
@@ -119,7 +121,9 @@ public class IgniteCacheTestSuite7 {
GridTestUtils.addTestIfNeeded(suite, CacheConfigurationSerializationOnDiscoveryTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, CacheConfigurationSerializationOnExchangeTest.class, ignoredTests);
- GridTestUtils.addTestIfNeeded(suite, CachePartitionLostWhileClearingTest.class, ignoredTests);
+ GridTestUtils.addTestIfNeeded(suite, CachePartitionLostAfterSupplierHasLeftTest.class, ignoredTests);
+ GridTestUtils.addTestIfNeeded(suite, CachePartitionLossWithPersistenceTest.class, ignoredTests);
+ GridTestUtils.addTestIfNeeded(suite, IgniteLostPartitionsOnLeaveBaselineSelfTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, TxCrossCacheMapOnInvalidTopologyTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, TxCrossCacheRemoteMultiplePartitionReservationTest.class, ignoredTests);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite8.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite8.java
index 894918e..a040ddf 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite8.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite8.java
@@ -20,6 +20,7 @@ package org.apache.ignite.testsuites;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import org.apache.ignite.cache.ClientCreateCacheGroupOnJoinNodeMapsTest;
import org.apache.ignite.internal.processors.cache.CacheStoreTxPutAllMultiNodeTest;
import org.apache.ignite.internal.processors.cache.GridCacheOrderedPreloadingSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalanceOrderTest;
@@ -77,6 +78,7 @@ public class IgniteCacheTestSuite8 {
GridTestUtils.addTestIfNeeded(suite, CacheStoreTxPutAllMultiNodeTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, CleanupRestoredCachesSlowTest.class, ignoredTests);
+ GridTestUtils.addTestIfNeeded(suite, ClientCreateCacheGroupOnJoinNodeMapsTest.class, ignoredTests);
return suite;
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index 2638aa5..013d360 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -478,6 +478,10 @@ public class GridMapQueryExecutor {
nodeRess.remove(reqId, segmentId, qryResults);
qryResults.close();
+
+ // If a query is cancelled before execution is started partitions have to be released.
+ if (!lazy || !qryResults.isAllClosed())
+ qryResults.releaseQueryContext();
}
else
releaseReservations(qctx);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/PartitionReservationManager.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/PartitionReservationManager.java
index 5e986d0..4177594 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/PartitionReservationManager.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/PartitionReservationManager.java
@@ -26,7 +26,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.cache.PartitionLossPolicy;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.managers.communication.GridIoPolicy;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -43,8 +42,6 @@ import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.F;
import org.jetbrains.annotations.Nullable;
-import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_SAFE;
-import static org.apache.ignite.cache.PartitionLossPolicy.READ_WRITE_SAFE;
import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
@@ -83,25 +80,6 @@ public class PartitionReservationManager implements PartitionsExchangeAware {
}
/**
- * Decide whether to ignore or proceed with lost partition.
- *
- * @param cctx Cache context.
- * @param part Partition.
- * @throws IgniteCheckedException If failed.
- */
- private static void ignoreLostPartitionIfPossible(GridCacheContext cctx, GridDhtLocalPartition part)
- throws IgniteCheckedException {
- PartitionLossPolicy plc = cctx.config().getPartitionLossPolicy();
-
- if (plc != null) {
- if (plc == READ_ONLY_SAFE || plc == READ_WRITE_SAFE) {
- throw new CacheInvalidStateException("Failed to execute query because cache partition has been " +
- "lost [cacheName=" + cctx.name() + ", part=" + part + ']');
- }
- }
- }
-
- /**
* @param cctx Cache context.
* @param p Partition ID.
* @return Partition.
@@ -224,7 +202,7 @@ public class PartitionReservationManager implements PartitionsExchangeAware {
if (partState != OWNING) {
if (partState == LOST)
- ignoreLostPartitionIfPossible(cctx, part);
+ failQueryOnLostData(cctx, part);
else {
return new PartitionReservation(reserved,
String.format("Failed to reserve partitions " +
@@ -271,7 +249,7 @@ public class PartitionReservationManager implements PartitionsExchangeAware {
if (partState != OWNING) {
if (partState == LOST)
- ignoreLostPartitionIfPossible(cctx, part);
+ failQueryOnLostData(cctx, part);
else {
return new PartitionReservation(reserved,
String.format("Failed to reserve partitions for " +
@@ -325,6 +303,16 @@ public class PartitionReservationManager implements PartitionsExchangeAware {
}
/**
+ * @param cctx Cache context.
+ * @param part Partition.
+ */
+ private static void failQueryOnLostData(GridCacheContext cctx, GridDhtLocalPartition part)
+ throws IgniteCheckedException {
+ throw new CacheInvalidStateException("Failed to execute query because cache partition has been " +
+ "lost [cacheName=" + cctx.name() + ", part=" + part + ']');
+ }
+
+ /**
* Cleanup group reservations cache on change affinity version.
*/
@Override public void onDoneAfterTopologyUnlock(final GridDhtPartitionsExchangeFuture fut) {
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReducePartitionMapper.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReducePartitionMapper.java
index b748ef2..394bbb5 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReducePartitionMapper.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReducePartitionMapper.java
@@ -25,14 +25,15 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.IntStream;
import javax.cache.CacheException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.CacheServerNotFoundException;
-import org.apache.ignite.cache.PartitionLossPolicy;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
import org.apache.ignite.internal.util.typedef.F;
@@ -41,8 +42,6 @@ import org.h2.util.IntArray;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_SAFE;
-import static org.apache.ignite.cache.PartitionLossPolicy.READ_WRITE_SAFE;
import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
/**
@@ -86,17 +85,15 @@ public class ReducePartitionMapper {
for (int cacheId : cacheIds) {
GridCacheContext<?, ?> cctx = cacheContext(cacheId);
- PartitionLossPolicy plc = cctx.config().getPartitionLossPolicy();
-
- if (plc != READ_ONLY_SAFE && plc != READ_WRITE_SAFE)
- continue;
-
Collection<Integer> lostParts = cctx.topology().lostPartitions();
- for (int part : lostParts) {
- if (parts == null || Arrays.binarySearch(parts, part) >= 0) {
- throw new CacheException("Failed to execute query because cache partition has been " +
- "lost [cacheName=" + cctx.name() + ", part=" + part + ']');
+ if (!lostParts.isEmpty()) {
+ int lostPart = parts == null ? lostParts.iterator().next() :
+ IntStream.of(parts).filter(lostParts::contains).findFirst().orElse(-1);
+
+ if (lostPart >= 0) {
+ throw new CacheException(new CacheInvalidStateException("Failed to execute query because cache " +
+ "partition has been lostPart [cacheName=" + cctx.name() + ", part=" + lostPart + ']'));
}
}
}
@@ -419,8 +416,8 @@ public class ReducePartitionMapper {
List<ClusterNode> owners = cctx.topology().owners(p);
if (F.isEmpty(owners)) {
- // Handle special case: no mapping is configured for a partition.
- if (F.isEmpty(cctx.affinity().assignment(NONE).get(p))) {
+ // Handle special case: no mapping is configured for a partition or a lost partition is found.
+ if (F.isEmpty(cctx.affinity().assignment(NONE).get(p)) || cctx.topology().lostPartitions().contains(p)) {
partLocs[p] = UNMAPPED_PARTS; // Mark unmapped partition.
continue;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java
index 909d0dd..27bd8ad 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java
@@ -76,6 +76,9 @@ public class IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest extends Igni
for (int i = GRID_CNT - RESTARTED_NODE_CNT; i < GRID_CNT; ++i)
startGrid(i);
+ // Wait for rebalancing to prevent data loss.
+ awaitPartitionMapExchange();
+
GridTestUtils.waitForCondition(new GridAbsPredicate() {
@Override public boolean apply() {
return !stopRestartThread.get();
@@ -87,14 +90,6 @@ public class IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest extends Igni
}, "restart-node");
}
- /** {@inheritDoc} */
- @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception {
- CacheConfiguration ccfg = super.cacheConfiguration(igniteInstanceName);
- ccfg.setBackups(0);
-
- return ccfg;
- }
-
/**
* @param cacheName Cache name.
* @param mode Atomicity mode.
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest.java
index 11b71da..81b3053 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest.java
@@ -24,8 +24,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
/**
* Test to validate https://issues.apache.org/jira/browse/IGNITE-2310.
*/
-public class IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest
- extends IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest {
+public class IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest extends IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest {
/** {@inheritDoc} */
@Override protected CacheAtomicityMode atomicityMode() {
return TRANSACTIONAL;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicSqlRestoreTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicSqlRestoreTest.java
index 1eb36fb..321fc28 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicSqlRestoreTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicSqlRestoreTest.java
@@ -357,6 +357,8 @@ public class IgniteDynamicSqlRestoreTest extends GridCommonAbstractTest implemen
IgniteEx ig = startGrid(1);
ig.cluster().active(true);
+ resetBaselineTopology();
+ ig.resetLostPartitions(Collections.singleton(TEST_CACHE_NAME));
IgniteCache<Object, Object> cache = ig.cache(TEST_CACHE_NAME);
@@ -486,6 +488,7 @@ public class IgniteDynamicSqlRestoreTest extends GridCommonAbstractTest implemen
ig.cluster().active(true);
ig = startGrid(1);
+ ig.resetLostPartitions(Collections.singleton(TEST_CACHE_NAME));
//then: config should be merged
try (IgniteDataStreamer<Object, Object> s = ig.dataStreamer(TEST_CACHE_NAME)) {
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java
index aceb09d..53dfd83 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java
@@ -17,10 +17,12 @@
package org.apache.ignite.internal.processors.cache;
+import java.util.List;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePartitionLossPolicySelfTest;
/**
@@ -28,57 +30,28 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePartit
*/
public class IndexingCachePartitionLossPolicySelfTest extends IgniteCachePartitionLossPolicySelfTest {
/** {@inheritDoc} */
- @Override protected CacheConfiguration<Integer, Integer> cacheConfiguration() {
- CacheConfiguration<Integer, Integer> ccfg = super.cacheConfiguration();
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
- ccfg.setIndexedTypes(Integer.class, Integer.class);
+ for (CacheConfiguration ccfg : cfg.getCacheConfiguration())
+ ccfg.setIndexedTypes(Integer.class, Integer.class);
- return ccfg;
+ return cfg;
}
/** {@inheritDoc} */
- @Override protected void checkQueryPasses(Ignite node, boolean loc, int... parts) {
- executeQuery(node, loc, parts);
- }
-
- /** {@inheritDoc} */
- @Override protected void checkQueryFails(Ignite node, boolean loc, int... parts) {
- // TODO: Local queries ignore partition loss, see https://issues.apache.org/jira/browse/IGNITE-7039.
- if (loc)
- return;
-
- try {
- executeQuery(node, loc, parts);
-
- fail("Exception is not thrown.");
- }
- catch (Exception e) {
- boolean exp = e.getMessage() != null &&
- e.getMessage().contains("Failed to execute query because cache partition has been lost");
-
- if (!exp)
- throw e;
- }
- }
-
- /**
- * Execute SQL query on a given node.
- *
- * @param parts Partitions.
- * @param node Node.
- * @param loc Local flag.
- */
- private static void executeQuery(Ignite node, boolean loc, int... parts) {
- IgniteCache cache = node.cache(DEFAULT_CACHE_NAME);
+ @Override protected List<?> runQuery(Ignite ig, String cacheName, boolean loc, int part) {
+ IgniteCache cache = ig.cache(cacheName);
SqlFieldsQuery qry = new SqlFieldsQuery("SELECT * FROM Integer");
- if (parts != null && parts.length != 0)
- qry.setPartitions(parts);
+ if (part != -1)
+ qry.setPartitions(part);
- if (loc)
- qry.setLocal(true);
+ // TODO https://issues.apache.org/jira/browse/IGNITE-7039
+ // if (loc)
+ // qry.setLocal(true);
- cache.query(qry).getAll();
+ return cache.query(qry).getAll();
}
}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryReservationOnUnstableTopologyTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryReservationOnUnstableTopologyTest.java
new file mode 100644
index 0000000..ab1e45a
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryReservationOnUnstableTopologyTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+/**
+ * Tests partition reservation releasing for queries over unstable topology.
+ */
+public class IgniteCacheQueryReservationOnUnstableTopologyTest extends GridCommonAbstractTest {
+ /** */
+ private static final int PARTS = 32;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setConsistentId(igniteInstanceName);
+
+ cfg.setCacheConfiguration(new CacheConfiguration<Integer, Integer>(DEFAULT_CACHE_NAME)
+ .setIndexedTypes(Integer.class, Integer.class)
+ .setAffinity(new RendezvousAffinityFunction(false, PARTS)).setBackups(1));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+ }
+
+ /** */
+ @Test
+ public void testQueryReservationRelease() throws Exception {
+ doTestQueryReservationRelease(false);
+ }
+
+ /** */
+ @Test
+ public void testQueryReservationRelease_SmallPage() throws Exception {
+ doTestQueryReservationRelease(true);
+ }
+
+ /**
+ * @param smallPage {@code True} to set small page size.
+ *
+ * @throws Exception If failed.
+ */
+ private void doTestQueryReservationRelease(boolean smallPage) throws Exception {
+ IgniteEx crd = startGrids(2);
+
+ awaitPartitionMapExchange();
+
+ final int keys = PARTS * 5;
+
+ for (int i = 0; i < keys; i++)
+ crd.cache(DEFAULT_CACHE_NAME).put(i, i);
+
+ AtomicBoolean stop = new AtomicBoolean();
+
+ IgniteInternalFuture<?> qryFut = multithreadedAsync(new Runnable() {
+ @Override public void run() {
+ while (!stop.get()) {
+ SqlFieldsQuery qry = new SqlFieldsQuery("SELECT * FROM Integer");
+
+ if (smallPage)
+ qry.setPageSize(1);
+
+ assertEquals(keys, grid(0).cache(DEFAULT_CACHE_NAME).query(qry).getAll().size());
+ }
+ }
+ }, 3, "qry-thread");
+
+ doSleep(100);
+
+ startGrid(2);
+ startGrid(3);
+
+ stop.set(true);
+
+ awaitPartitionMapExchange();
+
+ qryFut.get();
+ }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteCacheGroupsWithRestartsTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteCacheGroupsWithRestartsTest.java
index 45cbad1..d6b3aff 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteCacheGroupsWithRestartsTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteCacheGroupsWithRestartsTest.java
@@ -325,6 +325,8 @@ public class IgniteCacheGroupsWithRestartsTest extends GridCommonAbstractTest {
assertNull(ignite.cachex(getCacheName(0)));
+ ignite.resetLostPartitions(Arrays.asList(getCacheName(0), getCacheName(1), getCacheName(2)));
+
awaitPartitionMapExchange();
VisorFindAndDeleteGarbageInPersistenceTaskResult taskResult = doFindAndRemove.apply(ex1, false);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
index 87cb8ab..c34468f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
@@ -25,6 +25,8 @@ import java.nio.ByteOrder;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -75,10 +77,12 @@ import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry;
import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntryType;
@@ -245,6 +249,8 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
stopAllGrids();
cleanPersistenceDir();
@@ -252,6 +258,8 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
+ super.afterTest();
+
stopAllGrids();
cleanPersistenceDir();
@@ -796,6 +804,11 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
IgniteCache<Object, Object> cache = cacheGrid.cache(CACHE_NAME);
+ // Expecting lost partitions.
+ assertFalse(cache.lostPartitions().isEmpty());
+
+ cacheGrid.resetLostPartitions(Collections.singleton(CACHE_NAME));
+
for (int i = 0; i < ENTRY_COUNT; i++)
assertEquals(new IndexedObject(i), cache.get(i));
@@ -844,6 +857,12 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
cacheGrid = startGrid(1);
IgniteCache<Object, Object> cache = cacheGrid.cache(CACHE_NAME);
+
+ // Expecting lost partitions.
+ assertFalse(cache.lostPartitions().isEmpty());
+
+ cacheGrid.resetLostPartitions(Collections.singleton(CACHE_NAME));
+
IgniteCache<Object, Object> locCache = cacheGrid.cache(LOC_CACHE_NAME);
for (int i = 0; i < LARGE_ENTRY_COUNT; i++) {
@@ -892,9 +911,15 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
startGrid(1);
- Boolean res = rmt.call(new VerifyCallable());
+ final GridDhtPartitionTopology top = ctrlGrid.cachex(CACHE_NAME).context().topology();
+
+ waitForReadyTopology(top, new AffinityTopologyVersion(3, 0));
+
+ assertFalse(top.lostPartitions().isEmpty());
+
+ int res = rmt.call(new VerifyCallable());
- assertTrue(res);
+ assertEquals(0, res);
}
/**
@@ -928,9 +953,15 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
startGrid(1);
- Boolean res = rmt.call(new VerifyLargeCallable());
+ final GridDhtPartitionTopology top = ctrlGrid.cachex(CACHE_NAME).context().topology();
- assertTrue(res);
+ waitForReadyTopology(top, new AffinityTopologyVersion(3, 0));
+
+ assertFalse(top.lostPartitions().isEmpty());
+
+ int res = rmt.call(new VerifyLargeCallable());
+
+ assertEquals(0, res);
}
/**
@@ -1847,13 +1878,13 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
/**
*
*/
- private static class VerifyCallable implements IgniteCallable<Boolean> {
+ private static class VerifyCallable implements IgniteCallable<Integer> {
/** */
@IgniteInstanceResource
private Ignite ignite;
/** {@inheritDoc} */
- @Override public Boolean call() throws Exception {
+ @Override public Integer call() throws Exception {
try {
boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
@Override public boolean applyx() {
@@ -1868,6 +1899,15 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
}
IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+
+ // Partitions are expected to be lost on killed node.
+ final Collection<Integer> lost = cache.lostPartitions();
+
+ if (cache.getConfiguration(CacheConfiguration.class).getAffinity().partitions() != lost.size())
+ return 1;
+
+ ignite.resetLostPartitions(Collections.singleton(CACHE_NAME));
+
IgniteCache<Object, Object> locCache = ignite.cache(LOC_CACHE_NAME);
for (int i = 0; i < ENTRY_COUNT; i++) {
@@ -1877,7 +1917,7 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
if (val == null) {
ignite.log().warning("Failed to find a value for PARTITIONED cache key: " + i);
- return false;
+ return 2;
}
}
@@ -1887,12 +1927,12 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
if (val == null) {
ignite.log().warning("Failed to find a value for LOCAL cache key: " + i);
- return false;
+ return 2;
}
}
}
- return true;
+ return 0;
}
}
@@ -2013,13 +2053,13 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
/**
*
*/
- private static class VerifyLargeCallable implements IgniteCallable<Boolean> {
+ private static class VerifyLargeCallable implements IgniteCallable<Integer> {
/** */
@IgniteInstanceResource
private Ignite ignite;
/** {@inheritDoc} */
- @Override public Boolean call() throws Exception {
+ @Override public Integer call() throws Exception {
try {
boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
@Override public boolean applyx() {
@@ -2035,6 +2075,14 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+ // Partitions are expected to be lost on killed node.
+ final Collection<Integer> lost = cache.lostPartitions();
+
+ if (cache.getConfiguration(CacheConfiguration.class).getAffinity().partitions() != lost.size())
+ return 1;
+
+ ignite.resetLostPartitions(Collections.singleton(CACHE_NAME));
+
for (int i = 0; i < LARGE_ENTRY_COUNT; i++) {
final long[] data = new long[LARGE_ARR_SIZE];
@@ -2045,13 +2093,13 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
if (val == null) {
ignite.log().warning("Failed to find a value for key: " + i);
- return false;
+ return 2;
}
assertTrue(Arrays.equals(data, (long[])val));
}
- return true;
+ return 0;
}
}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.java
index 1a7ddae..7ad4ad6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.java
@@ -64,9 +64,7 @@ public class IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest extends Ign
startGrids(gridCnt + 1);
- grid(0).active(true);
-
- stopGrid(gridCnt);
+ grid(0).cluster().active(true);
startGrid(gridCnt + 1);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.java
index 6c92310..af4547b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.java
@@ -55,9 +55,7 @@ public class IgniteBaselineLockPartitionOnAffinityRunTxCacheTest extends IgniteC
startGrids(gridCnt + 1);
- grid(0).active(true);
-
- stopGrid(gridCnt);
+ grid(0).cluster().active(true);
startGrid(gridCnt + 1);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite2.java
index f718a87..6396926 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite2.java
@@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQ
import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryNodeRestartSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryNodeRestartSelfTest2;
import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryNodeRestartTxSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryReservationOnUnstableTopologyTest;
import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryStopOnCancelOrTimeoutDistributedJoinSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.IgniteSqlQueryWithBaselineTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxMultiNodeBasicTest;
@@ -175,7 +176,9 @@ import org.junit.runners.Suite;
IgniteCacheLocalQueryDefaultTimeoutSelfTest.class,
- SqlQueriesTopologyMappingTest.class
+ SqlQueriesTopologyMappingTest.class,
+
+ IgniteCacheQueryReservationOnUnstableTopologyTest.class
})
public class IgniteBinaryCacheQueryTestSuite2 {
}
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PartitionLossTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PartitionLossTest.cs
index 333b3f5..d502550 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PartitionLossTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PartitionLossTest.cs
@@ -89,7 +89,7 @@ namespace Apache.Ignite.Core.Tests.Cache
[Test]
public void TestReadOnlyAll()
{
- TestPartitionLoss(PartitionLossPolicy.ReadOnlyAll, false, false);
+ TestPartitionLoss(PartitionLossPolicy.ReadOnlyAll, false, true);
}
/// <summary>
@@ -98,7 +98,7 @@ namespace Apache.Ignite.Core.Tests.Cache
[Test]
public void TestReadWriteAll()
{
- TestPartitionLoss(PartitionLossPolicy.ReadWriteAll, true, false);
+ TestPartitionLoss(PartitionLossPolicy.ReadWriteAll, true, true);
}
/// <summary>
@@ -139,10 +139,10 @@ namespace Apache.Ignite.Core.Tests.Cache
{
VerifyCacheOperations(cache, part, canWrite, safe);
- // Check recover cache.
+ // Check reads are possible from a cache in recovery mode.
var recoverCache = cache.WithPartitionRecover();
- recoverCache[part] = part;
- Assert.AreEqual(part, recoverCache[part]);
+ int res;
+ Assert.IsFalse(recoverCache.TryGet(part, out res));
}
// Reset and verify.
@@ -167,8 +167,8 @@ namespace Apache.Ignite.Core.Tests.Cache
var ex = Assert.Throws<CacheException>(() => cache.TryGet(part, out val));
Assert.AreEqual(string.Format(
"class org.apache.ignite.internal.processors.cache.CacheInvalidStateException" +
- ": Failed to execute cache operation (all partition owners have left the grid, " +
- "partition data has been lost) [cacheName={0}, part={1}," +
+ ": Failed to execute the cache operation (all partition owners have left the grid, " +
+ "partition data has been lost) [cacheName={0}, partition={1}," +
" key=UserKeyCacheObjectImpl [part={1}, val={1}, hasValBytes=false]]",
CacheName, part), ex.Message);
}
@@ -185,8 +185,8 @@ namespace Apache.Ignite.Core.Tests.Cache
var ex = Assert.Throws<CacheException>(() => cache.Put(part, part));
Assert.AreEqual(string.Format(
"class org.apache.ignite.internal.processors.cache.CacheInvalidStateException: " +
- "Failed to execute cache operation (all partition owners have left the grid, " +
- "partition data has been lost) [cacheName={0}, part={1}, key={1}]",
+ "Failed to execute the cache operation (all partition owners have left the grid, " +
+ "partition data has been lost) [cacheName={0}, partition={1}, key={1}]",
CacheName, part), ex.Message);
}
else
@@ -199,7 +199,7 @@ namespace Apache.Ignite.Core.Tests.Cache
{
var ex = Assert.Throws<CacheException>(() => cache.Put(part, part));
Assert.AreEqual(string.Format(
- "class org.apache.ignite.IgniteCheckedException: " +
+ "class org.apache.ignite.internal.processors.cache.CacheInvalidStateException: " +
"Failed to write to cache (cache is moved to a read-only state): {0}",
CacheName), ex.Message);
}