You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dg...@apache.org on 2018/10/26 12:17:06 UTC
ignite git commit: IGNITE-7196 Restore binary state before node join
to topology - Fixes #4520.
Repository: ignite
Updated Branches:
refs/heads/master 28e3dec5b -> c7449f6c6
IGNITE-7196 Restore binary state before node join to topology - Fixes #4520.
Signed-off-by: Dmitriy Govorukhin <dm...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c7449f6c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c7449f6c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c7449f6c
Branch: refs/heads/master
Commit: c7449f6c6a318debce84883aba69e6c028af5e48
Parents: 28e3dec
Author: Maxim Muzafarov <ma...@gmail.com>
Authored: Fri Oct 26 15:16:40 2018 +0300
Committer: Dmitriy Govorukhin <dm...@gmail.com>
Committed: Fri Oct 26 15:16:40 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/IgniteKernal.java | 2 +
.../pagemem/store/IgnitePageStoreManager.java | 12 +-
.../pagemem/wal/IgniteWriteAheadLogManager.java | 7 +-
.../wal/record/MemoryRecoveryRecord.java | 7 +-
.../processors/cache/GridCacheProcessor.java | 2 +-
.../GridDhtPartitionsExchangeFuture.java | 76 ++----
.../cache/mvcc/MvccProcessorImpl.java | 18 +-
.../persistence/DatabaseLifecycleListener.java | 13 +-
.../GridCacheDatabaseSharedManager.java | 256 +++++++++++++------
.../IgniteCacheDatabaseSharedManager.java | 87 +++++--
.../persistence/file/FilePageStoreManager.java | 37 ++-
.../persistence/metastorage/MetaStorage.java | 5 -
.../wal/FileWriteAheadLogManager.java | 27 +-
.../wal/FsyncModeFileWriteAheadLogManager.java | 27 +-
.../file/IgnitePdsDiskErrorsRecoveringTest.java | 12 +-
.../persistence/db/wal/WalCompactionTest.java | 15 ++
.../pagemem/NoOpPageStoreManager.java | 6 +
.../persistence/pagemem/NoOpWALManager.java | 5 -
.../db/wal/IgniteWalRecoveryTest.java | 123 +++++++++
19 files changed, 502 insertions(+), 235 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 1f9a5e8..250fbd7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1046,6 +1046,8 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
fillNodeAttributes(clusterProc.updateNotifierEnabled());
ctx.cache().context().database().notifyMetaStorageSubscribersOnReadyForRead();
+
+ ctx.cache().context().database().startMemoryRestore(ctx);
}
catch (Throwable e) {
U.error(
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index d7c61e9..1408383 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.pagemem.store;
import java.nio.ByteBuffer;
import java.util.Map;
+import java.util.function.Predicate;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.pagemem.PageMemory;
@@ -243,11 +244,20 @@ public interface IgnitePageStoreManager extends GridCacheSharedManager, IgniteCh
public void cleanupPersistentSpace(CacheConfiguration cacheConfiguration) throws IgniteCheckedException;
/**
- * Cleanup persistent space for all caches.
+ * Cleanup persistent space for all caches except metastore.
*/
public void cleanupPersistentSpace() throws IgniteCheckedException;
/**
+ * Cleanup cache store whether it matches the provided predicate and if matched
+ * store was previously initizlized.
+ *
+ * @param cacheGrpPred Predicate to match by id cache group stores to clean.
+ * @param cleanFiles {@code True} to delete all persisted files related to particular store.
+ */
+ public void cleanupPageStoreIfMatch(Predicate<Integer> cacheGrpPred, boolean cleanFiles);
+
+ /**
* Creates and initializes cache work directory retrieved from {@code cacheCfg}.
*
* @param cacheCfg Cache configuration.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index 68428d0..679eec9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -47,6 +47,8 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
/**
* Resumes logging after start. When WAL manager is started, it will skip logging any updates until this
* method is called to avoid logging changes induced by the state restore procedure.
+ *
+ * @throws IgniteCheckedException If fails.
*/
public void resumeLogging(WALPointer lastWrittenPtr) throws IgniteCheckedException;
@@ -178,9 +180,4 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
* @param grpId Group id.
*/
public boolean disabled(int grpId);
-
- /**
- * Cleanup all directories relating to WAL (e.g. work WAL dir, archive WAL dir).
- */
- public void cleanupWalDirectories() throws IgniteCheckedException;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java
index 8843eee..92658cc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java
@@ -20,8 +20,13 @@ package org.apache.ignite.internal.pagemem.wal.record;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
- * Marker that we start memory recovering
+ * Marker that we start memory recovering.
+ *
+ * @deprecated Previously, used to track node started\stopped states. But in fact only
+ * mark files created by method GridCacheDatabaseSharedManager#nodeStart(WALPointer)
+ * used. Should be removed in 3.0 release.
*/
+@Deprecated
public class MemoryRecoveryRecord extends WALRecord {
/** Create timestamp, millis */
private long time;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 2394ad1..d33e929 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -2264,7 +2264,7 @@ public class GridCacheProcessor extends GridProcessorAdapter implements Metastor
* @param affNode {@code true} if it is affinity node for cache.
* @throws IgniteCheckedException if failed.
*/
- private void preparePageStore(DynamicCacheDescriptor desc, boolean affNode) throws IgniteCheckedException {
+ public void preparePageStore(DynamicCacheDescriptor desc, boolean affNode) throws IgniteCheckedException {
if (sharedCtx.pageStore() != null && affNode)
initializationProtector.protect(
desc.groupDescriptor().groupId(),
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index d54b1ab..23e47e5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -50,7 +50,6 @@ import org.apache.ignite.cache.CacheRebalanceMode;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.NearCacheConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
import org.apache.ignite.internal.IgniteDiagnosticAware;
@@ -83,7 +82,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.GridCacheUtils;
-import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext;
import org.apache.ignite.internal.processors.cache.StateChangeRequest;
import org.apache.ignite.internal.processors.cache.WalStateAbstractMessage;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter;
@@ -870,13 +868,22 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
* @throws IgniteCheckedException If failed.
*/
private IgniteInternalFuture<?> initCachesOnLocalJoin() throws IgniteCheckedException {
- boolean baselineNode = isLocalNodeInBaseline();
-
- if (!baselineNode) {
+ if (!isLocalNodeInBaseline()) {
cctx.exchange().exchangerBlockingSectionBegin();
try {
- cctx.cache().cleanupCachesDirectories();
+ cctx.database().cleanupRestoredCaches();
+
+ for (DynamicCacheDescriptor desc : cctx.cache().cacheDescriptors().values()) {
+ if (CU.isPersistentCache(desc.cacheConfiguration(),
+ cctx.gridConfig().getDataStorageConfiguration())) {
+ // Perform cache init from scratch.
+ cctx.cache().preparePageStore(desc, true);
+ }
+ }
+
+ // Set initial node started marker.
+ cctx.database().nodeStart(null);
}
finally {
cctx.exchange().exchangerBlockingSectionEnd();
@@ -892,34 +899,11 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
cctx.exchange().exchangerBlockingSectionEnd();
}
- LocalJoinCachesContext locJoinCtx = exchActions == null ? null : exchActions.localJoinContext();
-
- List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> caches = locJoinCtx == null ? null :
- locJoinCtx.caches();
-
- if (!cctx.kernalContext().clientNode()) {
- List<DynamicCacheDescriptor> startDescs = new ArrayList<>();
-
- if (caches != null) {
- for (T2<DynamicCacheDescriptor, NearCacheConfiguration> c : caches) {
- DynamicCacheDescriptor startDesc = c.get1();
-
- if (CU.isPersistentCache(startDesc.cacheConfiguration(), cctx.gridConfig().getDataStorageConfiguration()))
- startDescs.add(startDesc);
- }
- }
-
- cctx.exchange().exchangerBlockingSectionBegin();
-
- try {
- cctx.database().readCheckpointAndRestoreMemory(startDescs, !baselineNode);
- }
- finally {
- cctx.exchange().exchangerBlockingSectionEnd();
- }
- }
+ if (!cctx.kernalContext().clientNode())
+ cctx.database().onDoneRestoreBinaryMemory();
- IgniteInternalFuture<?> cachesRegistrationFut = cctx.cache().startCachesOnLocalJoin(initialVersion(), locJoinCtx);
+ IgniteInternalFuture<?> cachesRegistrationFut = cctx.cache().startCachesOnLocalJoin(initialVersion(),
+ exchActions == null ? null : exchActions.localJoinContext());
ensureClientCachesStarted();
@@ -1080,26 +1064,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
cctx.exchange().exchangerBlockingSectionEnd();
}
- if (!cctx.kernalContext().clientNode()) {
- List<DynamicCacheDescriptor> startDescs = new ArrayList<>();
-
- for (ExchangeActions.CacheActionData startReq : exchActions.cacheStartRequests()) {
- DynamicCacheDescriptor desc = startReq.descriptor();
-
- if (CU.isPersistentCache(desc.cacheConfiguration(),
- cctx.gridConfig().getDataStorageConfiguration()))
- startDescs.add(desc);
- }
-
- cctx.exchange().exchangerBlockingSectionBegin();
-
- try {
- cctx.database().readCheckpointAndRestoreMemory(startDescs, false);
- }
- finally {
- cctx.exchange().exchangerBlockingSectionEnd();
- }
- }
+ if (!cctx.kernalContext().clientNode())
+ cctx.database().onDoneRestoreBinaryMemory();
assert registerCachesFuture == null : "No caches registration should be scheduled before new caches have started.";
@@ -2148,9 +2114,9 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
cctx.database().releaseHistoryForExchange();
- cctx.database().rebuildIndexesIfNeeded(this);
-
if (err == null) {
+ cctx.database().rebuildIndexesIfNeeded(this);
+
for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
if (!grp.isLocal())
grp.topology().onExchangeDone(this, grp.affinity().readyAffinity(res), false);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java
index a304bef..bf51103 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java
@@ -324,10 +324,22 @@ public class MvccProcessorImpl extends GridProcessorAdapter implements MvccProce
}
/** {@inheritDoc} */
- @SuppressWarnings("ConstantConditions")
- @Override public void beforeMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {
+ @Override public void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {
+ txLogPageStoreInit(mgr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void beforeResumeWalLogging(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {
+ // In case of blt changed we should re-init TX_LOG cache.
+ txLogPageStoreInit(mgr);
+ }
+
+ /**
+ * @param mgr Database shared manager.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void txLogPageStoreInit(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {
assert CU.isPersistenceEnabled(ctx.config());
- assert txLog == null;
ctx.cache().context().pageStore().initialize(TX_LOG_CACHE_ID, 1,
TX_LOG_CACHE_NAME, mgr.dataRegion(TX_LOG_CACHE_NAME).memoryMetrics());
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java
index f96cdd9..ae65c77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java
@@ -23,7 +23,6 @@ import org.apache.ignite.IgniteCheckedException;
*
*/
public interface DatabaseLifecycleListener {
-
/**
* @param mgr Database shared manager.
*
@@ -31,10 +30,16 @@ public interface DatabaseLifecycleListener {
void onInitDataRegions(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException;
/**
- * @param mgr Page store manager.
- *
+ * @param mgr Database shared manager.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException;
+
+ /**
+ * @param mgr Database shared manager.
+ * @throws IgniteCheckedException If failed.
*/
- void beforeMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException;
+ public void beforeResumeWalLogging(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException;
/**
* @param mgr Database shared manager.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 3e2e6a1..4af1d8e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -53,6 +53,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.LongAdder;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -96,7 +97,6 @@ import org.apache.ignite.internal.pagemem.wal.record.CacheState;
import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
-import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
@@ -159,8 +159,8 @@ import org.apache.ignite.lang.IgniteOutClosure;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.mxbean.DataStorageMetricsMXBean;
import org.apache.ignite.thread.IgniteThread;
-import org.jetbrains.annotations.NotNull;
import org.apache.ignite.thread.IgniteThreadPoolExecutor;
+import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jsr166.ConcurrentLinkedHashMap;
@@ -168,6 +168,7 @@ import static java.nio.file.StandardOpenOption.READ;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_READ_LOCK_TIMEOUT;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME;
import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD;
@@ -282,6 +283,14 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** */
private boolean stopping;
+ /**
+ * The position of last seen WAL pointer. Used for resumming logging from this pointer.
+ *
+ * If binary memory recovery pefrormed on node start, the checkpoint END pointer will store
+ * not the last WAL pointer and can't be used for resumming logging.
+ */
+ private volatile WALPointer walTail;
+
/** Checkpoint runner thread pool. If null tasks are to be run in single thread */
@Nullable private IgniteThreadPoolExecutor asyncRunner;
@@ -330,7 +339,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** Number of pages in current checkpoint at the beginning of checkpoint. */
private volatile int currCheckpointPagesCnt;
- /** */
+ /**
+ * MetaStorage instance. Value {@code null} means storage not initialized yet.
+ * Guarded by {@link GridCacheDatabaseSharedManager#checkpointReadLock()}
+ */
private MetaStorage metaStorage;
/** */
@@ -434,6 +446,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
+ * Create metastorage data region configuration with enabled persistence by default.
+ *
* @param storageCfg Data storage configuration.
* @return Data region configuration.
*/
@@ -520,10 +534,43 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
}
- /**
- * Cleanup checkpoint directory.
- */
+ /** {@inheritDoc} */
+ @Override public void cleanupRestoredCaches() {
+ if (dataRegionMap == null)
+ return;
+
+ for (CacheGroupDescriptor grpDesc : cctx.cache().cacheGroupDescriptors().values()) {
+ String regionName = grpDesc.config().getDataRegionName();
+
+ DataRegion region = dataRegionMap.get(regionName == null ? DFLT_DATA_REG_DEFAULT_NAME : regionName);
+
+ if (region == null)
+ continue;
+
+ int partitions = grpDesc.config().getAffinity().partitions();
+
+ if (region.pageMemory() instanceof PageMemoryEx) {
+ PageMemoryEx memEx = (PageMemoryEx)region.pageMemory();
+
+ for (int partId = 0; partId < partitions; partId++)
+ memEx.invalidate(grpDesc.groupId(), partId);
+ }
+ }
+
+ storeMgr.cleanupPageStoreIfMatch(
+ new Predicate<Integer>() {
+ @Override public boolean test(Integer grpId) {
+ return MetaStorage.METASTORAGE_CACHE_ID != grpId;
+ }
+ },
+ true);
+ }
+
+ /** {@inheritDoc} */
@Override public void cleanupCheckpointDirectory() throws IgniteCheckedException {
+ if (cpHistory != null)
+ cpHistory = new CheckpointHistory(cctx.kernalContext());
+
try {
try (DirectoryStream<Path> files = Files.newDirectoryStream(cpDir.toPath())) {
for (Path path : files)
@@ -645,7 +692,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
checkpointReadLock();
try {
- restoreMemory(status, true, storePageMem, false);
+ restoreMemory(status, true, storePageMem, Collections.emptySet());
metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true);
@@ -658,12 +705,18 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
notifyMetastorageReadyForRead();
}
finally {
- checkpointReadUnlock();
- }
+ metaStorage = null;
+
+ storePageMem.stop(true);
- metaStorage = null;
+ cctx.pageStore().cleanupPageStoreIfMatch(new Predicate<Integer>() {
+ @Override public boolean test(Integer grpId) {
+ return MetaStorage.METASTORAGE_CACHE_ID == grpId;
+ }
+ }, false);
- storePageMem.stop(true);
+ checkpointReadUnlock();
+ }
}
catch (StorageException e) {
cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
@@ -680,7 +733,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
snapshotMgr = cctx.snapshot();
- if (!cctx.localNode().isClient()) {
+ if (!cctx.kernalContext().clientNode()) {
initDataBase();
registrateMetricsMBean();
@@ -789,11 +842,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/** {@inheritDoc} */
- @Override public void readCheckpointAndRestoreMemory(
- List<DynamicCacheDescriptor> cachesToStart,
- boolean restoreMetastorageOnly
- ) throws IgniteCheckedException {
- assert !cctx.localNode().isClient();
+ @Override public void onDoneRestoreBinaryMemory() throws IgniteCheckedException {
+ assert !cctx.kernalContext().clientNode();
long time = System.currentTimeMillis();
@@ -801,50 +851,88 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
try {
for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext()))
- lsnr.beforeMemoryRestore(this);
+ lsnr.beforeResumeWalLogging(this);
- if (!F.isEmpty(cachesToStart)) {
- for (DynamicCacheDescriptor desc : cachesToStart) {
- if (CU.affinityNode(cctx.localNode(), desc.cacheConfiguration().getNodeFilter()))
- storeMgr.initializeForCache(desc.groupDescriptor(), new StoredCacheData(desc.cacheConfiguration()));
- }
- }
+ cctx.pageStore().initializeForMetastorage();
CheckpointStatus status = readCheckpointStatus();
- cctx.pageStore().initializeForMetastorage();
+ // Binary memory should be recovered at startup.
+ assert !status.needRestoreMemory() : status;
+
+ WALPointer statusEndPtr = CheckpointStatus.NULL_PTR.equals(status.endPtr) ? null : status.endPtr;
+
+ // If binary memory recovery occurs resume from the last walTail in the other case from END checkpoint.
+ WALPointer walPtr = walTail == null ? statusEndPtr : walTail;
+
+ cctx.wal().resumeLogging(walPtr);
+
+ walTail = null;
metaStorage = new MetaStorage(
cctx,
dataRegionMap.get(METASTORE_DATA_REGION_NAME),
- (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME)
+ (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME),
+ false
);
- WALPointer restore = restoreMemory(status, restoreMetastorageOnly, (PageMemoryEx) metaStorage.pageMemory(), true);
+ // Init metastore only after WAL logging resumed. Can't do it earlier because
+ // MetaStorage first initialization also touches WAL, look at #isWalDeltaRecordNeeded.
+ metaStorage.init(this);
- if (restore == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) {
- throw new StorageException("Restore wal pointer = " + restore + ", while status.endPtr = " +
- status.endPtr + ". Can't restore memory - critical part of WAL archive is missing.");
- }
+ notifyMetastorageReadyForReadWrite();
- // First, bring memory to the last consistent checkpoint state if needed.
- // This method should return a pointer to the last valid record in the WAL.
- cctx.wal().resumeLogging(restore);
+ for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext()))
+ lsnr.afterMemoryRestore(this);
+ }
+ catch (IgniteCheckedException e) {
+ if (X.hasCause(e, StorageException.class, IOException.class))
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
- WALPointer ptr = cctx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
+ throw e;
+ }
+ finally {
+ checkpointReadUnlock();
- if (ptr != null) {
- cctx.wal().flush(ptr, true);
+ U.log(log, "Resume logging performed in " + (System.currentTimeMillis() - time) + " ms.");
+ }
+ }
- nodeStart(ptr);
- }
+ /**
+ * @param cacheGrps Cache groups to restore.
+ * @return Last seen WAL pointer during binary memory recovery.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected WALPointer restoreBinaryMemory(Set<Integer> cacheGrps) throws IgniteCheckedException {
+ assert !cctx.kernalContext().clientNode();
- metaStorage.init(this);
+ long time = System.currentTimeMillis();
- notifyMetastorageReadyForReadWrite();
+ checkpointReadLock();
+ try {
for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext()))
- lsnr.afterMemoryRestore(this);
+ lsnr.beforeBinaryMemoryRestore(this);
+
+ cctx.pageStore().initializeForMetastorage();
+
+ CheckpointStatus status = readCheckpointStatus();
+
+ // First, bring memory to the last consistent checkpoint state if needed.
+ // This method should return a pointer to the last valid record in the WAL.
+ WALPointer tailWalPtr = restoreMemory(status,
+ false,
+ (PageMemoryEx)dataRegionMap.get(METASTORE_DATA_REGION_NAME).pageMemory(),
+ cacheGrps);
+
+ if (tailWalPtr == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) {
+ throw new StorageException("Restore wal pointer = " + tailWalPtr + ", while status.endPtr = " +
+ status.endPtr + ". Can't restore memory - critical part of WAL archive is missing.");
+ }
+
+ nodeStart(tailWalPtr);
+
+ return tailWalPtr;
}
catch (IgniteCheckedException e) {
if (X.hasCause(e, StorageException.class, IOException.class))
@@ -860,14 +948,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
}
- /**
- * Creates file with current timestamp and specific "node-started.bin" suffix
- * and writes into memory recovery pointer.
- *
- * @param ptr Memory recovery wal pointer.
- */
- private void nodeStart(WALPointer ptr) throws IgniteCheckedException {
- FileWALPointer p = (FileWALPointer)ptr;
+ /** {@inheritDoc} */
+ @Override public void nodeStart(@Nullable WALPointer ptr) throws IgniteCheckedException {
+ FileWALPointer p = (FileWALPointer)(ptr == null ? CheckpointStatus.NULL_PTR : ptr);
String fileName = U.currentTimeMillis() + NODE_STARTED_FILE_NAME_SUFFIX;
String tmpFileName = fileName + FilePageStoreManager.TMP_SUFFIX;
@@ -1957,11 +2040,33 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
}
+ /** {@inheritDoc} */
+ @Override public void startMemoryRestore(GridKernalContext kctx) throws IgniteCheckedException {
+ if (kctx.clientNode())
+ return;
+
+ // Preform early regions startup before restoring state.
+ initAndStartRegions(kctx.config().getDataStorageConfiguration());
+
+ // Only presistence caches to start.
+ for (DynamicCacheDescriptor desc : cctx.cache().cacheDescriptors().values()) {
+ if (CU.isPersistentCache(desc.cacheConfiguration(), cctx.gridConfig().getDataStorageConfiguration()))
+ storeMgr.initializeForCache(desc.groupDescriptor(), new StoredCacheData(desc.cacheConfiguration()));
+ }
+
+ final WALPointer restoredPtr = restoreBinaryMemory(cctx.cache().cacheGroupDescriptors().keySet());
+
+ walTail = restoredPtr;
+
+ if (restoredPtr != null)
+ U.log(log, "Binary memory state restored at node startup [restoredPtr=" + restoredPtr + ']');
+ }
+
/**
* @param status Checkpoint status.
* @param metastoreOnly If {@code True} restores Metastorage only.
* @param storePageMem Metastore page memory.
- * @param finalizeCp If {@code True}, finalizes checkpoint on recovery.
+ * @param cacheGrps Cache groups to restore.
* @throws IgniteCheckedException If failed.
* @throws StorageException In case I/O error occurred during operations with storage.
*/
@@ -1969,7 +2074,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
CheckpointStatus status,
boolean metastoreOnly,
PageMemoryEx storePageMem,
- boolean finalizeCp
+ Set<Integer> cacheGrps
) throws IgniteCheckedException {
assert !metastoreOnly || storePageMem != null;
@@ -1994,8 +2099,14 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, lastArchivedSegment, log);
- Collection<Integer> ignoreGrps = metastoreOnly ? Collections.emptySet() :
- F.concat(false, initiallyGlobalWalDisabledGrps, initiallyLocalWalDisabledGrps);
+ // Always perform recovery at least meta storage cache.
+ Set<Integer> restoreGrps = new HashSet<>(Collections.singletonList(METASTORAGE_CACHE_ID));
+
+ if (!metastoreOnly && !F.isEmpty(cacheGrps)) {
+ restoreGrps.addAll(cacheGrps.stream()
+ .filter(g -> !initiallyGlobalWalDisabledGrps.contains(g) && !initiallyLocalWalDisabledGrps.contains(g))
+ .collect(Collectors.toSet()));
+ }
int applied = 0;
@@ -2015,10 +2126,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
// several repetitive restarts and the same pages may have changed several times.
int grpId = pageRec.fullPageId().groupId();
- if (metastoreOnly && grpId != METASTORAGE_CACHE_ID)
- continue;
-
- if (!ignoreGrps.contains(grpId)) {
+ if (restoreGrps.contains(grpId)) {
long pageId = pageRec.fullPageId().pageId();
PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
@@ -2051,10 +2159,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
{
int grpId = metaStateRecord.groupId();
- if (metastoreOnly && grpId != METASTORAGE_CACHE_ID)
- continue;
-
- if (ignoreGrps.contains(grpId))
+ if (!restoreGrps.contains(grpId))
continue;
int partId = metaStateRecord.partitionId();
@@ -2075,10 +2180,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
{
int grpId = destroyRecord.groupId();
- if (metastoreOnly && grpId != METASTORAGE_CACHE_ID)
- continue;
-
- if (ignoreGrps.contains(grpId))
+ if (!restoreGrps.contains(grpId))
continue;
PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
@@ -2096,10 +2198,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
int grpId = r.groupId();
- if (metastoreOnly && grpId != METASTORAGE_CACHE_ID)
- continue;
-
- if (!ignoreGrps.contains(grpId)) {
+ if (restoreGrps.contains(grpId)) {
long pageId = r.pageId();
PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
@@ -2129,7 +2228,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
}
- if (!finalizeCp)
+ if (metastoreOnly)
return null;
WALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer();
@@ -2259,6 +2358,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/**
* @param status Last registered checkpoint status.
+ * @param metastoreOnly If {@code True} only records related to metastorage will be processed.
* @throws IgniteCheckedException If failed to apply updates.
* @throws StorageException If IO exception occurred while reading write-ahead log.
*/
@@ -2299,9 +2399,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
for (DataEntry dataEntry : dataRec.writeEntries()) {
int cacheId = dataEntry.cacheId();
- int grpId = cctx.cache().cacheDescriptor(cacheId).groupId();
+ DynamicCacheDescriptor cacheDesc = cctx.cache().cacheDescriptor(cacheId);
- if (!ignoreGrps.contains(grpId)) {
+ // Can empty in case recovery node on blt changed.
+ if (cacheDesc == null)
+ continue;
+
+ if (!ignoreGrps.contains(cacheDesc.groupId())) {
GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
applyUpdate(cacheCtx, dataEntry);
@@ -2799,8 +2903,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/**
* @param cp Checkpoint entry.
* @param type Checkpoint type.
+ * @return Checkpoint file name.
*/
- private static String checkpointFileName(CheckpointEntry cp, CheckpointEntryType type) {
+ public static String checkpointFileName(CheckpointEntry cp, CheckpointEntryType type) {
return checkpointFileName(cp.timestamp(), cp.checkpointId(), type);
}
@@ -4228,7 +4333,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
- * @return {@code True} if need to apply page log to restore tree structure.
+ * @return {@code True} if need perform binary memory recovery. Only records {@link PageDeltaRecord}
+ * and {@link PageSnapshot} needs to be applyed from {@link #cpStartId}.
*/
public boolean needRestoreMemory() {
return !F.eq(cpStartId, cpEndId) && !F.eq(NULL_UUID, cpStartId);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index db6b987..589b495 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -37,6 +37,7 @@ import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
import org.apache.ignite.internal.mem.DirectMemoryRegion;
import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
@@ -45,7 +46,6 @@ import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
@@ -100,6 +100,9 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
private volatile boolean dataRegionsInitialized;
/** */
+ private volatile boolean dataRegionsStarted;
+
+ /** */
protected Map<String, DataRegionMetrics> memMetricsMap;
/** */
@@ -238,6 +241,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
initDataRegions0(memCfg);
dataRegionsInitialized = true;
+
+ U.log(log, "Configured data regions initialized successfully [total=" + dataRegionMap.size() + ']');
}
/**
@@ -648,14 +653,19 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
}
/**
- * @param cachesToStart Started caches.
- * @param restoreMetastorageOnly Apply updates only for metastorage.
- * @throws IgniteCheckedException If failed.
+ * @throws IgniteCheckedException If fails.
*/
- public void readCheckpointAndRestoreMemory(
- List<DynamicCacheDescriptor> cachesToStart,
- boolean restoreMetastorageOnly
- ) throws IgniteCheckedException {
+ public void onDoneRestoreBinaryMemory() throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /**
+ * Creates file with current timestamp and specific "node-started.bin" suffix
+ * and writes into memory recovery pointer.
+ *
+ * @param ptr Memory recovery wal pointer.
+ */
+ public void nodeStart(@Nullable WALPointer ptr) throws IgniteCheckedException {
// No-op.
}
@@ -781,7 +791,15 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
}
/**
- * No-op for non-persistent storage.
+ * Method will perform cleanup cache page memory and each cache partition store.
+ */
+ public void cleanupRestoredCaches() {
+ // No-op.
+ }
+
+ /**
+ * Clean checkpoint directory {@link GridCacheDatabaseSharedManager#cpDir}. The operation
+ * is necessary when local node joined to baseline topology with different consistentId.
*/
public void cleanupCheckpointDirectory() throws IgniteCheckedException {
// No-op.
@@ -836,6 +854,16 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
}
/**
+ * Perform memory restore before {@link GridDiscoveryManager} start.
+ *
+ * @param kctx Current kernal context.
+ * @throws IgniteCheckedException If fails.
+ */
+ public void startMemoryRestore(GridKernalContext kctx) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /**
* Called when all partitions have been fully restored and pre-created on node start.
*
* @throws IgniteCheckedException If failed.
@@ -1138,23 +1166,46 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
/** {@inheritDoc} */
@Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
- if (cctx.kernalContext().clientNode() && cctx.kernalContext().config().getDataStorageConfiguration() == null)
+ if (kctx.clientNode() && kctx.config().getDataStorageConfiguration() == null)
return;
- DataStorageConfiguration memCfg = cctx.kernalContext().config().getDataStorageConfiguration();
+ initAndStartRegions(kctx.config().getDataStorageConfiguration());
- assert memCfg != null;
+ for (DatabaseLifecycleListener lsnr : getDatabaseListeners(kctx))
+ lsnr.afterInitialise(this);
+ }
- initDataRegions(memCfg);
+ /**
+ * @param cfg Current data storage configuration.
+ * @throws IgniteCheckedException If fails.
+ */
+ protected void initAndStartRegions(DataStorageConfiguration cfg) throws IgniteCheckedException {
+ assert cfg != null;
+
+ initDataRegions(cfg);
+
+ startDataRegions(cfg);
+ }
+
+ /**
+ * @param cfg Regions configuration.
+ * @throws IgniteCheckedException If fails.
+ */
+ private void startDataRegions(DataStorageConfiguration cfg) throws IgniteCheckedException {
+ if (dataRegionsStarted)
+ return;
+
+ assert cfg != null;
registerMetricsMBeans();
startMemoryPolicies();
- initPageMemoryDataStructures(memCfg);
+ initPageMemoryDataStructures(cfg);
- for (DatabaseLifecycleListener lsnr : getDatabaseListeners(kctx))
- lsnr.afterInitialise(this);
+ dataRegionsStarted = true;
+
+ U.log(log, "Configured data regions started successfully [total=" + dataRegionMap.size() + ']');
}
/** {@inheritDoc} */
@@ -1163,7 +1214,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
}
/**
- * @param shutdown Shutdown.
+ * @param shutdown {@code True} to force memory regions shutdown.
*/
private void onDeActivate(boolean shutdown) {
for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext()))
@@ -1189,6 +1240,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
}
dataRegionsInitialized = false;
+
+ dataRegionsStarted = false;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index c6cd9e5..e05cb71 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -32,11 +32,14 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
@@ -227,14 +230,28 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
}
/** {@inheritDoc} */
+ @Override public void cleanupPageStoreIfMatch(Predicate<Integer> cacheGrpPred, boolean cleanFiles) {
+ Map<Integer, CacheStoreHolder> filteredStores = idxCacheStores.entrySet().stream()
+ .filter(e -> cacheGrpPred.test(e.getKey()))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+ IgniteCheckedException ex = shutdown(filteredStores.values(), cleanFiles);
+
+ if (ex != null)
+ U.error(log, "Failed to gracefully stop page store managers", ex);
+
+ idxCacheStores.entrySet().removeIf(e -> cacheGrpPred.test(e.getKey()));
+
+ U.log(log, "Cleanup cache stores [total=" + filteredStores.keySet().size() +
+ ", left=" + idxCacheStores.size() + ", cleanFiles=" + cleanFiles + ']');
+ }
+
+ /** {@inheritDoc} */
@Override public void stop0(boolean cancel) {
if (log.isDebugEnabled())
log.debug("Stopping page store manager.");
- IgniteCheckedException ex = shutdown(false);
-
- if (ex != null)
- U.error(log, "Failed to gracefully stop page store manager", ex);
+ cleanupPageStoreIfMatch(p -> true, false);
}
/** {@inheritDoc} */
@@ -253,8 +270,6 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
" topVer=" + cctx.discovery().topologyVersionEx() + " ]");
stop0(true);
-
- idxCacheStores.clear();
}
/** {@inheritDoc} */
@@ -287,6 +302,8 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
/** {@inheritDoc} */
@Override public void initialize(int cacheId, int partitions, String workingDir, AllocatedPageTracker tracker)
throws IgniteCheckedException {
+ assert storeWorkDir != null;
+
if (!idxCacheStores.containsKey(cacheId)) {
CacheStoreHolder holder = initDir(
new File(storeWorkDir, workingDir),
@@ -304,6 +321,8 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
/** {@inheritDoc} */
@Override public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cacheData) throws IgniteCheckedException {
+ assert storeWorkDir != null;
+
int grpId = grpDesc.groupId();
if (!idxCacheStores.containsKey(grpId)) {
@@ -317,6 +336,8 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
/** {@inheritDoc} */
@Override public void initializeForMetastorage() throws IgniteCheckedException {
+ assert storeWorkDir != null;
+
int grpId = MetaStorage.METASTORAGE_CACHE_ID;
if (!idxCacheStores.containsKey(grpId)) {
@@ -898,10 +919,10 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
/**
* @param cleanFiles {@code True} if the stores should delete it's files upon close.
*/
- private IgniteCheckedException shutdown(boolean cleanFiles) {
+ private IgniteCheckedException shutdown(Collection<CacheStoreHolder> holders, boolean cleanFiles) {
IgniteCheckedException ex = null;
- for (CacheStoreHolder holder : idxCacheStores.values())
+ for (CacheStoreHolder holder : holders)
ex = shutdown(holder, cleanFiles, ex);
return ex;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index 3981d4d..4a243aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -136,11 +136,6 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
}
/** */
- public MetaStorage(GridCacheSharedContext cctx, DataRegion memPlc, DataRegionMetricsImpl memMetrics) {
- this(cctx, memPlc, memMetrics, false);
- }
-
- /** */
public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedException {
getOrAllocateMetas();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 0b03128..61b1f65 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -33,10 +33,8 @@ import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.ClosedByInterruptException;
-import java.nio.file.DirectoryStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
-import java.nio.file.Path;
import java.sql.Time;
import java.util.ArrayList;
import java.util.Arrays;
@@ -688,6 +686,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
@Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
assert currHnd == null;
assert lastPtr == null || lastPtr instanceof FileWALPointer;
+ assert (isArchiverEnabled() && archiver != null) || (!isArchiverEnabled() && archiver == null) :
+ "Trying to restore FileWriteHandle on deactivated write ahead log manager";
FileWALPointer filePtr = (FileWALPointer)lastPtr;
@@ -1485,29 +1485,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
checkFiles(0, false, null, null);
}
- /** {@inheritDoc} */
- @Override public void cleanupWalDirectories() throws IgniteCheckedException {
- try {
- try (DirectoryStream<Path> files = Files.newDirectoryStream(walWorkDir.toPath())) {
- for (Path path : files)
- Files.delete(path);
- }
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to cleanup wal work directory: " + walWorkDir, e);
- }
-
- try {
- try (DirectoryStream<Path> files = Files.newDirectoryStream(walArchiveDir.toPath())) {
- for (Path path : files)
- Files.delete(path);
- }
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to cleanup wal archive directory: " + walArchiveDir, e);
- }
- }
-
/**
* Clears whole the file, fills with zeros for Default mode.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index d1e0ebc..def9bf2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -28,10 +28,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
-import java.nio.file.DirectoryStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
-import java.nio.file.Path;
import java.sql.Time;
import java.util.ArrayList;
import java.util.Arrays;
@@ -539,6 +537,8 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
@Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
assert currentHnd == null;
assert lastPtr == null || lastPtr instanceof FileWALPointer;
+ assert (isArchiverEnabled() && archiver != null) || (!isArchiverEnabled() && archiver == null) :
+ "Trying to restore FileWriteHandle on deactivated write ahead log manager";
FileWALPointer filePtr = (FileWALPointer)lastPtr;
@@ -1022,29 +1022,6 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
return cctx.walState().isDisabled(grpId);
}
- /** {@inheritDoc} */
- @Override public void cleanupWalDirectories() throws IgniteCheckedException {
- try {
- try (DirectoryStream<Path> files = Files.newDirectoryStream(walWorkDir.toPath())) {
- for (Path path : files)
- Files.delete(path);
- }
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to cleanup wal work directory: " + walWorkDir, e);
- }
-
- try {
- try (DirectoryStream<Path> files = Files.newDirectoryStream(walArchiveDir.toPath())) {
- for (Path path : files)
- Files.delete(path);
- }
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to cleanup wal archive directory: " + walArchiveDir, e);
- }
- }
-
/**
* Lists files in archive directory and returns the index of last archived file.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
index 23bc3fb..2f4e4a3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
@@ -25,7 +25,6 @@ import java.nio.file.OpenOption;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheRebalanceMode;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -176,15 +175,18 @@ public class IgnitePdsDiskErrorsRecoveringTest extends GridCommonAbstractTest {
boolean activationFailed = false;
try {
grid = startGrid(0);
- grid.cluster().active(true);
}
- catch (IgniteException e) {
- log.warning("Activation test exception", e);
+ catch (IgniteCheckedException e) {
+ boolean interrupted = Thread.interrupted();
+
+ if (interrupted)
+ log.warning("Ignore interrupted excpetion [" +
+ "thread=" + Thread.currentThread().getName() + ']', e);
activationFailed = true;
}
- Assert.assertTrue("Activation must be failed", activationFailed);
+ Assert.assertTrue("Ignite instance startup must be failed", activationFailed);
// Grid should be automatically stopped after checkpoint fail.
awaitStop(grid);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
index dcc2280..3374860 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
@@ -21,6 +21,7 @@ import java.io.FilenameFilter;
import java.io.RandomAccessFile;
import java.util.Arrays;
import java.util.Comparator;
+import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -221,6 +222,20 @@ public class WalCompactionTest extends GridCommonAbstractTest {
}
assertFalse(fail);
+
+ // Check compation successfully reset on blt changed.
+ stopAllGrids();
+
+ Ignite ignite = startGrids(2);
+
+ ignite.cluster().active(true);
+
+ resetBaselineTopology();
+
+ // This node will join to different blt.
+ startGrid(2);
+
+ awaitPartitionMapExchange();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
index 39c7dc9..44071ea 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.GridKernalContext;
@@ -235,6 +236,11 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
}
/** {@inheritDoc} */
+ @Override public void cleanupPageStoreIfMatch(Predicate<Integer> cacheGrpPred, boolean cleanFiles) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Override public boolean checkAndInitCacheWorkDir(CacheConfiguration cacheCfg) throws IgniteCheckedException {
return false;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index 0beeed7..ea3ed2f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -108,11 +108,6 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
}
/** {@inheritDoc} */
- @Override public void cleanupWalDirectories() throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
@Override public void start(GridCacheSharedContext cctx) throws IgniteCheckedException {
// No-op.
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7449f6c/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
index d80b5e2..388fb19 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
@@ -18,8 +18,12 @@
package org.apache.ignite.internal.processors.cache.persistence.db.wal;
import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -38,6 +42,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteCompute;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheMode;
@@ -53,9 +58,11 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.DiscoverySpiTestListener;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageUtils;
import org.apache.ignite.internal.pagemem.wal.WALIterator;
@@ -67,14 +74,21 @@ import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntryType;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.PA;
import org.apache.ignite.internal.util.typedef.PAX;
@@ -94,6 +108,8 @@ import org.apache.ignite.transactions.TransactionConcurrency;
import org.apache.ignite.transactions.TransactionIsolation;
import org.junit.Assert;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_DATA_FILENAME;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
/**
@@ -116,6 +132,9 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
private static final String RENAMED_CACHE_NAME = "partitioned0";
/** */
+ private static final String CACHE_TO_DESTROY_NAME = "destroyCache";
+
+ /** */
private static final String LOC_CACHE_NAME = "local";
/** */
@@ -466,6 +485,110 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
}
/**
+ * Check binary recover completes successfully when node stopped at the middle of checkpoint.
+ * Destroy cache_data.bin file for particular cache to emulate missing {@link DynamicCacheDescriptor}
+ * file (binary recovery should complete successfully in this case).
+ *
+ * @throws Exception if failed.
+ */
+ public void testBinaryRecoverBeforePMEWhenMiddleCheckpoint() throws Exception {
+ startGrids(3);
+
+ IgniteEx ig2 = grid(2);
+
+ ig2.cluster().active(true);
+
+ IgniteCache<Object, Object> cache = ig2.cache(CACHE_NAME);
+
+ for (int i = 1; i <= 4_000; i++)
+ cache.put(i, new BigObject(i));
+
+ BigObject objToCheck;
+
+ ig2.getOrCreateCache(CACHE_TO_DESTROY_NAME).put(1, objToCheck = new BigObject(1));
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ig2
+ .context().cache().context().database();
+
+ IgniteInternalFuture<?> cpFinishFut = dbMgr.forceCheckpoint("force checkpoint").finishFuture();
+
+ // Delete checkpoint END file to emulate node stopped at the middle of checkpoint.
+ cpFinishFut.listen(new IgniteInClosureX<IgniteInternalFuture>() {
+ @Override public void applyx(IgniteInternalFuture fut0) throws IgniteCheckedException {
+ try {
+ CheckpointEntry cpEntry = dbMgr.checkpointHistory().lastCheckpoint();
+
+ String cpEndFileName = GridCacheDatabaseSharedManager.checkpointFileName(cpEntry,
+ CheckpointEntryType.END);
+
+ Files.delete(Paths.get(dbMgr.checkpointDirectory().getAbsolutePath(), cpEndFileName));
+
+ log.info("Checkpoint marker removed [cpEndFileName=" + cpEndFileName + ']');
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+ });
+
+ // Resolve cache directory. Emulating cache destroy in the middle of checkpoint.
+ IgniteInternalCache<Object, Object> destoryCache = ig2.cachex(CACHE_TO_DESTROY_NAME);
+
+ FilePageStoreManager pageStoreMgr = (FilePageStoreManager)destoryCache.context().shared().pageStore();
+
+ File destroyCacheWorkDir = pageStoreMgr.cacheWorkDir(destoryCache.configuration());
+
+ // Stop the whole cluster
+ stopAllGrids();
+
+ // Delete cache_data.bin file for this cache. Binary recovery should complete successfully after it.
+ final File[] files = destroyCacheWorkDir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(final File dir, final String name) {
+ return name.endsWith(CACHE_DATA_FILENAME);
+ }
+ });
+
+ assertTrue(files.length > 0);
+
+ for (final File file : files)
+ assertTrue("Can't remove " + file.getAbsolutePath(), file.delete());
+
+ startGrids(2);
+
+ // Preprare Ignite instance configuration with additional Discovery checks.
+ final String ig2Name = getTestIgniteInstanceName(2);
+
+ final IgniteConfiguration onJoinCfg = optimize(getConfiguration(ig2Name));
+
+ // Check restore beeing called before PME and joining node to cluster.
+ ((IgniteDiscoverySpi)onJoinCfg.getDiscoverySpi())
+ .setInternalListener(new DiscoverySpiTestListener() {
+ @Override public void beforeJoin(ClusterNode locNode, IgniteLogger log) {
+ String nodeName = locNode.attribute(ATTR_IGNITE_INSTANCE_NAME);
+
+ GridCacheSharedContext sharedCtx = ((IgniteEx)ignite(getTestIgniteInstanceIndex(nodeName)))
+ .context()
+ .cache()
+ .context();
+
+ if (nodeName.equals(ig2Name)) {
+ // Checkpoint history initialized on node start.
+ assertFalse(((GridCacheDatabaseSharedManager)sharedCtx.database())
+ .checkpointHistory().checkpoints().isEmpty());
+ }
+
+ super.beforeJoin(locNode, log);
+ }
+ });
+
+ Ignite restoredIg2 = startGrid(ig2Name, onJoinCfg);
+
+ awaitPartitionMapExchange();
+
+ assertEquals(restoredIg2.cache(CACHE_TO_DESTROY_NAME).get(1), objToCheck);
+ }
+
+ /**
* @throws Exception if failed.
*/
public void testWalRolloverMultithreadedDefault() throws Exception {