You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ti...@apache.org on 2023/03/09 20:00:35 UTC
[ignite] branch master updated (04304d4e9e5 -> a8e16899514)
This is an automated email from the ASF dual-hosted git repository.
timoninmaxim pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
omit 04304d4e9e5 IGNITE-17177 Added incremental snapshots (#10539)
new a8e16899514 IGNITE-17177 Added incremental snapshots (#10539)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (04304d4e9e5)
\
N -- N -- N refs/heads/master (a8e16899514)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
[ignite] 01/01: IGNITE-17177 Added incremental snapshots (#10539)
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
timoninmaxim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit a8e16899514348ed7b9404e7d122ed430f8e1acd
Author: Maksim Timonin <ti...@apache.org>
AuthorDate: Thu Mar 9 22:52:10 2023 +0300
IGNITE-17177 Added incremental snapshots (#10539)
Co-authored-by: Nikolay Izhikov <ni...@apache.org>
---
.../ignite/common/ComputeTaskPermissionsTest.java | 8 +-
.../ignite/codegen/MessageCodeGenerator.java | 2 +
.../commandline/indexreader/IgniteIndexReader.java | 13 +-
.../snapshot/SnapshotCreateCommand.java | 12 +-
.../snapshot/SnapshotCreateCommandOption.java | 6 +-
.../snapshot/SnapshotRestoreCommand.java | 21 +-
.../snapshot/SnapshotRestoreCommandOption.java | 4 +
.../snapshot/SnapshotStatusCommand.java | 180 +++-
.../commandline/snapshot/SnapshotSubcommand.java | 5 +-
.../org/apache/ignite/util/CdcCommandTest.java | 2 +-
.../util/GridCommandHandlerAbstractTest.java | 13 +
.../apache/ignite/util/GridCommandHandlerTest.java | 144 ++-
.../java/org/apache/ignite/IgniteSnapshot.java | 24 +-
.../org/apache/ignite/internal/cdc/CdcMain.java | 18 +-
.../communication/GridIoMessageFactory.java | 9 +-
.../systemview/walker/SnapshotViewWalker.java | 8 +-
.../pagemem/wal/IgniteWriteAheadLogManager.java | 23 +
.../record/IncrementalSnapshotFinishRecord.java | 103 ++
.../wal/record/IncrementalSnapshotStartRecord.java | 51 +
.../internal/pagemem/wal/record/WALRecord.java | 8 +-
.../processors/cache/GridCacheIoManager.java | 14 +
.../processors/cache/GridCacheProcessor.java | 6 +-
.../processors/cache/GridLocalConfigManager.java | 37 +-
.../distributed/dht/GridDhtTxFinishFuture.java | 2 +-
.../distributed/dht/GridDhtTxPrepareFuture.java | 4 +-
...dNearOptimisticSerializableTxPrepareFuture.java | 2 +-
.../near/GridNearOptimisticTxPrepareFuture.java | 2 +-
.../near/GridNearPessimisticTxPrepareFuture.java | 2 +-
.../distributed/near/GridNearTxFinishFuture.java | 2 +-
.../cache/distributed/near/GridNearTxLocal.java | 3 +
.../cache/persistence/CacheStripedExecutor.java | 146 +++
.../GridCacheDatabaseSharedManager.java | 187 +---
.../persistence/file/FilePageStoreManager.java | 8 +
.../snapshot/AbstractSnapshotFutureTask.java | 7 +
.../snapshot/IgniteSnapshotManager.java | 942 ++++++++++++++---
.../snapshot/IncrementalSnapshotAwareMessage.java | 214 ++++
.../snapshot/IncrementalSnapshotFutureTask.java | 251 +++++
.../snapshot/IncrementalSnapshotMarkWalFuture.java | 202 ++++
.../snapshot/IncrementalSnapshotMetadata.java | 126 +++
.../persistence/snapshot/SnapshotFutureTask.java | 19 +-
.../snapshot/SnapshotFutureTaskResult.java | 52 +
.../snapshot/SnapshotHandlerRestoreTask.java | 3 +-
.../persistence/snapshot/SnapshotMXBeanImpl.java | 30 +-
.../persistence/snapshot/SnapshotMetadata.java | 20 +-
.../snapshot/SnapshotMetadataCollectorTask.java | 113 ---
.../snapshot/SnapshotMetadataVerificationTask.java | 230 +++++
...va => SnapshotMetadataVerificationTaskArg.java} | 23 +-
.../SnapshotMetadataVerificationTaskResult.java | 75 ++
.../snapshot/SnapshotOperationRequest.java | 24 +-
.../snapshot/SnapshotPartitionsVerifyTask.java | 3 +-
.../snapshot/SnapshotRestoreProcess.java | 517 +++++++++-
.../cache/persistence/wal/FileDescriptor.java | 18 +-
.../persistence/wal/FileWriteAheadLogManager.java | 103 +-
.../wal/aware/SegmentArchivedStorage.java | 4 +-
.../cache/persistence/wal/aware/SegmentAware.java | 14 +-
.../wal/aware/SegmentCompressStorage.java | 21 +
.../wal/aware/SegmentReservationStorage.java | 4 +-
.../wal/serializer/RecordDataV1Serializer.java | 75 ++
.../cache/transactions/IgniteInternalTx.java | 10 +
.../cache/transactions/IgniteTxAdapter.java | 23 +
.../cache/transactions/IgniteTxHandler.java | 115 ++-
.../cache/transactions/IgniteTxManager.java | 52 +
.../apache/ignite/internal/util/IgniteUtils.java | 47 +
.../util/distributed/DistributedProcess.java | 26 +-
.../internal/util/distributed/InitMessage.java | 11 +-
.../apache/ignite/internal/util/lang/GridFunc.java | 4 +-
.../visor/cdc/VisorCdcDeleteLostSegmentsTask.java | 5 +-
.../visor/snapshot/VisorSnapshotCreateTask.java | 7 +-
.../visor/snapshot/VisorSnapshotCreateTaskArg.java | 14 +-
.../visor/snapshot/VisorSnapshotRestoreTask.java | 2 +-
.../snapshot/VisorSnapshotRestoreTaskArg.java | 15 +-
.../visor/snapshot/VisorSnapshotStatusTask.java | 59 +-
.../org/apache/ignite/mxbean/SnapshotMXBean.java | 37 +
.../ignite/spi/systemview/view/SnapshotView.java | 83 +-
.../main/resources/META-INF/classnames.properties | 6 +-
.../ignite/internal/metric/SystemViewSelfTest.java | 44 +-
.../cache/persistence/pagemem/NoOpWALManager.java | 16 +
.../snapshot/AbstractSnapshotSelfTest.java | 63 +-
.../snapshot/IgniteClusterSnapshotCheckTest.java | 2 +-
.../snapshot/IgniteClusterSnapshotHandlerTest.java | 2 +-
.../IgniteClusterSnapshotRestoreSelfTest.java | 4 +-
.../snapshot/IgniteClusterSnapshotSelfTest.java | 51 +-
.../IgniteClusterSnapshotStreamerTest.java | 4 +-
.../IgniteClusterSnapshotWalRecordTest.java | 91 +-
.../snapshot/IgniteSnapshotMXBeanTest.java | 103 +-
.../snapshot/IncrementalSnapshotMetricTest.java | 210 ++++
.../snapshot/IncrementalSnapshotRebalanceTest.java | 173 ++++
.../snapshot/IncrementalSnapshotTest.java | 400 ++++++++
.../AbstractIncrementalSnapshotBlockingTest.java | 372 +++++++
...actIncrementalSnapshotMessagesBlockingTest.java | 172 ++++
.../AbstractIncrementalSnapshotTest.java | 330 ++++++
...AbstractIncrementalSnapshotWalBlockingTest.java | 77 ++
.../ConcurrentTxsIncrementalSnapshotTest.java | 267 +++++
.../IncrementalSnapshotCheckBeforeRestoreTest.java | 371 +++++++
.../IncrementalSnapshotJoiningClientTest.java | 274 +++++
...mentalSnapshotNoBackupMessagesBlockingTest.java | 82 ++
...IncrementalSnapshotNoBackupWALBlockingTest.java | 91 ++
.../IncrementalSnapshotNodeFailureTest.java | 170 ++++
.../IncrementalSnapshotRestoreTest.java | 1062 ++++++++++++++++++++
...alSnapshotSingleBackupMessagesBlockingTest.java | 81 ++
...ementalSnapshotSingleBackupWALBlockingTest.java | 41 +
...entalSnapshotTwoBackupMessagesBlockingTest.java | 85 ++
...ncrementalSnapshotTwoBackupWALBlockingTest.java | 31 +
.../IncrementalSnapshotTxRecoveryTest.java | 184 ++++
.../IncrementalSnapshotWarnAtomicCachesTest.java | 253 +++++
.../snapshot/incremental/TransactionTestCase.java | 143 +++
.../util/DistributedProcessClientAwaitTest.java | 221 ++++
.../internal/util/lang/GridFuncSelfTest.java | 54 +
.../testframework/wal/record/RecordUtils.java | 19 +
.../ignite/testsuites/IgniteSnapshotTestSuite.java | 1 +
.../ignite/testsuites/IgniteUtilSelfTestSuite.java | 4 +
.../testsuites/IncrementalSnapshotsTestSuite.java | 62 ++
...ridCommandHandlerClusterByClassTest_help.output | 22 +-
...andHandlerClusterByClassWithSSLTest_help.output | 22 +-
...IncrementalSnapshotRestoreWithIndexingTest.java | 236 +++++
.../IgniteSnapshotWithIndexingTestSuite.java | 2 +
.../config/benchmark-inc-snapshot.properties | 64 ++
.../config/ignite-config-incremental-snapshots.xml | 59 ++
.../cache/IgniteIncrementalSnapshotsBenchmark.java | 101 ++
119 files changed, 10119 insertions(+), 672 deletions(-)
diff --git a/modules/clients/src/test/java/org/apache/ignite/common/ComputeTaskPermissionsTest.java b/modules/clients/src/test/java/org/apache/ignite/common/ComputeTaskPermissionsTest.java
index f1cb6989438..d10b78862b0 100644
--- a/modules/clients/src/test/java/org/apache/ignite/common/ComputeTaskPermissionsTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/common/ComputeTaskPermissionsTest.java
@@ -52,7 +52,7 @@ import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.client.GridClient;
import org.apache.ignite.internal.client.GridClientConfiguration;
import org.apache.ignite.internal.client.GridClientFactory;
-import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadataCollectorTask;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadataVerificationTask;
import org.apache.ignite.internal.processors.security.AbstractSecurityTest;
import org.apache.ignite.internal.processors.security.OperationSecurityContext;
import org.apache.ignite.internal.processors.security.PublicAccessJob;
@@ -113,7 +113,7 @@ public class ComputeTaskPermissionsTest extends AbstractSecurityTest {
private static final IgniteClosure SYSTEM_CLOSURE = new ToStringClosure<>();
/** */
- private static final ComputeTask SYSTEM_TASK = new SnapshotMetadataCollectorTask();
+ private static final ComputeTask SYSTEM_TASK = new SnapshotMetadataVerificationTask();
/** */
private static final AtomicInteger EXECUTED_TASK_CNTR = new AtomicInteger();
@@ -468,11 +468,11 @@ public class ComputeTaskPermissionsTest extends AbstractSecurityTest {
taskUnblockedLatch.countDown();
}
}
-
+
/** */
private IgniteCompute compute(int initiator, int executor) {
IgniteEx ignite = grid(initiator);
-
+
return ignite.compute(ignite.cluster().forNodeId(grid(executor).localNode().id()));
}
diff --git a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
index be4bc9aa2eb..2261acc9989 100644
--- a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
+++ b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
@@ -204,6 +204,8 @@ public class MessageCodeGenerator {
// gen.generateAndWrite(GridDhtTxFinishRequest.class);
// gen.generateAndWrite(GridDhtTxFinishResponse.class);
//
+// gen.generateAndWrite(IncrementalSnapshotAwareMessage.class);
+
// gen.generateAndWrite(GridCacheTxRecoveryRequest.class);
// gen.generateAndWrite(GridCacheTxRecoveryResponse.class);
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java
index a07aa254454..1e83514c678 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java
@@ -18,10 +18,7 @@
package org.apache.ignite.internal.commandline.indexreader;
import java.io.File;
-import java.io.IOException;
-import java.io.ObjectInputStream;
import java.nio.ByteBuffer;
-import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -61,6 +58,7 @@ import org.apache.ignite.internal.commandline.indexreader.ScanContext.PagesStati
import org.apache.ignite.internal.commandline.systemview.SystemViewCommand;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.GridLocalConfigManager;
import org.apache.ignite.internal.processors.cache.StoredCacheData;
import org.apache.ignite.internal.processors.cache.persistence.IndexStorageImpl;
import org.apache.ignite.internal.processors.cache.persistence.StorageException;
@@ -116,7 +114,6 @@ import static org.apache.ignite.internal.pagemem.PageIdUtils.itemId;
import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId;
import static org.apache.ignite.internal.pagemem.PageIdUtils.pageIndex;
import static org.apache.ignite.internal.pagemem.PageIdUtils.partId;
-import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_DATA_FILENAME;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_TEMPLATE;
import static org.apache.ignite.internal.util.GridUnsafe.allocateBuffer;
@@ -283,13 +280,13 @@ public class IgniteIndexReader implements AutoCloseable {
for (int i = 0; i < partCnt; i++)
partStores[i] = filePageStore(i, FLAG_DATA, storeFactory);
- Arrays.stream(root.listFiles(f -> f.getName().endsWith(CACHE_DATA_FILENAME))).forEach(f -> {
- try (ObjectInputStream stream = new ObjectInputStream(Files.newInputStream(f.toPath()))) {
- StoredCacheData data = (StoredCacheData)stream.readObject();
+ Arrays.stream(FilePageStoreManager.cacheDataFiles(root)).forEach(f -> {
+ try {
+ StoredCacheData data = GridLocalConfigManager.readCacheData(f, null, null);
storedCacheData.put(CU.cacheId(data.config().getName()), data);
}
- catch (ClassNotFoundException | IOException e) {
+ catch (IgniteCheckedException e) {
log.warning("Can't read stored cache data. Inline for this cache will not be analyzed [f=" + f.getName() + ']', e);
}
});
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java
index d01d1e2fc88..f0773be9306 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java
@@ -29,6 +29,7 @@ import org.apache.ignite.internal.visor.snapshot.VisorSnapshotCreateTaskArg;
import static org.apache.ignite.internal.commandline.CommandList.SNAPSHOT;
import static org.apache.ignite.internal.commandline.CommandLogger.optional;
import static org.apache.ignite.internal.commandline.snapshot.SnapshotCreateCommandOption.DESTINATION;
+import static org.apache.ignite.internal.commandline.snapshot.SnapshotCreateCommandOption.INCREMENTAL;
import static org.apache.ignite.internal.commandline.snapshot.SnapshotCreateCommandOption.SYNC;
/**
@@ -45,6 +46,7 @@ public class SnapshotCreateCommand extends SnapshotSubcommand {
String snpName = argIter.nextArg("Expected snapshot name.");
String snpPath = null;
boolean sync = false;
+ boolean incremental = false;
while (argIter.hasNextSubArg()) {
String arg = argIter.nextArg(null);
@@ -72,10 +74,15 @@ public class SnapshotCreateCommand extends SnapshotSubcommand {
sync = true;
}
+ else if (option == INCREMENTAL) {
+ if (incremental)
+ throw new IllegalArgumentException(INCREMENTAL.argName() + " arg specified twice.");
+ incremental = true;
+ }
}
- cmdArg = new VisorSnapshotCreateTaskArg(snpName, snpPath, sync);
+ cmdArg = new VisorSnapshotCreateTaskArg(snpName, snpPath, sync, incremental);
}
/** {@inheritDoc} */
@@ -84,8 +91,9 @@ public class SnapshotCreateCommand extends SnapshotSubcommand {
params.put(DESTINATION.argName() + " " + DESTINATION.arg(), DESTINATION.description());
params.put(SYNC.argName(), SYNC.description());
+ params.put(INCREMENTAL.argName(), INCREMENTAL.description());
usage(log, "Create cluster snapshot:", SNAPSHOT, params, name(), SNAPSHOT_NAME_ARG,
- optional(DESTINATION.argName(), DESTINATION.arg()), optional(SYNC.argName()));
+ optional(DESTINATION.argName(), DESTINATION.arg()), optional(SYNC.argName()), optional(INCREMENTAL.argName()));
}
}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java
index a6d216d1781..23761a0e5a2 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java
@@ -30,7 +30,11 @@ public enum SnapshotCreateCommandOption implements CommandArg {
/** Snapshot directory path. */
DESTINATION("--dest", "path", "Path to the directory where the snapshot will be saved. If not specified, " +
- "the default configured snapshot directory will be used.");
+ "the default configured snapshot directory will be used."),
+
+ /** Incremental snapshot flag. */
+ INCREMENTAL("--incremental", null, "Create an incremental snapshot for previously created full snapshot. " +
+ "Full snapshot must be accessible via --dest and snapshot_name.");
/** Name. */
private final String name;
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java
index db07092f3e1..bd14ef67619 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java
@@ -32,6 +32,7 @@ import org.apache.ignite.internal.visor.snapshot.VisorSnapshotRestoreTaskArg;
import static org.apache.ignite.internal.commandline.CommandList.SNAPSHOT;
import static org.apache.ignite.internal.commandline.CommandLogger.optional;
import static org.apache.ignite.internal.commandline.snapshot.SnapshotRestoreCommandOption.GROUPS;
+import static org.apache.ignite.internal.commandline.snapshot.SnapshotRestoreCommandOption.INCREMENT;
import static org.apache.ignite.internal.commandline.snapshot.SnapshotRestoreCommandOption.SOURCE;
import static org.apache.ignite.internal.commandline.snapshot.SnapshotRestoreCommandOption.SYNC;
import static org.apache.ignite.internal.commandline.snapshot.SnapshotSubcommands.RESTORE;
@@ -86,6 +87,7 @@ public class SnapshotRestoreCommand extends SnapshotSubcommand {
String snpName = argIter.nextArg("Expected snapshot name.");
VisorSnapshotRestoreTaskAction restoreAction = parseAction(argIter.peekNextArg());
String snpPath = null;
+ Integer incIdx = null;
Set<String> grpNames = null;
boolean sync = false;
@@ -128,6 +130,17 @@ public class SnapshotRestoreCommand extends SnapshotSubcommand {
snpPath = argIter.nextArg(errMsg);
}
+ else if (option == INCREMENT) {
+ if (incIdx != null)
+ throw new IllegalArgumentException(INCREMENT.argName() + " arg specified twice.");
+
+ String errMsg = "incremental snapshot index";
+
+ if (CommandArgIterator.isCommandOrOption(argIter.peekNextArg()))
+ throw new IllegalArgumentException("Expected " + errMsg + '.');
+
+ incIdx = argIter.nextIntArg(errMsg);
+ }
else if (option == SYNC) {
if (sync)
throw new IllegalArgumentException(SYNC.argName() + " arg specified twice.");
@@ -136,7 +149,7 @@ public class SnapshotRestoreCommand extends SnapshotSubcommand {
}
}
- cmdArg = new VisorSnapshotRestoreTaskArg(snpName, snpPath, sync, restoreAction, grpNames);
+ cmdArg = new VisorSnapshotRestoreTaskArg(snpName, snpPath, incIdx, sync, restoreAction, grpNames);
}
/** {@inheritDoc} */
@@ -146,10 +159,14 @@ public class SnapshotRestoreCommand extends SnapshotSubcommand {
startParams.put(GROUPS.argName() + " " + GROUPS.arg(), GROUPS.description());
startParams.put(SOURCE.argName() + " " + SOURCE.arg(), SOURCE.description());
+ startParams.put(INCREMENT.argName() + " " + INCREMENT.arg(), INCREMENT.description());
startParams.put(SYNC.argName(), SYNC.description());
usage(log, "Restore snapshot:", SNAPSHOT, startParams, RESTORE.toString(), SNAPSHOT_NAME_ARG,
- optional(GROUPS.argName(), GROUPS.arg()), optional(SOURCE.argName(), SOURCE.arg()), optional(SYNC.argName()));
+ optional(INCREMENT.argName(), INCREMENT.arg()),
+ optional(GROUPS.argName(), GROUPS.arg()),
+ optional(SOURCE.argName(), SOURCE.arg()),
+ optional(SYNC.argName()));
usage(log, "Snapshot restore operation status (Command deprecated. Use '" + SNAPSHOT + ' '
+ SnapshotSubcommands.STATUS + "' instead):", SNAPSHOT, params, RESTORE.toString(), SNAPSHOT_NAME_ARG, "--status");
usage(log, "Cancel snapshot restore operation (Command deprecated. Use '" + SNAPSHOT + ' '
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java
index eb66ccfde01..fa1fe23cb65 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java
@@ -31,6 +31,10 @@ public enum SnapshotRestoreCommandOption implements CommandArg {
SOURCE(SnapshotCheckCommandOption.SOURCE.argName(), SnapshotCheckCommandOption.SOURCE.arg(),
SnapshotCheckCommandOption.SOURCE.description()),
+ /** Incremental snapshot index. */
+ INCREMENT("--increment", "incrementIndex", "Incremental snapshot index. The command will restore " +
+ "snapshot and after that all its increments sequentially from 1 to the specified index."),
+
/** Synchronous execution flag. */
SYNC(SnapshotCreateCommandOption.SYNC.argName(), SnapshotCreateCommandOption.SYNC.arg(),
SnapshotCreateCommandOption.SYNC.description());
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusCommand.java
index 8a4dceaea12..9058d3003a2 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusCommand.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusCommand.java
@@ -18,6 +18,8 @@
package org.apache.ignite.internal.commandline.snapshot;
import java.text.DateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
@@ -28,10 +30,12 @@ import org.apache.ignite.internal.commandline.CommandArgIterator;
import org.apache.ignite.internal.commandline.systemview.SystemViewCommand;
import org.apache.ignite.internal.util.GridStringBuilder;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T5;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.visor.snapshot.VisorSnapshotStatusTask;
import org.apache.ignite.internal.visor.snapshot.VisorSnapshotStatusTask.SnapshotStatus;
+import org.apache.ignite.internal.visor.systemview.VisorSystemViewTask;
import static org.apache.ignite.internal.commandline.CommandList.SNAPSHOT;
import static org.apache.ignite.internal.commandline.snapshot.SnapshotSubcommands.STATUS;
@@ -58,6 +62,7 @@ public class SnapshotStatusCommand extends SnapshotSubcommand {
SnapshotStatus status = (SnapshotStatus)res;
boolean isCreating = status.operation() == VisorSnapshotStatusTask.SnapshotOperation.CREATE;
+ boolean isIncremental = status.incrementIndex() > 0;
GridStringBuilder s = new GridStringBuilder();
@@ -67,6 +72,11 @@ public class SnapshotStatusCommand extends SnapshotSubcommand {
s.a("Restore snapshot operation is in progress.").nl();
s.a("Snapshot name: ").a(status.name()).nl();
+ s.a("Incremental: ").a(isIncremental).nl();
+
+ if (isIncremental)
+ s.a("Increment index: ").a(status.incrementIndex()).nl();
+
s.a("Operation request ID: ").a(status.requestId()).nl();
s.a("Started at: ").a(DateFormat.getDateTimeInstance().format(new Date(status.startTime()))).nl();
s.a("Duration: ").a(X.timeSpan2DHMSM(System.currentTimeMillis() - status.startTime())).nl()
@@ -75,27 +85,22 @@ public class SnapshotStatusCommand extends SnapshotSubcommand {
log.info(s.toString());
- List<String> titles = isCreating ? F.asList("Node ID", "Processed, bytes", "Total, bytes", "Percent") :
- F.asList("Node ID", "Processed, partitions", "Total, partitions", "Percent");
-
- List<List<?>> rows = status.progress().entrySet().stream().sorted(Map.Entry.comparingByKey()).map(e -> {
- UUID nodeId = e.getKey();
- long processed = e.getValue().get1();
- long total = e.getValue().get2();
+ SnapshotTaskProgressDesc desc;
- if (total <= 0)
- return F.asList(nodeId, "unknown", "unknown", "unknown");
-
- String percent = (int)(processed * 100 / total) + "%";
+ if (isCreating && isIncremental)
+ desc = new CreateIncrementalSnapshotTaskProgressDesc();
+ else if (isCreating)
+ desc = new CreateFullSnapshotTaskProgressDesc();
+ else if (isIncremental)
+ desc = new RestoreIncrementalSnapshotTaskProgressDesc();
+ else
+ desc = new RestoreFullSnapshotTaskProgressDesc();
- if (isCreating)
- return F.asList(nodeId, U.humanReadableByteCount(processed), U.humanReadableByteCount(total), percent);
- else
- return F.asList(nodeId, processed, total, percent);
- }).collect(Collectors.toList());
+ List<List<?>> rows = status.progress().entrySet().stream().sorted(Map.Entry.comparingByKey())
+ .map(e -> desc.buildRow(e.getKey(), e.getValue()))
+ .collect(Collectors.toList());
- SystemViewCommand.printTable(titles, F.asList(STRING, NUMBER, NUMBER, NUMBER),
- rows, log);
+ SystemViewCommand.printTable(desc.titles(), desc.types(), rows, log);
log.info(U.nl());
}
@@ -110,4 +115,143 @@ public class SnapshotStatusCommand extends SnapshotSubcommand {
@Override public void printUsage(IgniteLogger log) {
usage(log, "Get the status of the current snapshot operation:", SNAPSHOT, STATUS.toString());
}
+
+ /** Describes progress of a snapshot task. */
+ private abstract static class SnapshotTaskProgressDesc {
+ /** Progress table columns titles. */
+ private final List<String> titles;
+
+ /** */
+ SnapshotTaskProgressDesc(List<String> titles) {
+ this.titles = Collections.unmodifiableList(titles);
+ }
+
+ /** @return Progress table columns titles. */
+ List<String> titles() {
+ return titles;
+ }
+
+ /** @return Progress table columns types. */
+ List<VisorSystemViewTask.SimpleType> types() {
+ List<VisorSystemViewTask.SimpleType> types = new ArrayList<>();
+
+ types.add(STRING);
+
+ for (int i = 0; i < titles().size() - 1; i++)
+ types.add(NUMBER);
+
+ return types;
+ }
+
+ /** @return Progress table data row. */
+ abstract List<?> buildRow(UUID nodeId, T5<Long, Long, Long, Long, Long> progress);
+ }
+
+ /** */
+ private static class CreateFullSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc {
+ /** */
+ CreateFullSnapshotTaskProgressDesc() {
+ super(F.asList("Node ID", "Processed, bytes", "Total, bytes", "Percent"));
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<?> buildRow(UUID nodeId, T5<Long, Long, Long, Long, Long> progress) {
+ long processed = progress.get1();
+ long total = progress.get2();
+
+ if (total <= 0)
+ return F.asList(nodeId, "unknown", "unknown", "unknown");
+
+ String percent = (int)(processed * 100 / total) + "%";
+
+ return F.asList(nodeId, U.humanReadableByteCount(processed), U.humanReadableByteCount(total), percent);
+ }
+ }
+
+ /** */
+ private static class CreateIncrementalSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc {
+ /** */
+ CreateIncrementalSnapshotTaskProgressDesc() {
+ super(F.asList("Node ID", "Progress"));
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<?> buildRow(UUID nodeId, T5<Long, Long, Long, Long, Long> progress) {
+ return F.asList(nodeId, "unknown");
+ }
+ }
+
+ /** */
+ private static class RestoreFullSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc {
+ /** */
+ RestoreFullSnapshotTaskProgressDesc() {
+ super(F.asList("Node ID", "Processed, partitions", "Total, partitions", "Percent"));
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<?> buildRow(UUID nodeId, T5<Long, Long, Long, Long, Long> progress) {
+ long processed = progress.get1();
+ long total = progress.get2();
+
+ if (total <= 0)
+ return F.asList(nodeId, "unknown", "unknown", "unknown");
+
+ String percent = (int)(processed * 100 / total) + "%";
+
+ return F.asList(nodeId, processed, total, percent);
+ }
+ }
+
+ /** */
+ private static class RestoreIncrementalSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc {
+ /** */
+ RestoreIncrementalSnapshotTaskProgressDesc() {
+ super(F.asList(
+ "Node ID",
+ "Processed, partitions",
+ "Total, partitions",
+ "Percent",
+ "Processed, WAL segments",
+ "Total, WAL segments",
+ "Percent",
+ "Processed, WAL entries"));
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<?> buildRow(UUID nodeId, T5<Long, Long, Long, Long, Long> progress) {
+ List<Object> result = new ArrayList<>();
+ result.add(nodeId);
+
+ long processedParts = progress.get1();
+ long totalParts = progress.get2();
+
+ if (totalParts <= 0)
+ result.addAll(F.asList("unknown", "unknown", "unknown"));
+ else {
+ String partsPercent = (int)(processedParts * 100 / totalParts) + "%";
+
+ result.add(F.asList(processedParts, totalParts, partsPercent));
+ }
+
+ long processedWalSegs = progress.get3();
+ long totalWalSegs = progress.get4();
+
+ if (processedWalSegs <= 0)
+ result.addAll(F.asList("unknown", "unknown", "unknown"));
+ else {
+ String walSegsPercent = (int)(processedWalSegs * 100 / totalWalSegs) + "%";
+
+ result.add(F.asList(processedWalSegs, totalWalSegs, walSegsPercent));
+ }
+
+ long processedWalEntries = progress.get5();
+
+ if (processedWalEntries <= 0)
+ result.add("unknown");
+ else
+ result.add(processedWalEntries);
+
+ return result;
+ }
+ }
}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotSubcommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotSubcommand.java
index a8a65a6235e..5ae3437b680 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotSubcommand.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotSubcommand.java
@@ -87,7 +87,10 @@ public abstract class SnapshotSubcommand extends AbstractCommand<Object> {
* @return General usage options.
*/
protected Map<String, String> generalUsageOptions() {
- return F.asMap(SNAPSHOT_NAME_ARG, "Snapshot name.");
+ return F.asMap(
+ SNAPSHOT_NAME_ARG,
+ "Snapshot name. In case incremental snapshot (--incremental) full snapshot name must be provided."
+ );
}
/**
diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/CdcCommandTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/CdcCommandTest.java
index d920e08ab29..b04090e485b 100644
--- a/modules/control-utility/src/test/java/org/apache/ignite/util/CdcCommandTest.java
+++ b/modules/control-utility/src/test/java/org/apache/ignite/util/CdcCommandTest.java
@@ -193,7 +193,7 @@ public class CdcCommandTest extends GridCommandHandlerAbstractTest {
File[] links = wal0.walCdcDirectory().listFiles(WAL_SEGMENT_FILE_FILTER);
assertEquals(expLinks.size(), links.length);
- Arrays.stream(links).map(File::toPath).map(CdcMain::segmentIndex)
+ Arrays.stream(links).map(File::toPath).map(FileWriteAheadLogManager::segmentIndex)
.allMatch(expLinks::contains);
}
diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
index e3555a2e310..02c583d5459 100644
--- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
+++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
@@ -135,6 +135,9 @@ public abstract class GridCommandHandlerAbstractTest extends GridCommonAbstractT
/** Persistence flag. */
private boolean persistent = true;
+ /** WAL compaction flag. */
+ private boolean walCompaction;
+
/**
* Persistence setter.
*
@@ -144,6 +147,15 @@ public abstract class GridCommandHandlerAbstractTest extends GridCommonAbstractT
persistent = pr;
}
+ /**
+ * WAL compaction setter.
+ *
+ * @param walCompaction {@code True} If WAL compaction enable.
+ **/
+ protected void walCompactionEnabled(boolean walCompaction) {
+ this.walCompaction = walCompaction;
+ }
+
/**
* Persistence getter.
*
@@ -259,6 +271,7 @@ public abstract class GridCommandHandlerAbstractTest extends GridCommonAbstractT
DataStorageConfiguration dsCfg = new DataStorageConfiguration()
.setWalMode(WALMode.LOG_ONLY)
+ .setWalCompactionEnabled(walCompaction)
.setCheckpointFrequency(checkpointFreq)
.setDefaultDataRegionConfiguration(
new DataRegionConfiguration().setMaxSize(50L * 1024 * 1024).setPersistenceEnabled(persistent)
diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index cd9c76b2e62..a76fb684890 100644
--- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -3163,7 +3163,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
// Invalid command syntax check.
assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "create", snpName, "blah"));
- assertContains(log, testOut.toString(), "Invalid argument: blah. Possible options: --sync, --dest.");
+ assertContains(log, testOut.toString(), "Invalid argument: blah. Possible options: --sync, --dest, --incremental.");
assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "create", snpName, "--sync", "blah"));
assertContains(log, testOut.toString(), "Invalid argument: blah.");
@@ -3301,7 +3301,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
assertContains(log, testOut.toString(), "Invalid argument: --sync.");
assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "restore", snpName, "blah"));
- assertContains(log, testOut.toString(), "Invalid argument: blah. Possible options: --groups, --src, --sync.");
+ assertContains(log, testOut.toString(), "Invalid argument: blah. Possible options: --groups, --src, --increment, --sync.");
assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "restore", snpName, "--status", "--sync"));
assertContains(log, testOut.toString(), "Invalid argument: --sync. Action \"--status\" does not support specified option.");
@@ -3310,7 +3310,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
assertContains(log, testOut.toString(), "Invalid argument: --start.");
assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "restore", snpName, "--start", "blah"));
- assertContains(log, testOut.toString(), "Invalid argument: blah. Possible options: --groups, --src, --sync.");
+ assertContains(log, testOut.toString(), "Invalid argument: blah. Possible options: --groups, --src, --increment, --sync.");
autoConfirmation = true;
@@ -3377,7 +3377,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "restore", snpName, cacheName1));
assertContains(log, testOut.toString(),
- "Invalid argument: " + cacheName1 + ". Possible options: --groups, --src, --sync.");
+ "Invalid argument: " + cacheName1 + ". Possible options: --groups, --src, --increment, --sync.");
// Restore single cache group.
assertEquals(EXIT_CODE_OK, execute(h, "--snapshot", "restore", snpName, "--groups", cacheName1));
@@ -3461,6 +3461,85 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
}
}
+ /** @throws Exception If fails. */
+ @Test
+ public void testIncrementalSnapshotRestore() throws Exception {
+ walCompactionEnabled(true);
+
+ int keysCnt = 100;
+ String snpName = "snapshot_06032023";
+ String cacheName1 = "cache1";
+
+ IgniteEx ig = startGrids(2);
+
+ ig.cluster().state(ACTIVE);
+
+ injectTestSystemOut();
+ injectTestSystemIn(CONFIRM_MSG);
+
+ createCacheAndPreload(ig, cacheName1, keysCnt, 32, null);
+
+ ig.snapshot().createSnapshot(snpName).get(getTestTimeout());
+
+ try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(cacheName1)) {
+ for (int i = keysCnt; i < keysCnt * 2; i++)
+ streamer.addData(i, i);
+ }
+
+ ig.snapshot().createIncrementalSnapshot(snpName).get(getTestTimeout());
+
+ IgniteCache<Integer, Integer> cache1 = ig.cache(cacheName1);
+
+ cache1.destroy();
+
+ awaitPartitionMapExchange();
+
+ assertNull(ig.cache(cacheName1));
+
+ CommandHandler h = new CommandHandler();
+
+ // Missed increment index.
+ assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "restore", snpName, "--increment"));
+ assertContains(log, testOut.toString(), "Expected incremental snapshot index");
+
+ // Wrong params.
+ assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "restore", snpName, "--increment", "wrong"));
+ assertContains(log, testOut.toString(), "Invalid value for incremental snapshot index");
+
+ // Missed increment index.
+ assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "restore", snpName, "--increment", "1", "--increment", "2"));
+ assertContains(log, testOut.toString(), "increment arg specified twice");
+
+ // Non existent increment.
+ assertEquals(EXIT_CODE_UNEXPECTED_ERROR, execute(h, "--snapshot", "restore", snpName, "--increment", "2", "--sync"));
+
+ // Succesfull restore.
+ assertEquals(EXIT_CODE_OK, execute(h, "--snapshot", "restore", snpName, "--increment", "1", "--sync"));
+
+ cache1 = ig.cache(cacheName1);
+
+ assertNotNull(cache1);
+
+ for (int i = 0; i < 2 * keysCnt; i++)
+ assertEquals(Integer.valueOf(i), cache1.get(i));
+
+ cache1.destroy();
+
+ awaitPartitionMapExchange();
+
+ assertNull(ig.cache(cacheName1));
+
+ // Specify full increment index value.
+ assertEquals(EXIT_CODE_OK, execute(h, "--snapshot", "restore", snpName, "--increment", "0000000000000001", "--sync"));
+
+ cache1 = ig.cache(cacheName1);
+
+ assertNotNull(cache1);
+
+ for (int i = 0; i < 2 * keysCnt; i++)
+ assertEquals(Integer.valueOf(i), cache1.get(i));
+ }
+
/** @throws Exception If fails. */
@Test
public void testSnapshotCreateCheckAndRestoreCustomDir() throws Exception {
@@ -3597,12 +3676,14 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
startGrid();
- checkSnapshotStatus(false, false, null);
+ checkSnapshotStatus(false, false, false, null);
}
/** @throws Exception If fails. */
@Test
public void testSnapshotStatus() throws Exception {
+ walCompactionEnabled(true);
+
String snapshotName = "snapshot1";
int keysCnt = 10_000;
@@ -3614,24 +3695,41 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
createCacheAndPreload(srv, keysCnt);
- checkSnapshotStatus(false, false, null);
+ checkSnapshotStatus(false, false, false, null);
TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(grid(1));
spi.blockMessages((node, msg) -> msg instanceof SingleNodeMessage);
+ // Create snapshot.
IgniteFuture<Void> fut = srv.snapshot().createSnapshot(snapshotName);
spi.waitForBlocked();
- checkSnapshotStatus(true, false, snapshotName);
+ checkSnapshotStatus(true, false, false, snapshotName);
+
+ spi.stopBlock();
+
+ fut.get(getTestTimeout());
+
+ checkSnapshotStatus(false, false, false, null);
+
+ // Create incremental snapshot.
+ spi.blockMessages((node, msg) -> msg instanceof SingleNodeMessage);
+
+ fut = srv.snapshot().createIncrementalSnapshot(snapshotName);
+
+ spi.waitForBlocked();
+
+ checkSnapshotStatus(true, false, true, snapshotName);
spi.stopBlock();
fut.get(getTestTimeout());
- checkSnapshotStatus(false, false, null);
+ checkSnapshotStatus(false, false, false, null);
+ // Restore snapshot.
srv.destroyCache(DEFAULT_CACHE_NAME);
awaitPartitionMapExchange();
@@ -3642,21 +3740,41 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
spi.waitForBlocked();
- checkSnapshotStatus(false, true, snapshotName);
+ checkSnapshotStatus(false, true, false, snapshotName);
+
+ spi.stopBlock();
+
+ fut.get(getTestTimeout());
+
+ checkSnapshotStatus(false, false, false, null);
+
+ // Restore incremental snapshot.
+ srv.destroyCache(DEFAULT_CACHE_NAME);
+
+ awaitPartitionMapExchange();
+
+ spi.blockMessages((node, msg) -> msg instanceof SingleNodeMessage);
+
+ fut = srv.snapshot().restoreSnapshot(snapshotName, F.asList(DEFAULT_CACHE_NAME), 1);
+
+ spi.waitForBlocked();
+
+ checkSnapshotStatus(false, true, true, snapshotName);
spi.stopBlock();
fut.get(getTestTimeout());
- checkSnapshotStatus(false, false, null);
+ checkSnapshotStatus(false, false, false, null);
}
/**
* @param isCreating {@code True} if create snapshot operation is in progress.
* @param isRestoring {@code True} if restore snapshot operation is in progress.
+ * @param isIncremental {@code True} if incremental snapshot operation.
* @param expName Expected snapshot name.
*/
- private void checkSnapshotStatus(boolean isCreating, boolean isRestoring, String expName) throws Exception {
+ private void checkSnapshotStatus(boolean isCreating, boolean isRestoring, boolean isIncremental, String expName) throws Exception {
Collection<Ignite> srvs = F.view(G.allGrids(), n -> !n.cluster().localNode().isClient());
assertTrue(waitForCondition(() -> srvs.stream().allMatch(
@@ -3686,8 +3804,12 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
else
assertContains(log, out, "Restore snapshot operation is in progress.");
+ assertContains(log, out, "Incremental: " + isIncremental);
assertContains(log, out, "Snapshot name: " + expName);
+ if (isIncremental)
+ assertContains(log, out, "Increment index: 1");
+
srvs.forEach(srv -> assertContains(log, out, srv.cluster().localNode().id().toString()));
}
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSnapshot.java b/modules/core/src/main/java/org/apache/ignite/IgniteSnapshot.java
index c18c9afe003..2f6cc9f4988 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSnapshot.java
@@ -40,10 +40,18 @@ public interface IgniteSnapshot {
* Create a consistent copy of all persistence cache groups from the whole cluster.
*
* @param name Snapshot unique name which satisfies the following name pattern [a-zA-Z0-9_].
- * @return Future which will be completed when a process ends.
+ * @return Future which will be completed when the process ends.
*/
public IgniteFuture<Void> createSnapshot(String name);
+ /**
+ * Create an incremental snapshot for an existing snapshot.
+ *
+ * @param snapshotName Snapshot name.
+ * @return Future which will be completed when the process ends.
+ */
+ public IgniteFuture<Void> createIncrementalSnapshot(String snapshotName);
+
/**
* Cancel running snapshot operation. All intermediate results of cancelled snapshot operation will be deleted.
* If snapshot already created this command will have no effect.
@@ -65,6 +73,20 @@ public interface IgniteSnapshot {
*/
public IgniteFuture<Void> restoreSnapshot(String name, @Nullable Collection<String> cacheGroupNames);
+ /**
+ * Restore cache group(s) from the snapshot and its increments. Snapshot is restored first and after that all increments
+ * are restored sequentially from the {@code 1} to the specified {@code incIdx}.
+ * <p>
+ * <b>NOTE:</b> Cache groups to be restored from the snapshot must not present in the cluster, if they present,
+ * they must be destroyed by the user (eg with {@link IgniteCache#destroy()}) before starting this operation.
+ *
+ * @param name Snapshot name.
+ * @param cacheGroupNames Cache groups to be restored or {@code null} to restore all cache groups from the snapshot.
+ * @param incIdx Index of incremental snapshot.
+ * @return Future which will be completed when restore operation finished.
+ */
+ public IgniteFuture<Void> restoreSnapshot(String name, @Nullable Collection<String> cacheGroupNames, int incIdx);
+
/**
* Cancel snapshot restore operation.
*
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java b/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java
index fb82e310a50..9ff1fc7adb3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java
@@ -52,8 +52,10 @@ import org.apache.ignite.internal.binary.BinaryUtils;
import org.apache.ignite.internal.cdc.WalRecordsConsumer.DataEntryIterator;
import org.apache.ignite.internal.processors.cache.GridLocalConfigManager;
import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
import org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneGridKernalContext;
@@ -79,10 +81,10 @@ import static org.apache.ignite.internal.IgnitionEx.initializeDefaultMBeanServer
import static org.apache.ignite.internal.binary.BinaryUtils.METADATA_FILE_SUFFIX;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_RECORD_V2;
import static org.apache.ignite.internal.processors.cache.GridCacheUtils.UTILITY_CACHE_NAME;
-import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_DATA_FILENAME;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_DIR_PREFIX;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_GRP_DIR_PREFIX;
import static org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.segmentIndex;
import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName;
/**
@@ -445,7 +447,7 @@ public class CdcMain implements Runnable {
// Need unseen WAL segments only.
.filter(p -> WAL_SEGMENT_FILE_FILTER.accept(p.toFile()) && !seen.contains(p))
.peek(seen::add) // Adds to seen.
- .sorted(Comparator.comparingLong(CdcMain::segmentIndex)) // Sort by segment index.
+ .sorted(Comparator.comparingLong(FileWriteAheadLogManager::segmentIndex)) // Sort by segment index.
.peek(p -> {
long nextSgmnt = segmentIndex(p);
@@ -705,7 +707,7 @@ public class CdcMain implements Runnable {
.filter(File::exists)
// Cache group directory can contain several cache data files.
// See GridLocalConfigManager#cacheConfigurationFile(CacheConfiguration<?, ?>)
- .flatMap(cacheDir -> Arrays.stream(cacheDir.listFiles(f -> f.getName().endsWith(CACHE_DATA_FILENAME))))
+ .flatMap(cacheDir -> Arrays.stream(FilePageStoreManager.cacheDataFiles(cacheDir)))
.map(f -> {
try {
CdcCacheEvent evt = GridLocalConfigManager.readCacheData(
@@ -814,16 +816,6 @@ public class CdcMain implements Runnable {
}
}
- /**
- * @param segment WAL segment file.
- * @return Segment index.
- */
- public static long segmentIndex(Path segment) {
- String fn = segment.getFileName().toString();
-
- return Long.parseLong(fn.substring(0, fn.indexOf('.')));
- }
-
/** Stops the application. */
public void stop() {
synchronized (this) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 51aade1e347..3c0c2fbc78f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -136,6 +136,7 @@ import org.apache.ignite.internal.processors.cache.mvcc.msg.MvccQuerySnapshotReq
import org.apache.ignite.internal.processors.cache.mvcc.msg.MvccRecoveryFinishedMessage;
import org.apache.ignite.internal.processors.cache.mvcc.msg.MvccSnapshotResponse;
import org.apache.ignite.internal.processors.cache.mvcc.msg.MvccTxSnapshotRequest;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.IncrementalSnapshotAwareMessage;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesFailureMessage;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesRequestMessage;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryRequest;
@@ -382,6 +383,9 @@ public class GridIoMessageFactory implements MessageFactoryProvider {
factory.register(SnapshotFilesRequestMessage.TYPE_CODE, SnapshotFilesRequestMessage::new);
factory.register(SnapshotFilesFailureMessage.TYPE_CODE, SnapshotFilesFailureMessage::new);
+ // Incremental snapshot.
+ factory.register(IncrementalSnapshotAwareMessage.TYPE_CODE, IncrementalSnapshotAwareMessage::new);
+
// Index statistics.
factory.register(StatisticsKeyMessage.TYPE_CODE, StatisticsKeyMessage::new);
factory.register(StatisticsDecimalMessage.TYPE_CODE, StatisticsDecimalMessage::new);
@@ -390,8 +394,11 @@ public class GridIoMessageFactory implements MessageFactoryProvider {
factory.register(StatisticsRequest.TYPE_CODE, StatisticsRequest::new);
factory.register(StatisticsResponse.TYPE_CODE, StatisticsResponse::new);
- // [-3..119] [124..129] [-23..-28] [-36..-55] - this
+ // [-3..119] [124..129] [-23..-28] [-36..-55] [183..188] - this
// [120..123] - DR
+ // [-44, 0..2, 42, 200..204, 210, 302] - Use in tests.
+ // [300..307, 350..352] - CalciteMessageFactory.
+ // [400] - Incremental snapshot.
// [-4..-22, -30..-35, -54..-57] - SQL
// [2048..2053] - Snapshots
// [-42..-37] - former hadoop.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/SnapshotViewWalker.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/SnapshotViewWalker.java
index bee5fd35b21..bf65e9f3dd3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/SnapshotViewWalker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/SnapshotViewWalker.java
@@ -33,6 +33,9 @@ public class SnapshotViewWalker implements SystemViewRowAttributeWalker<Snapshot
v.accept(1, "consistentId", String.class);
v.accept(2, "baselineNodes", String.class);
v.accept(3, "cacheGroups", String.class);
+ v.accept(4, "snapshotRecordSegment", Long.class);
+ v.accept(5, "incrementIndex", Integer.class);
+ v.accept(6, "type", String.class);
}
/** {@inheritDoc} */
@@ -41,10 +44,13 @@ public class SnapshotViewWalker implements SystemViewRowAttributeWalker<Snapshot
v.accept(1, "consistentId", String.class, row.consistentId());
v.accept(2, "baselineNodes", String.class, row.baselineNodes());
v.accept(3, "cacheGroups", String.class, row.cacheGroups());
+ v.accept(4, "snapshotRecordSegment", Long.class, row.snapshotRecordSegment());
+ v.accept(5, "incrementIndex", Integer.class, row.incrementIndex());
+ v.accept(6, "type", String.class, row.type());
}
/** {@inheritDoc} */
@Override public int count() {
- return 4;
+ return 7;
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index a2e1d481129..35a01a0ad83 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -17,9 +17,11 @@
package org.apache.ignite.internal.pagemem.wal;
+import java.io.File;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.pagemem.wal.record.RolloverType;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.processors.cache.GridCacheSharedManager;
@@ -232,4 +234,25 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
* Start automatically releasing segments when reaching {@link DataStorageConfiguration#getMaxWalArchiveSize()}.
*/
void startAutoReleaseSegments();
+
+ /**
+ * Archive directory if any.
+ *
+ * @return Archive directory.
+ */
+ @Nullable File archiveDir();
+
+ /**
+ * @param idx Segment index.
+ * @return Compressed archive segment.
+ */
+ @Nullable File compactedSegment(long idx);
+
+ /**
+ * Blocks current thread while segment with the {@code idx} not compressed.
+ * If segment compressed, already, returns immediately.
+ *
+ * @param idx Segment index.
+ */
+ void awaitCompacted(long idx) throws IgniteInterruptedCheckedException;
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/IncrementalSnapshotFinishRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/IncrementalSnapshotFinishRecord.java
new file mode 100644
index 00000000000..a71c78d4d50
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/IncrementalSnapshotFinishRecord.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.pagemem.wal.record;
+
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * This record is written to WAL after incremental snapshot finished on a baseline node.
+ * <p>
+ * During recovery node must apply:
+ * <ul>
+ * <li>1. Transactions committed before {@link IncrementalSnapshotStartRecord} except those contained in {@link #excluded()}.</li>
+ * <li>2. Transactions committed between {@link IncrementalSnapshotStartRecord} and {@link IncrementalSnapshotFinishRecord}
+ * and contained in {@link #included()}.</li>
+ * </ul>
+ */
+public class IncrementalSnapshotFinishRecord extends WALRecord {
+ /** Incremental snapshot ID. */
+ @GridToStringInclude
+ private final UUID id;
+
+ /**
+ * Set of transactions committed between {@link IncrementalSnapshotStartRecord} and {@link IncrementalSnapshotFinishRecord}
+ * to include into the incremental snapshot.
+ */
+ @GridToStringInclude
+ private final Set<GridCacheVersion> included;
+
+ /**
+ * Set of transactions committed before {@link IncrementalSnapshotStartRecord} to exclude from the incremental snapshot.
+ */
+ @GridToStringInclude
+ private final Set<GridCacheVersion> excluded;
+
+ /** */
+ public IncrementalSnapshotFinishRecord(UUID id, Set<GridCacheVersion> included, Set<GridCacheVersion> excluded) {
+ this.id = id;
+ this.included = included;
+ this.excluded = excluded;
+ }
+
+ /** */
+ public Set<GridCacheVersion> included() {
+ return included;
+ }
+
+ /** */
+ public Set<GridCacheVersion> excluded() {
+ return excluded;
+ }
+
+ /** */
+ public UUID id() {
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public RecordType type() {
+ return RecordType.INCREMENTAL_SNAPSHOT_FINISH_RECORD;
+ }
+
+ /**
+ * Calculating the size of the record.
+ *
+ * @return Size in bytes.
+ */
+ public int dataSize() {
+ int size = 16 + 4 + 4; // ID, included and excluded tx count.
+
+ for (GridCacheVersion v: included)
+ size += CacheVersionIO.size(v, false);
+
+ for (GridCacheVersion v: excluded)
+ size += CacheVersionIO.size(v, false);
+
+ return size;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IncrementalSnapshotFinishRecord.class, this);
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/IncrementalSnapshotStartRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/IncrementalSnapshotStartRecord.java
new file mode 100644
index 00000000000..40fad249707
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/IncrementalSnapshotStartRecord.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.pagemem.wal.record;
+
+import java.util.UUID;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * This record is written to WAL at the moment when incremental snapshot starts on a baseline node.
+ */
+public class IncrementalSnapshotStartRecord extends WALRecord {
+ /** Incremental snapshot ID. */
+ @GridToStringInclude
+ private final UUID id;
+
+ /** @param id Incremental snapshot ID. */
+ public IncrementalSnapshotStartRecord(UUID id) {
+ this.id = id;
+ }
+
+ /** @return Incremental snapshot ID. */
+ public UUID id() {
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public RecordType type() {
+ return RecordType.INCREMENTAL_SNAPSHOT_START_RECORD;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IncrementalSnapshotStartRecord.class, this);
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
index c94038fdb1a..baa51c35286 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
@@ -282,7 +282,13 @@ public abstract class WALRecord {
ENCRYPTED_OUT_OF_ORDER_UPDATE(74, LOGICAL),
/** ClusterSnapshot start. */
- CLUSTER_SNAPSHOT(75, LOGICAL);
+ CLUSTER_SNAPSHOT(75, LOGICAL),
+
+ /** Incremental snapshot start record. */
+ INCREMENTAL_SNAPSHOT_START_RECORD(76, LOGICAL),
+
+ /** Incremental snapshot finish record. */
+ INCREMENTAL_SNAPSHOT_FINISH_RECORD(77, LOGICAL);
/** Index for serialization. Should be consistent throughout all versions. */
private final int idx;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 78607345d2a..cb7bef39cf8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -1410,6 +1410,20 @@ public class GridCacheIoManager extends GridCacheSharedManagerAdapter {
addHandler(hndId, type, c, cacheHandlers);
}
+ /**
+ * Finds cache message handler by its ID and message class.
+ *
+ * @param hndId Message handler ID.
+ * @param msgCls Message class.
+ * @return Message handler.
+ */
+ public <Msg extends GridCacheMessage> IgniteBiInClosure<UUID, ? super Msg> cacheHandler(
+ int hndId,
+ Class<? extends GridCacheMessage> msgCls
+ ) {
+ return cacheHandlers.clsHandlers.get(new ListenerKey(hndId, msgCls));
+ }
+
/**
* @param hndId Message handler ID.
* @param type Type of message.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 6cb8c84b5b0..893893b8edd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3053,7 +3053,11 @@ public class GridCacheProcessor extends GridProcessorAdapter {
WalStateManager walStateMgr = new WalStateManager(ctx);
- IgniteSnapshotManager snapshotMgr = new IgniteSnapshotManager(ctx);
+ IgniteSnapshotManager snapshotMgr = ctx.plugins().createComponent(IgniteSnapshotManager.class);
+
+ if (snapshotMgr == null)
+ snapshotMgr = new IgniteSnapshotManager(ctx);
+
IgniteCacheSnapshotManager snpMgr = ctx.plugins().createComponent(IgniteCacheSnapshotManager.class);
if (snpMgr == null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridLocalConfigManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridLocalConfigManager.java
index 238b8538fce..755b5fc9e3b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridLocalConfigManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridLocalConfigManager.java
@@ -20,10 +20,9 @@ package org.apache.ignite.internal.processors.cache;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.ObjectInputStream;
import java.io.OutputStream;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
@@ -62,6 +61,7 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.marshaller.Marshaller;
import org.apache.ignite.marshaller.MarshallerUtils;
+import org.jetbrains.annotations.Nullable;
import static java.nio.file.Files.newDirectoryStream;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT;
@@ -197,18 +197,26 @@ public class GridLocalConfigManager {
/**
* @param conf File with stored cache data.
+ * @param marshaller Marshaller.
+ * @param cfg Ignite configuration.
* @return Cache data.
* @throws IgniteCheckedException If failed.
*/
public static StoredCacheData readCacheData(
File conf,
- Marshaller marshaller,
- IgniteConfiguration cfg
+ @Nullable Marshaller marshaller,
+ @Nullable IgniteConfiguration cfg
) throws IgniteCheckedException {
- try (InputStream stream = new BufferedInputStream(new FileInputStream(conf))) {
+ try (InputStream stream = new BufferedInputStream(Files.newInputStream(conf.toPath()))) {
+ if (marshaller == null || cfg == null) {
+ try (ObjectInputStream ostream = new ObjectInputStream(stream)) {
+ return (StoredCacheData)ostream.readObject();
+ }
+ }
+
return marshaller.unmarshal(stream, U.resolveClassLoader(cfg));
}
- catch (IgniteCheckedException | IOException e) {
+ catch (IgniteCheckedException | IOException | ClassNotFoundException e) {
throw new IgniteCheckedException("An error occurred during cache configuration loading from file [file=" +
conf.getAbsolutePath() + "]", e);
}
@@ -221,7 +229,7 @@ public class GridLocalConfigManager {
*/
public void writeCacheData(StoredCacheData cacheData, File conf) throws IgniteCheckedException {
// Pre-existing file will be truncated upon stream open.
- try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(conf))) {
+ try (OutputStream stream = new BufferedOutputStream(Files.newOutputStream(conf.toPath()))) {
marshaller.marshal(cacheData, stream);
}
catch (IOException e) {
@@ -352,9 +360,18 @@ public class GridLocalConfigManager {
* @param lsnr Instance of listener to add.
*/
public void addConfigurationChangeListener(BiConsumer<String, File> lsnr) {
- assert chgLock.isWriteLockedByCurrentThread();
+ if (chgLock.isWriteLockedByCurrentThread())
+ lsnrs.add(lsnr);
+ else {
+ chgLock.writeLock().lock();
- lsnrs.add(lsnr);
+ try {
+ lsnrs.add(lsnr);
+ }
+ finally {
+ chgLock.writeLock().unlock();
+ }
+ }
}
/**
@@ -481,7 +498,7 @@ public class GridLocalConfigManager {
* @param ccfg Cache configuration.
* @return Cache configuration file with respect to {@link CacheConfiguration#getGroupName} value.
*/
- private File cacheConfigurationFile(CacheConfiguration<?, ?> ccfg) {
+ public File cacheConfigurationFile(CacheConfiguration<?, ?> ccfg) {
File cacheWorkDir = cacheWorkDir(ccfg);
return ccfg.getGroupName() == null ? new File(cacheWorkDir, CACHE_DATA_FILENAME) :
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
index 4e5034a109e..a78653d9b7b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
@@ -509,7 +509,7 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCacheCompoundIdentity
+ n.id()));
}
else {
- cctx.io().send(n, req, tx.ioPolicy());
+ cctx.tm().sendTransactionMessage(n, req, tx, tx.ioPolicy());
if (msgLog.isDebugEnabled()) {
msgLog.debug("DHT finish fut, sent request dht [txId=" + tx.nearXidVersion() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index e5008e6ac89..d3ea93089ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -878,7 +878,7 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
}
try {
- cctx.io().send(tx.nearNodeId(), res, tx.ioPolicy());
+ cctx.tm().sendTransactionMessage(tx.nearNodeId(), res, tx, tx.ioPolicy());
if (msgLog.isDebugEnabled()) {
msgLog.debug("DHT prepare fut, sent response [txId=" + tx.nearXidVersion() +
@@ -1514,7 +1514,7 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
assert req.transactionNodes() != null;
try {
- cctx.io().send(n, req, tx.ioPolicy());
+ cctx.tm().sendTransactionMessage(n, req, tx, tx.ioPolicy());
if (msgLog.isDebugEnabled()) {
msgLog.debug("DHT prepare fut, sent request dht [txId=" + tx.nearXidVersion() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
index 69fb67a8db0..cd39b95e998 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
@@ -484,7 +484,7 @@ public class GridNearOptimisticSerializableTxPrepareFuture extends GridNearOptim
m.reads(),
m.writes());
- cctx.io().send(primary, req, tx.ioPolicy());
+ cctx.tm().sendTransactionMessage(primary, req, tx, tx.ioPolicy());
}
catch (ClusterTopologyCheckedException e) {
e.retryReadyFuture(cctx.nextAffinityReadyFuture(tx.topologyVersion()));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index bfa1c7f9227..68b32610bc8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -577,7 +577,7 @@ public class GridNearOptimisticTxPrepareFuture extends GridNearOptimisticTxPrepa
}
else {
try {
- cctx.io().send(n, req, tx.ioPolicy());
+ cctx.tm().sendTransactionMessage(n, req, tx, tx.ioPolicy());
if (msgLog.isDebugEnabled()) {
msgLog.debug("Near optimistic prepare fut, sent request [txId=" + tx.nearXidVersion() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index 336c3eca340..0988ac2ccc5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -407,7 +407,7 @@ public class GridNearPessimisticTxPrepareFuture extends GridNearTxPrepareFutureA
add((IgniteInternalFuture)fut);
try {
- cctx.io().send(primary, req, tx.ioPolicy());
+ cctx.tm().sendTransactionMessage(primary, req, tx, tx.ioPolicy());
if (msgLog.isDebugEnabled()) {
msgLog.debug("Near pessimistic prepare, sent request [txId=" + tx.nearXidVersion() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index 10afb69ad64..c5dd5efdb66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -807,7 +807,7 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
add(fut); // Append new future.
try {
- cctx.io().send(n, req, tx.ioPolicy());
+ cctx.tm().sendTransactionMessage(n, req, tx, tx.ioPolicy());
if (msgLog.isDebugEnabled()) {
msgLog.debug("Near finish fut, sent request [" +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index 659daa6c3ca..352791bf501 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -4009,6 +4009,9 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
commitOrRollbackStartTime.compareAndSet(0, System.nanoTime());
+ if (!onePhaseCommit)
+ incrementalSnapshotId(cctx.snapshotMgr().incrementalSnapshotId());
+
try {
// Make sure that here are no exceptions.
f.get();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheStripedExecutor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheStripedExecutor.java
new file mode 100644
index 00000000000..e735fc0fe72
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheStripedExecutor.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteInterruptedException;
+import org.apache.ignite.internal.util.StripedExecutor;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_RECOVERY_SEMAPHORE_PERMITS;
+import static org.apache.ignite.IgniteSystemProperties.getInteger;
+import static org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointReadWriteLock.CHECKPOINT_LOCK_HOLD_COUNT;
+
+/** Wrapper over {@link StripedExecutor}, that groups submitted tasks by cache group and partition. */
+public class CacheStripedExecutor {
+ /** Error appeared during submitted task execution. */
+ private final AtomicReference<IgniteCheckedException> error = new AtomicReference<>();
+
+ /** Delegate striped executor. */
+ private final StripedExecutor exec;
+
+ /** Limit number of concurrent tasks submitted to the executor. Helps to avoid OOM error. */
+ private final Semaphore semaphore;
+
+ /** */
+ public CacheStripedExecutor(StripedExecutor exec) {
+ this.exec = exec;
+
+ semaphore = new Semaphore(semaphorePermits(exec));
+ }
+
+ /**
+ * Submit task to striped executor.
+ *
+ * @param task Runnable task.
+ * @param grpId Group ID.
+ * @param partId Partition ID.
+ */
+ public void submit(Runnable task, int grpId, int partId) {
+ int stripes = exec.stripesCount();
+
+ int stripe = U.stripeIdx(stripes, grpId, partId);
+
+ assert stripe >= 0 && stripe <= stripes : "idx=" + stripe + ", stripes=" + stripes;
+
+ try {
+ semaphore.acquire();
+ }
+ catch (InterruptedException e) {
+ throw new IgniteInterruptedException(e);
+ }
+
+ exec.execute(stripe, () -> {
+ // WA for avoid assert check in PageMemory, that current thread hold chpLock.
+ CHECKPOINT_LOCK_HOLD_COUNT.set(1);
+
+ try {
+ task.run();
+ }
+ catch (Throwable err) {
+ onError(new IgniteCheckedException("Failed to execute submitted task [grpId=" + grpId + ", partId=" + partId + ']', err));
+ }
+ finally {
+ CHECKPOINT_LOCK_HOLD_COUNT.set(0);
+
+ semaphore.release();
+ }
+ });
+ }
+
+ /**
+ * Awaits while all submitted tasks completed.
+ *
+ * @throws IgniteCheckedException if any os submitted tasks failed.
+ */
+ public void awaitApplyComplete() throws IgniteCheckedException {
+ try {
+ // Await completion apply tasks in all stripes.
+ exec.awaitComplete();
+ }
+ catch (InterruptedException e) {
+ throw new IgniteInterruptedException(e);
+ }
+
+ // Checking error after all task applied.
+ IgniteCheckedException err = error.get();
+
+ if (err != null)
+ throw err;
+ }
+
+ /** @return {@code true} if any of submitted tasks failed. */
+ public boolean error() {
+ return error.get() != null;
+ }
+
+ /** @param e Error appeared during submitted task execution. */
+ public void onError(IgniteCheckedException e) {
+ error.compareAndSet(null, e);
+ }
+
+ /** @return Underlying striped executor. */
+ public StripedExecutor executor() {
+ return exec;
+ }
+
+ /**
+ * Calculate the maximum number of concurrent tasks for apply through the striped executor.
+ *
+ * @param exec Striped executor.
+ * @return Number of permits.
+ */
+ private int semaphorePermits(StripedExecutor exec) {
+ // 4 task per-stripe by default.
+ int permits = exec.stripesCount() * 4;
+
+ long maxMemory = Runtime.getRuntime().maxMemory();
+
+ // Heuristic calculation part of heap size as a maximum number of concurrent tasks.
+ int permits0 = (int)((maxMemory * 0.2) / (4096 * 2));
+
+ // May be for small heap. Get a low number of permits.
+ if (permits0 < permits)
+ permits = permits0;
+
+ // Property for override any calculation.
+ return getInteger(IGNITE_RECOVERY_SEMAPHORE_PERMITS, permits);
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 5d9a1e35c46..7cb74ed16d1 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -37,7 +37,6 @@ import java.util.NoSuchElementException;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
@@ -50,7 +49,6 @@ import org.apache.ignite.DataRegionMetrics;
import org.apache.ignite.DataRegionMetricsProvider;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteInterruptedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.SystemProperty;
@@ -176,7 +174,6 @@ import static java.util.stream.Collectors.toSet;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_DEFRAGMENTATION_REGION_SIZE_PERCENTAGE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PREFER_WAL_REBALANCE;
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_RECOVERY_SEMAPHORE_PERMITS;
import static org.apache.ignite.IgniteSystemProperties.getBoolean;
import static org.apache.ignite.IgniteSystemProperties.getInteger;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT;
@@ -2169,11 +2166,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
", walArchive=" + persistenceCfg.getWalArchivePath() + "]");
}
- AtomicReference<Throwable> applyError = new AtomicReference<>();
-
- StripedExecutor exec = cctx.kernalContext().pools().getStripedExecutorService();
-
- Semaphore semaphore = new Semaphore(semaphorePertmits(exec));
+ CacheStripedExecutor exec = new CacheStripedExecutor(cctx.kernalContext().pools().getStripedExecutorService());
long start = U.currentTimeMillis();
@@ -2187,7 +2180,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
try {
while (restoreBinaryState.hasNext()) {
- if (applyError.get() != null)
+ if (exec.error())
break;
WALRecord rec = restoreBinaryState.next();
@@ -2217,13 +2210,12 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
catch (Throwable t) {
U.error(log, "Failed to apply page snapshot. rec=[" + pageSnapshot + ']');
- applyError.compareAndSet(
- null,
+ exec.onError(
(t instanceof IgniteCheckedException) ?
(IgniteCheckedException)t :
new IgniteCheckedException("Failed to apply page snapshot", t));
}
- }, groupId, partId, exec, semaphore
+ }, groupId, partId, exec
);
}
@@ -2248,14 +2240,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
catch (Throwable t) {
U.error(log, "Failed to cancel or wait partition destroy. rec=[" + metaStateRecord + ']');
- applyError.compareAndSet(
- null,
+ exec.onError(
(t instanceof IgniteCheckedException) ?
(IgniteCheckedException)t :
new IgniteCheckedException("Failed to cancel or wait partition destroy", t));
}
}
- }, groupId, partId, exec, semaphore);
+ }, groupId, partId, exec);
}
break;
@@ -2271,7 +2262,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
pageMem.invalidate(groupId, partId);
schedulePartitionDestroy(groupId, partId);
- }, groupId, partId, exec, semaphore);
+ }, groupId, partId, exec);
}
break;
@@ -2295,13 +2286,12 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
catch (Throwable t) {
U.error(log, "Failed to apply page delta. rec=[" + pageDelta + ']');
- applyError.compareAndSet(
- null,
+ exec.onError(
(t instanceof IgniteCheckedException) ?
(IgniteCheckedException)t :
new IgniteCheckedException("Failed to apply page delta", t));
}
- }, groupId, partId, exec, semaphore);
+ }, groupId, partId, exec);
}
}
}
@@ -2309,7 +2299,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
finally {
it.close();
- awaitApplyComplete(exec, applyError);
+ exec.awaitApplyComplete();
}
if (!finalizeState)
@@ -2327,59 +2317,12 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
log.info("Finished applying memory changes [changesApplied=" + applied +
", time=" + (U.currentTimeMillis() - start) + " ms]");
- finalizeCheckpointOnRecovery(status.cpStartTs, status.cpStartId, status.startPtr, exec);
+ finalizeCheckpointOnRecovery(status.cpStartTs, status.cpStartId, status.startPtr, exec.executor());
}
return restoreBinaryState;
}
- /**
- * Calculate the maximum number of concurrent tasks for apply through the striped executor.
- *
- * @param exec Striped executor.
- * @return Number of permits.
- */
- private int semaphorePertmits(StripedExecutor exec) {
- // 4 task per-stripe by default.
- int permits = exec.stripesCount() * 4;
-
- long maxMemory = Runtime.getRuntime().maxMemory();
-
- // Heuristic calculation part of heap size as a maximum number of concurrent tasks.
- int permits0 = (int)((maxMemory * 0.2) / (4096 * 2));
-
- // May be for small heap. Get a low number of permits.
- if (permits0 < permits)
- permits = permits0;
-
- // Property for override any calculation.
- return getInteger(IGNITE_RECOVERY_SEMAPHORE_PERMITS, permits);
- }
-
- /**
- * @param exec Striped executor.
- * @param applyError Check error reference.
- */
- private void awaitApplyComplete(
- StripedExecutor exec,
- AtomicReference<Throwable> applyError
- ) throws IgniteCheckedException {
- try {
- // Await completion apply tasks in all stripes.
- exec.awaitComplete();
- }
- catch (InterruptedException e) {
- throw new IgniteInterruptedException(e);
- }
-
- // Checking error after all task applied.
- Throwable error = applyError.get();
-
- if (error != null)
- throw error instanceof IgniteCheckedException
- ? (IgniteCheckedException)error : new IgniteCheckedException(error);
- }
-
/**
* @param consumer Runnable task.
* @param grpId Group Id.
@@ -2390,64 +2333,17 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
Consumer<PageMemoryEx> consumer,
int grpId,
int partId,
- StripedExecutor exec,
- Semaphore semaphore
+ CacheStripedExecutor exec
) throws IgniteCheckedException {
assert consumer != null;
assert exec != null;
- assert semaphore != null;
PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId);
if (pageMem == null)
return;
- stripedApply(() -> consumer.accept(pageMem), grpId, partId, exec, semaphore);
- }
-
- /**
- * @param run Runnable task.
- * @param grpId Group Id.
- * @param partId Partition Id.
- * @param exec Striped executor.
- */
- public void stripedApply(
- Runnable run,
- int grpId,
- int partId,
- StripedExecutor exec,
- Semaphore semaphore
- ) {
- assert run != null;
- assert exec != null;
- assert semaphore != null;
-
- int stripes = exec.stripesCount();
-
- int stripe = U.stripeIdx(stripes, grpId, partId);
-
- assert stripe >= 0 && stripe <= stripes : "idx=" + stripe + ", stripes=" + stripes;
-
- try {
- semaphore.acquire();
- }
- catch (InterruptedException e) {
- throw new IgniteInterruptedException(e);
- }
-
- exec.execute(stripe, () -> {
- // WA for avoid assert check in PageMemory, that current thread hold chpLock.
- CHECKPOINT_LOCK_HOLD_COUNT.set(1);
-
- try {
- run.run();
- }
- finally {
- CHECKPOINT_LOCK_HOLD_COUNT.set(0);
-
- semaphore.release();
- }
- });
+ exec.submit(() -> consumer.accept(pageMem), grpId, partId);
}
/**
@@ -2661,15 +2557,11 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
long start = U.currentTimeMillis();
- AtomicReference<Throwable> applyError = new AtomicReference<>();
-
AtomicLong applied = new AtomicLong();
long lastArchivedSegment = cctx.wal().lastArchivedSegment();
- StripedExecutor exec = cctx.kernalContext().pools().getStripedExecutorService();
-
- Semaphore semaphore = new Semaphore(semaphorePertmits(exec));
+ CacheStripedExecutor exec = new CacheStripedExecutor(cctx.kernalContext().pools().getStripedExecutorService());
Map<GroupPartitionId, Integer> partitionRecoveryStates = new HashMap<>();
@@ -2753,7 +2645,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
if (cacheDesc == null)
continue;
- stripedApply(() -> {
+ exec.submit(() -> {
GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
if (skipRemovedIndexUpdates(cacheCtx.groupId(), PageIdAllocator.INDEX_PARTITION))
@@ -2766,11 +2658,11 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
U.error(log, "Failed to apply data entry, dataEntry=" + dataEntry +
", ptr=" + dataRec.position());
- applyError.compareAndSet(null, e);
+ exec.onError(e);
}
applied.incrementAndGet();
- }, cacheDesc.groupId(), dataEntry.partitionId(), exec, semaphore);
+ }, cacheDesc.groupId(), dataEntry.partitionId());
}
break;
@@ -2815,10 +2707,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
catch (IgniteCheckedException e) {
U.error(log, "Failed to apply page delta, " + pageDelta);
- applyError.compareAndSet(null, e);
+ exec.onError(e);
}
- }, pageDelta.groupId(), partId(pageDelta.pageId()), exec, semaphore);
+ }, pageDelta.groupId(), partId(pageDelta.pageId()), exec);
break;
@@ -2864,7 +2756,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
", partitionId=" + rec0.partitionId() + "]", e);
}
- stripedApply(() -> {
+ exec.submit(() -> {
try {
part.updateClearVersion(rec0.clearVersion());
@@ -2880,9 +2772,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
catch (IgniteCheckedException e) {
U.error(log, "Failed to apply partition clearing record, " + rec0);
- applyError.compareAndSet(null, e);
+ exec.onError(e);
}
- }, rec0.groupId(), rec0.partitionId(), exec, semaphore);
+ }, rec0.groupId(), rec0.partitionId());
}
break;
@@ -2899,7 +2791,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
cctx.kernalContext().query().skipFieldLookup(false);
}
- awaitApplyComplete(exec, applyError);
+ exec.awaitApplyComplete();
if (log.isInfoEnabled())
log.info("Finished applying WAL changes [updatesApplied=" + applied +
@@ -2958,6 +2850,23 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
GridDhtLocalPartition locPart = cacheCtx.topology().forceCreatePartition(partId);
+ if (applyDataEntry(cacheCtx, locPart, dataEntry) && dataEntry.partitionCounter() != 0)
+ cacheCtx.offheap().dataStore(locPart).updateInitialCounter(dataEntry.partitionCounter() - 1, 1);
+ }
+
+ /**
+ * Applies data entry.
+ *
+ * @param cacheCtx Cache context.
+ * @param locPart Local partition.
+ * @param dataEntry Data entry.
+ * @return {@code true} if entry has applied, otherwise {@code false}.
+ */
+ public boolean applyDataEntry(
+ GridCacheContext<?, ?> cacheCtx,
+ GridDhtLocalPartition locPart,
+ DataEntry dataEntry
+ ) throws IgniteCheckedException {
switch (dataEntry.op()) {
case CREATE:
case UPDATE:
@@ -2982,10 +2891,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
null);
}
- if (dataEntry.partitionCounter() != 0)
- cacheCtx.offheap().dataStore(locPart).updateInitialCounter(dataEntry.partitionCounter() - 1, 1);
-
- break;
+ return true;
case DELETE:
if (dataEntry instanceof MvccDataEntry) {
@@ -2999,16 +2905,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
((MvccDataEntry)dataEntry).mvccVer());
}
else
- cacheCtx.offheap().remove(cacheCtx, dataEntry.key(), partId, locPart);
-
- if (dataEntry.partitionCounter() != 0)
- cacheCtx.offheap().dataStore(locPart).updateInitialCounter(dataEntry.partitionCounter() - 1, 1);
+ cacheCtx.offheap().remove(cacheCtx, dataEntry.key(), locPart.id(), locPart);
- break;
+ return true;
case READ:
// do nothing
- break;
+ return false;
default:
throw new IgniteCheckedException("Invalid operation for WAL entry update: " + dataEntry.op());
@@ -3250,6 +3153,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
if (lastCpTs != 0)
metaStorage.write(checkpointInapplicableCpAndGroupIdToKey(lastCpTs, grpId), true);
+
+ cctx.snapshotMgr().disableIncrementalSnapshotsCreation(metaStorage, grpId);
}
catch (IgniteCheckedException e) {
log.error("Failed to mark last checkpoint as inapplicable for WAL rebalance for group: " + grpId, e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 726321d4e74..b1e75502b65 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -981,6 +981,14 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
throw new IgniteException("Directory doesn't match the cache or cache group prefix: " + dir);
}
+ /**
+ * @param root Root directory.
+ * @return Array of cache data files.
+ */
+ public static File[] cacheDataFiles(File root) {
+ return root.listFiles(f -> f.getName().endsWith(CACHE_DATA_FILENAME));
+ }
+
/** {@inheritDoc} */
@Override public boolean hasIndexStore(int grpId) {
return !grpsWithoutIdx.contains(grpId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotFutureTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotFutureTask.java
index cb9486a96d7..7ccc2f3944c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotFutureTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotFutureTask.java
@@ -121,6 +121,13 @@ abstract class AbstractSnapshotFutureTask<T> extends GridFutureAdapter<T> {
return reqId;
}
+ /**
+ * @return Set of cache groups included into snapshot operation.
+ */
+ public Set<Integer> affectedCacheGroups() {
+ return parts.keySet();
+ }
+
/**
* Initiates snapshot task.
*
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
index 1ec1842eb5f..a82e2e8eec5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
@@ -20,7 +20,6 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot;
import java.io.BufferedInputStream;
import java.io.Closeable;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -70,6 +69,7 @@ import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Predicate;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
@@ -78,6 +78,7 @@ import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSnapshot;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.binary.BinaryType;
+import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.compute.ComputeTask;
import org.apache.ignite.configuration.CacheConfiguration;
@@ -112,10 +113,14 @@ import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
import org.apache.ignite.internal.managers.systemview.walker.SnapshotViewWalker;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.store.PageStore;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotFinishRecord;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
import org.apache.ignite.internal.processors.cache.CacheObjectContext;
import org.apache.ignite.internal.processors.cache.CacheType;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
import org.apache.ignite.internal.processors.cache.GridLocalConfigManager;
@@ -130,6 +135,7 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStor
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadWriteMetastorage;
@@ -137,8 +143,10 @@ import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPa
import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
import org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneGridKernalContext;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.tree.DataRow;
import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
@@ -157,6 +165,7 @@ import org.apache.ignite.internal.util.GridCloseableIteratorAdapter;
import org.apache.ignite.internal.util.distributed.DistributedProcess;
import org.apache.ignite.internal.util.distributed.InitMessage;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
+import org.apache.ignite.internal.util.future.GridCompoundIdentityFuture;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
@@ -187,6 +196,7 @@ import static java.nio.file.StandardOpenOption.READ;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_SNAPSHOT_SEQUENTIAL_WRITE;
import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_BINARY_METADATA_PATH;
import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_MARSHALLER_PATH;
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_PATH;
import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_FAILED;
import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_FINISHED;
import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_STARTED;
@@ -208,6 +218,7 @@ import static org.apache.ignite.internal.pagemem.PageIdUtils.flag;
import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId;
import static org.apache.ignite.internal.pagemem.PageIdUtils.pageIndex;
import static org.apache.ignite.internal.pagemem.PageIdUtils.toDetailString;
+import static org.apache.ignite.internal.processors.cache.GridCacheUtils.baselineNode;
import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.binaryWorkDir;
import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.resolveBinaryWorkDir;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME;
@@ -225,11 +236,13 @@ import static org.apache.ignite.internal.processors.cache.persistence.tree.io.Pa
import static org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO.getType;
import static org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO.getVersion;
import static org.apache.ignite.internal.processors.configuration.distributed.DistributedLongProperty.detachedLongProperty;
+import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName;
import static org.apache.ignite.internal.processors.task.TaskExecutionOptions.options;
import static org.apache.ignite.internal.util.GridUnsafe.bufferAddress;
import static org.apache.ignite.internal.util.IgniteUtils.isLocalNodeCoordinator;
import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.END_SNAPSHOT;
import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.START_SNAPSHOT;
+import static org.apache.ignite.internal.util.io.GridFileUtils.ensureHardLinkAvailable;
import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_SNAPSHOT;
import static org.apache.ignite.spi.systemview.view.SnapshotView.SNAPSHOT_SYS_VIEW;
import static org.apache.ignite.spi.systemview.view.SnapshotView.SNAPSHOT_SYS_VIEW_DESC;
@@ -240,6 +253,7 @@ import static org.apache.ignite.spi.systemview.view.SnapshotView.SNAPSHOT_SYS_VI
* These major actions available:
* <ul>
* <li>Create snapshot of the whole cluster cache groups by triggering PME to achieve consistency.</li>
+ * <li>Create incremental snapshot of transactional cache groups by using Consistent Cut algorithm.</li>
* </ul>
*/
public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
@@ -274,6 +288,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/** Snapshot metrics prefix. */
public static final String SNAPSHOT_METRICS = "snapshot";
+ /** Incremental snapshot metrics prefix. */
+ public static final String INCREMENTAL_SNAPSHOT_METRICS = metricName("snapshot", "incremental");
+
/** Snapshot metafile extension. */
public static final String SNAPSHOT_METAFILE_EXT = ".smf";
@@ -295,6 +312,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/** Metastorage key to save currently running snapshot directory path. */
private static final String SNP_RUNNING_DIR_KEY = "snapshot-running-dir";
+ /** Prefix for meta store records which means that incremental snapshot creation is disabled for a cache group. */
+ private static final String INC_SNP_DISABLED_KEY_PREFIX = "grp-inc-snp-disabled-";
+
/** Default value of {@link IgniteSystemProperties#IGNITE_SNAPSHOT_SEQUENTIAL_WRITE}. */
public static final boolean DFLT_IGNITE_SNAPSHOT_SEQUENTIAL_WRITE = true;
@@ -326,6 +346,12 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/** Total snapshot files count which receiver should expect to receive. */
private static final String SNP_PARTITIONS_CNT = "partsCnt";
+ /** Incremental snapshots directory name. */
+ public static final String INC_SNP_DIR = "increments";
+
+ /** Pattern for incremental snapshot directory names. */
+ public static final Pattern INC_SNP_NAME_PATTERN = U.fixedLengthNumberNamePattern(null);
+
/**
* Local buffer to perform copy-on-write operations with pages for {@code SnapshotFutureTask.PageStoreSerialWriter}s.
* It is important to have only one buffer per thread (instead of creating each buffer per
@@ -403,12 +429,28 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/** Last seen cluster snapshot operation. */
private volatile ClusterSnapshotFuture lastSeenSnpFut = new ClusterSnapshotFuture();
+ /** Last seen incremental snapshot operation. */
+ private volatile ClusterSnapshotFuture lastSeenIncSnpFut;
+
/** Snapshot operation handlers. */
private final SnapshotHandlers handlers = new SnapshotHandlers();
/** Manager to receive responses of remote snapshot requests. */
private final SequentialRemoteSnapshotManager snpRmtMgr;
+ /** Incremental snapshot ID. */
+ private volatile UUID incSnpId;
+
+ /**
+ * While incremental snapshot is running every node wraps outgoing transaction messages into {@link IncrementalSnapshotAwareMessage}.
+ * Nodes start wrapping messages from the moment snapshot started on the node, and finsihes after all baseline
+ * nodes completed {@link #markWalFut}. This future completes after last {@link IncrementalSnapshotAwareMessage} was sent.
+ */
+ private volatile GridFutureAdapter<?> wrapMsgsFut;
+
+ /** Future that completes after {@link IncrementalSnapshotFinishRecord} was written. */
+ private volatile @Nullable IncrementalSnapshotMarkWalFuture markWalFut;
+
/** Snapshot transfer rate limit in bytes/sec. */
private final DistributedLongProperty snapshotTransferRate = detachedLongProperty(SNAPSHOT_TRANSFER_RATE_DMS_KEY);
@@ -428,7 +470,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
this::processLocalSnapshotStartStageResult, SnapshotStartDiscoveryMessage::new);
endSnpProc = new DistributedProcess<>(ctx, END_SNAPSHOT, this::initLocalSnapshotEndStage,
- this::processLocalSnapshotEndStageResult);
+ this::processLocalSnapshotEndStageResult, (reqId, req) -> new InitMessage<>(reqId, END_SNAPSHOT, req, true));
marsh = MarshallerUtils.jdkMarshaller(ctx.igniteInstanceName());
@@ -538,24 +580,44 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
String.class,
"The error message of last started cluster snapshot request which fail with an error. " +
"This value will be empty if last snapshot request has been completed successfully.");
- mreg.register("LocalSnapshotNames", this::localSnapshotNames, List.class,
+ mreg.register("LocalSnapshotNames", () -> localSnapshotNames(null), List.class,
"The list of names of all snapshots currently saved on the local node with respect to " +
"the configured via IgniteConfiguration snapshot working path.");
mreg.register("LastRequestId", () -> Optional.ofNullable(lastSeenSnpFut.rqId).map(UUID::toString).orElse(""),
String.class, "The ID of the last started snapshot operation.");
mreg.register("CurrentSnapshotTotalSize", () -> {
- SnapshotFutureTask task = currentSnapshotTask();
+ SnapshotFutureTask task = currentSnapshotTask(SnapshotFutureTask.class);
return task == null ? -1 : task.totalSize();
}, "Estimated size of current cluster snapshot in bytes on this node. The value may grow during snapshot creation.");
mreg.register("CurrentSnapshotProcessedSize", () -> {
- SnapshotFutureTask task = currentSnapshotTask();
+ SnapshotFutureTask task = currentSnapshotTask(SnapshotFutureTask.class);
return task == null ? -1 : task.processedSize();
}, "Processed size of current cluster snapshot in bytes on this node.");
+ MetricRegistry incSnpMReg = cctx.kernalContext().metric().registry(INCREMENTAL_SNAPSHOT_METRICS);
+
+ incSnpMReg.register("snapshotName",
+ () -> Optional.ofNullable(lastSeenIncSnpFut).map(f -> f.name).orElse(""),
+ String.class,
+ "The name of full snapshot for which the last incremental snapshot created on this node.");
+ incSnpMReg.register("incrementIndex",
+ () -> Optional.ofNullable(lastSeenIncSnpFut).map(f -> f.incIdx).orElse(0),
+ "Ihe index of the last incremental snapshot created on this node.");
+ incSnpMReg.register("startTime",
+ () -> Optional.ofNullable(lastSeenIncSnpFut).map(f -> f.startTime).orElse(0L),
+ "The system time of the last incremental snapshot creation start time on this node.");
+ incSnpMReg.register("endTime",
+ () -> Optional.ofNullable(lastSeenIncSnpFut).map(f -> f.endTime).orElse(0L),
+ "The system time of the last incremental snapshot creation end time on this node.");
+ incSnpMReg.register("error",
+ () -> Optional.ofNullable(lastSeenIncSnpFut).map(GridFutureAdapter::error).map(Object::toString).orElse(""),
+ String.class,
+ "The error message of last started incremental snapshot on this node.");
+
restoreCacheGrpProc.registerMetrics();
cctx.exchange().registerExchangeAwareComponent(this);
@@ -604,8 +666,23 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
SNAPSHOT_SYS_VIEW,
SNAPSHOT_SYS_VIEW_DESC,
new SnapshotViewWalker(),
- () -> F.flatCollections(F.transform(localSnapshotNames(), name -> readSnapshotMetadatas(name, null))),
- this::snapshotViewSupplier);
+ () -> F.flatCollections(F.transform(localSnapshotNames(null), name -> {
+ List<SnapshotView> views = new ArrayList<>();
+
+ for (SnapshotMetadata m: readSnapshotMetadatas(name, null)) {
+ List<File> dirs = snapshotCacheDirectories(m.snapshotName(), null, m.folderName(), grpName -> true);
+
+ Collection<String> cacheGrps = F.viewReadOnly(dirs, FilePageStoreManager::cacheGroupName);
+
+ views.add(new SnapshotView(m, cacheGrps));
+ }
+
+ for (IncrementalSnapshotMetadata m: readIncrementalSnapshotMetadatas(name))
+ views.add(new SnapshotView(m));
+
+ return views;
+ })),
+ Function.identity());
}
/** {@inheritDoc} */
@@ -739,6 +816,42 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
return snpPath == null ? new File(locSnpDir, snpName) : new File(snpPath, snpName);
}
+ /**
+ * Returns path to specific incremental snapshot.
+ * For example, {@code "work/snapshots/mybackup/increments/0000000000000001"}.
+ *
+ * @param snpName Snapshot name.
+ * @param snpPath Snapshot directory path.
+ * @param incIdx Increment index.
+ * @return Local snapshot directory where snapshot files are located.
+ */
+ public File incrementalSnapshotLocalDir(String snpName, @Nullable String snpPath, int incIdx) {
+ return new File(incrementalSnapshotsLocalRootDir(snpName, snpPath), U.fixedLengthNumberName(incIdx, null));
+ }
+
+ /**
+ * Returns root folder for incremental snapshot.
+ * For example, {@code "work/snapshots/mybackup/increments/"}.
+ *
+ * @param snpName Snapshot name.
+ * @param snpPath Snapshot directory path.
+ * @return Local snapshot directory where snapshot files are located.
+ */
+ public File incrementalSnapshotsLocalRootDir(String snpName, @Nullable String snpPath) {
+ return new File(snapshotLocalDir(snpName, snpPath), INC_SNP_DIR);
+ }
+
+ /**
+ * @param incSnpDir Incremental snapshot directory.
+ * @param consId Consistent ID.
+ * @return WALs directory for specified incremental snapshot.
+ */
+ public static File incrementalSnapshotWalsDir(File incSnpDir, String consId) {
+ String folderName = U.maskForFileName(consId);
+
+ return incSnpDir.toPath().resolve(DFLT_WAL_PATH).resolve(folderName).toFile();
+ }
+
/**
* @param snpName Snapshot name.
* @return Local snapshot directory for snapshot with given name.
@@ -776,6 +889,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
clusterSnpReq = req;
+ if (req.incremental())
+ handleIncrementalSnapshotId(req.requestId(), cctx.discovery().topologyVersion());
+
if (!CU.baselineNode(cctx.localNode(), cctx.kernalContext().state().clusterState()))
return new GridFinishedFuture<>();
@@ -813,6 +929,183 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
"on the local node [missed=" + leftGrps + ", nodeId=" + cctx.localNodeId() + ']'));
}
+ if (req.incremental()) {
+ SnapshotMetadata meta;
+
+ try {
+ meta = readSnapshotMetadata(new File(
+ snapshotLocalDir(req.snapshotName(), req.snapshotPath()),
+ snapshotMetaFileName(cctx.localNode().consistentId().toString())
+ ));
+
+ checkIncrementalCanBeCreated(req.snapshotName(), req.snapshotPath(), meta);
+ }
+ catch (IgniteCheckedException | IOException e) {
+ return new GridFinishedFuture<>(e);
+ }
+
+ return initLocalIncrementalSnapshot(req, meta);
+ }
+ else
+ return initLocalFullSnapshot(req, grpIds, comprGrpIds, withMetaStorage);
+ }
+
+ /**
+ * Handles received incremental snapshot ID from remote node.
+ *
+ * @param id Incremental snapshot ID.
+ * @param topVer Incremental snapshot topology version.
+ */
+ public void handleIncrementalSnapshotId(UUID id, long topVer) {
+ // TODO: IGNITE-18599 handle if `id != incSnpId`.
+ if (incSnpId != null)
+ return;
+
+ synchronized (snpOpMux) {
+ if (incSnpId != null) {
+ if (!incSnpId.equals(id))
+ U.warn(log, "Received incremental snapshot ID differs from the current [rcvId=" + id + ", currId=" + incSnpId + ']');
+
+ return;
+ }
+
+ wrapMsgsFut = new GridFutureAdapter<>();
+
+ cctx.tm().txMessageTransformer((msg, tx) -> new IncrementalSnapshotAwareMessage(
+ msg, id, tx == null ? null : tx.incrementalSnapshotId(), topVer));
+
+ markWalFut = baselineNode(cctx.localNode(), cctx.kernalContext().state().clusterState())
+ ? new IncrementalSnapshotMarkWalFuture(cctx, id, topVer) : null;
+
+ incSnpId = id;
+ }
+
+ if (markWalFut != null)
+ cctx.kernalContext().pools().getSnapshotExecutorService().submit(markWalFut::init);
+ }
+
+ /**
+ * @param req Request on snapshot creation.
+ * @param meta Full snapshot metadata.
+ * @return Future which will be completed when a snapshot has been started.
+ */
+ private IgniteInternalFuture<SnapshotOperationResponse> initLocalIncrementalSnapshot(
+ SnapshotOperationRequest req,
+ SnapshotMetadata meta
+ ) {
+ File incSnpDir = incrementalSnapshotLocalDir(req.snapshotName(), req.snapshotPath(), req.incrementIndex());
+ WALPointer lowPtr;
+
+ if (req.incrementIndex() == 1)
+ lowPtr = meta.snapshotRecordPointer();
+ else {
+ int prevIdx = req.incrementIndex() - 1;
+
+ IncrementalSnapshotMetadata prevIncSnpMeta;
+
+ try {
+ prevIncSnpMeta = readIncrementalSnapshotMetadata(req.snapshotName(), req.snapshotPath(), prevIdx);
+ }
+ catch (IgniteCheckedException | IOException e) {
+ return new GridFinishedFuture<>(e);
+ }
+
+ lowPtr = prevIncSnpMeta.incrementalSnapshotPointer();
+ }
+
+ IgniteInternalFuture<SnapshotOperationResponse> task0 = registerTask(req.snapshotName(), new IncrementalSnapshotFutureTask(
+ cctx,
+ req.operationalNodeId(),
+ req.requestId(),
+ meta,
+ req.snapshotPath(),
+ req.incrementIndex(),
+ tmpWorkDir,
+ ioFactory,
+ lowPtr,
+ markWalFut
+ )).chain(fut -> {
+ if (fut.error() != null)
+ throw F.wrap(fut.error());
+
+ assert incSnpDir.exists() : "Incremental snapshot directory must exists";
+
+ IncrementalSnapshotMetadata incMeta = new IncrementalSnapshotMetadata(
+ req.requestId(),
+ req.snapshotName(),
+ req.incrementIndex(),
+ cctx.localNode().consistentId().toString(),
+ pdsSettings.folderName(),
+ markWalFut.result()
+ );
+
+ storeSnapshotMeta(
+ incMeta,
+ new File(incSnpDir, snapshotMetaFileName(pdsSettings.folderName()))
+ );
+
+ return new SnapshotOperationResponse();
+ });
+
+ if (task0.isDone())
+ return task0;
+
+ if (log.isDebugEnabled()) {
+ log.debug("Incremental snapshot operation submited for execution " +
+ "[snpName=" + req.snapshotName() + ", incIdx=" + req.incrementIndex());
+ }
+
+ cctx.kernalContext().pools().getSnapshotExecutorService().submit(() -> {
+ SnapshotOperationRequest snpReq = clusterSnpReq;
+
+ AbstractSnapshotFutureTask<?> task = locSnpTasks.get(snpReq.snapshotName());
+
+ if (task == null)
+ return;
+
+ if (log.isDebugEnabled()) {
+ log.debug("Incremental snapshot operation started " +
+ "[snpName=" + req.snapshotName() + ", incIdx=" + req.incrementIndex());
+ }
+
+ writeSnapshotDirectoryToMetastorage(incSnpDir);
+
+ task.start();
+ });
+
+ return task0;
+ }
+
+ /**
+ * @param snpName Full snapshot name.
+ * @param snpPath Optional path to snapshot, if differs from default.
+ * @param incIdx Index of incremental snapshot.
+ * @return Read incremental snapshot metadata.
+ */
+ public IncrementalSnapshotMetadata readIncrementalSnapshotMetadata(
+ String snpName,
+ @Nullable String snpPath,
+ int incIdx
+ ) throws IgniteCheckedException, IOException {
+ return readFromFile(new File(
+ incrementalSnapshotLocalDir(snpName, snpPath, incIdx),
+ snapshotMetaFileName(pdsSettings.folderName())
+ ));
+ }
+
+ /**
+ * @param req Request
+ * @param grpIds Groups.
+ * @param comprGrpIds Compressed Groups.
+ * @param withMetaStorage Flag to include metastorage.
+ * @return Create snapshot future.
+ */
+ private IgniteInternalFuture<SnapshotOperationResponse> initLocalFullSnapshot(
+ SnapshotOperationRequest req,
+ List<Integer> grpIds,
+ Collection<Integer> comprGrpIds,
+ boolean withMetaStorage
+ ) {
Map<Integer, Set<Integer>> parts = new HashMap<>();
// Prepare collection of pairs group and appropriate cache partition to be snapshot.
@@ -855,6 +1148,8 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
snpDir.mkdirs();
+ SnapshotFutureTaskResult res = (SnapshotFutureTaskResult)fut.result();
+
SnapshotMetadata meta = new SnapshotMetadata(req.requestId(),
req.snapshotName(),
cctx.localNode().consistentId().toString(),
@@ -863,7 +1158,8 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
grpIds,
comprGrpIds,
blts,
- (Set<GroupPartitionId>)fut.result(),
+ res.parts(),
+ res.snapshotPointer(),
cctx.gridConfig().getEncryptionSpi().masterKeyDigest()
);
@@ -874,13 +1170,13 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
File smf = new File(snpDir, snapshotMetaFileName(cctx.localNode().consistentId().toString()));
- storeSnapshotMeta(req, smf);
+ storeSnapshotMeta(req.meta(), smf);
log.info("Snapshot metafile has been created: " + smf.getAbsolutePath());
return new SnapshotOperationResponse(handlers.invokeAll(SnapshotHandlerType.CREATE, ctx));
}
- catch (IOException | IgniteCheckedException e) {
+ catch (IgniteCheckedException e) {
throw F.wrap(e);
}
}, snapshotExecutorService());
@@ -892,11 +1188,24 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
* @param err Errors.
*/
private void processLocalSnapshotStartStageResult(UUID id, Map<UUID, SnapshotOperationResponse> res, Map<UUID, Throwable> err) {
+ SnapshotOperationRequest snpReq = clusterSnpReq;
+
+ if (snpReq != null && F.eq(id, snpReq.requestId()) && snpReq.incremental()) {
+ cctx.tm().txMessageTransformer(null);
+
+ GridCompoundIdentityFuture<IgniteInternalTx> activeTxsFut = new GridCompoundIdentityFuture<>();
+
+ for (IgniteInternalTx tx: cctx.tm().activeTransactions())
+ activeTxsFut.add(tx.finishFuture());
+
+ activeTxsFut.markInitialized();
+
+ activeTxsFut.listen(f -> wrapMsgsFut.onDone());
+ }
+
if (cctx.kernalContext().clientNode())
return;
- SnapshotOperationRequest snpReq = clusterSnpReq;
-
boolean cancelled = err.values().stream().anyMatch(e -> e instanceof IgniteFutureCancelledCheckedException);
if (snpReq == null || !snpReq.requestId().equals(id)) {
@@ -952,16 +1261,15 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/**
* Stores snapshot metadata.
*
- * @param snpReq Snapshot operation request containing snapshot meta.
+ * @param meta Metadata to store.
* @param smf File to store.
*/
- private void storeSnapshotMeta(SnapshotOperationRequest snpReq, File smf)
- throws IgniteCheckedException, IOException {
+ public <M extends Serializable> void storeSnapshotMeta(M meta, File smf) {
if (smf.exists())
throw new IgniteException("Snapshot metafile must not exist: " + smf.getAbsolutePath());
try (OutputStream out = Files.newOutputStream(smf.toPath())) {
- byte[] bytes = U.marshal(marsh, snpReq.meta());
+ byte[] bytes = U.marshal(marsh, meta);
int blockSize = SNAPSHOT_LIMITED_TRANSFER_BLOCK_SIZE_BYTES;
for (int off = 0; off < bytes.length; off += blockSize) {
@@ -972,6 +1280,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
out.write(bytes, off, len);
}
}
+ catch (IOException | IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
}
/**
@@ -1031,32 +1342,48 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
private IgniteInternalFuture<SnapshotOperationResponse> initLocalSnapshotEndStage(SnapshotOperationRequest req) {
SnapshotOperationRequest snpReq = clusterSnpReq;
- if (snpReq == null || !F.eq(req.requestId(), snpReq.requestId()) || cctx.kernalContext().clientNode())
+ if (snpReq == null || !F.eq(req.requestId(), snpReq.requestId()))
return new GridFinishedFuture<>();
- try {
- if (req.error() != null) {
- snpReq.error(req.error());
+ IgniteInternalFuture<?> prepFut = req.incremental() ? wrapMsgsFut : new GridFinishedFuture<>();
- deleteSnapshot(snapshotLocalDir(req.snapshotName(), req.snapshotPath()), pdsSettings.folderName());
- }
- else if (!F.isEmpty(req.warnings())) {
- // Pass the warnings further to the next stage for the case when snapshot started from not coordinator.
- if (!isLocalNodeCoordinator(cctx.discovery()))
- snpReq.warnings(req.warnings());
+ return prepFut.chain(r -> {
+ if (cctx.kernalContext().clientNode())
+ return null;
+
+ try {
+ if (req.error() != null) {
+ snpReq.error(req.error());
- snpReq.meta().warnings(Collections.unmodifiableList(req.warnings()));
+ if (req.incremental())
+ U.delete(incrementalSnapshotLocalDir(req.snapshotName(), req.snapshotPath(), req.incrementIndex()));
+ else
+ deleteSnapshot(snapshotLocalDir(req.snapshotName(), req.snapshotPath()), pdsSettings.folderName());
+ }
+ else if (!F.isEmpty(req.warnings())) {
+ // Pass the warnings further to the next stage for the case when snapshot started from not coordinator.
+ if (!isLocalNodeCoordinator(cctx.discovery()))
+ snpReq.warnings(req.warnings());
- storeWarnings(snpReq);
- }
+ snpReq.meta().warnings(Collections.unmodifiableList(req.warnings()));
- removeLastMetaStorageKey();
- }
- catch (Exception e) {
- return new GridFinishedFuture<>(e);
- }
+ storeWarnings(snpReq);
+ }
+
+ removeLastMetaStorageKey();
+
+ if (req.error() == null) {
+ Collection<Integer> grpIds = req.groups().stream().map(CU::cacheId).collect(Collectors.toList());
- return new GridFinishedFuture<>(new SnapshotOperationResponse());
+ enableIncrementalSnapshotsCreation(grpIds);
+ }
+ }
+ catch (Exception e) {
+ throw F.wrap(e);
+ }
+
+ return new SnapshotOperationResponse();
+ }, cctx.kernalContext().pools().getSnapshotExecutorService());
}
/**
@@ -1083,7 +1410,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
File smf = new File(snpDir, snapshotMetaFileName(cctx.localNode().consistentId().toString()));
try {
- storeSnapshotMeta(snpReq, tempSmf);
+ storeSnapshotMeta(snpReq.meta(), tempSmf);
Files.move(tempSmf.toPath(), smf.toPath(), StandardCopyOption.ATOMIC_MOVE,
StandardCopyOption.REPLACE_EXISTING);
@@ -1115,6 +1442,16 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
Set<UUID> endFail = new HashSet<>(snpReq.nodes());
endFail.removeAll(res.keySet());
+ if (snpReq.incremental()) {
+ wrapMsgsFut = null;
+ markWalFut = null;
+
+ incSnpId = null;
+
+ if (clusterSnpFut != null && endFail.isEmpty() && snpReq.error() == null)
+ warnAtomicCachesInIncrementalSnapshot(snpReq.snapshotName(), snpReq.incrementIndex(), snpReq.groups());
+ }
+
clusterSnpReq = null;
synchronized (snpOpMux) {
@@ -1234,7 +1571,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/**
* @return List of all known snapshots on the local node.
*/
- public List<String> localSnapshotNames() {
+ public List<String> localSnapshotNames(@Nullable String snpPath) {
if (cctx.kernalContext().clientNode())
throw new UnsupportedOperationException("Client nodes can not perform this operation.");
@@ -1242,12 +1579,41 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
return Collections.emptyList();
synchronized (snpOpMux) {
- return Arrays.stream(locSnpDir.listFiles(File::isDirectory))
+ File[] dirs = (snpPath == null ? locSnpDir : new File(snpPath)).listFiles(File::isDirectory);
+
+ if (dirs == null)
+ return Collections.emptyList();
+
+ return Arrays.stream(dirs)
.map(File::getName)
.collect(Collectors.toList());
}
}
+ /**
+ * @param snpName Full snapshot name.
+ * @param snpPath Snapshot path.
+ * @return Maximum existing incremental snapshot index.
+ */
+ private int maxLocalIncrementSnapshot(String snpName, @Nullable String snpPath) {
+ if (cctx.kernalContext().clientNode())
+ throw new UnsupportedOperationException("Client and daemon nodes can not perform this operation.");
+
+ synchronized (snpOpMux) {
+ File[] incDirs = incrementalSnapshotsLocalRootDir(snpName, snpPath).listFiles(File::isDirectory);
+
+ if (incDirs == null)
+ return 0;
+
+ return Arrays.stream(incDirs)
+ .map(File::getName)
+ .filter(name -> INC_SNP_NAME_PATTERN.matcher(name).matches())
+ .mapToInt(Integer::parseInt)
+ .max()
+ .orElse(0);
+ }
+ }
+
/** {@inheritDoc} */
@Override public IgniteFuture<Void> cancelSnapshot(String name) {
return new IgniteFutureImpl<>(cancelSnapshot0(name).chain(f -> null));
@@ -1375,19 +1741,33 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
}
/**
+ * Checks snapshot.
+ *
* @param name Snapshot name.
* @param snpPath Snapshot directory path.
* @return Future with the result of execution snapshot partitions verify task, which besides calculating partition
* hashes of {@link IdleVerifyResultV2} also contains the snapshot metadata distribution across the cluster.
*/
- public IgniteInternalFuture<SnapshotPartitionsVerifyTaskResult> checkSnapshot(String name,
- @Nullable String snpPath) {
+ public IgniteInternalFuture<SnapshotPartitionsVerifyTaskResult> checkSnapshot(String name, @Nullable String snpPath) {
+ return checkSnapshot(name, snpPath, -1);
+ }
+
+ /**
+ * Checks snapshot and its increments.
+ *
+ * @param name Snapshot name.
+ * @param snpPath Snapshot directory path.
+ * @param incIdx Incremental snapshot index.
+ * @return Future with the result of execution snapshot partitions verify task, which besides calculating partition
+ * hashes of {@link IdleVerifyResultV2} also contains the snapshot metadata distribution across the cluster.
+ */
+ public IgniteInternalFuture<SnapshotPartitionsVerifyTaskResult> checkSnapshot(String name, @Nullable String snpPath, int incIdx) {
A.notNullOrEmpty(name, "Snapshot name cannot be null or empty.");
A.ensure(U.alphanumericUnderscore(name), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_");
cctx.kernalContext().security().authorize(ADMIN_SNAPSHOT);
- return checkSnapshot(name, snpPath, null, false).chain(f -> {
+ return checkSnapshot(name, snpPath, null, false, incIdx).chain(f -> {
try {
return f.get();
}
@@ -1407,6 +1787,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
* @param grps Collection of cache group names to check.
* @param includeCustomHandlers {@code True} to invoke all user-defined {@link SnapshotHandlerType#RESTORE}
* handlers, otherwise only system consistency check will be performed.
+ * @param incIdx Incremental snapshot index.
* @return Future with the result of execution snapshot partitions verify task, which besides calculating partition
* hashes of {@link IdleVerifyResultV2} also contains the snapshot metadata distribution across the cluster.
*/
@@ -1414,7 +1795,8 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
String name,
@Nullable String snpPath,
@Nullable Collection<String> grps,
- boolean includeCustomHandlers
+ boolean includeCustomHandlers,
+ int incIdx
) {
A.notNullOrEmpty(name, "Snapshot name cannot be null or empty.");
A.ensure(U.alphanumericUnderscore(name), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_");
@@ -1428,15 +1810,15 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
Collection<ClusterNode> bltNodes = F.view(cctx.discovery().serverNodes(AffinityTopologyVersion.NONE),
(node) -> CU.baselineNode(node, kctx0.state().clusterState()));
- SnapshotMetadataCollectorTaskArg taskArg = new SnapshotMetadataCollectorTaskArg(name, snpPath);
+ SnapshotMetadataVerificationTaskArg taskArg = new SnapshotMetadataVerificationTaskArg(name, snpPath, incIdx);
kctx0.task().execute(
- SnapshotMetadataCollectorTask.class,
+ SnapshotMetadataVerificationTask.class,
taskArg,
options(bltNodes)
).listen(f0 -> {
- if (f0.error() == null) {
- Map<ClusterNode, List<SnapshotMetadata>> metas = f0.result();
+ if (f0.error() == null && F.isEmpty(f0.result().exceptions())) {
+ Map<ClusterNode, List<SnapshotMetadata>> metas = f0.result().meta();
Map<Integer, String> grpIds = grps == null ? Collections.emptyMap() :
grps.stream().collect(Collectors.toMap(CU::cacheId, v -> v));
@@ -1527,7 +1909,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
});
}
else {
- if (f0.error() instanceof IgniteSnapshotVerifyException)
+ if (f0.error() == null)
+ res.onDone(new IgniteSnapshotVerifyException(f0.result().exceptions()));
+ else if (f0.error() instanceof IgniteSnapshotVerifyException)
res.onDone(new SnapshotPartitionsVerifyTaskResult(null,
new IdleVerifyResultV2(((IgniteSnapshotVerifyException)f0.error()).exceptions())));
else
@@ -1558,7 +1942,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
* @param consId Node consistent id to read metadata for.
* @return Snapshot metadata instance.
*/
- public SnapshotMetadata readSnapshotMetadata(File snpDir, String consId) {
+ public SnapshotMetadata readSnapshotMetadata(File snpDir, String consId) throws IgniteCheckedException, IOException {
return readSnapshotMetadata(new File(snpDir, snapshotMetaFileName(consId)));
}
@@ -1566,26 +1950,31 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
* @param smf File denoting to snapshot metafile.
* @return Snapshot metadata instance.
*/
- private SnapshotMetadata readSnapshotMetadata(File smf) {
- if (!smf.exists())
- throw new IgniteException("Snapshot metafile cannot be read due to it doesn't exist: " + smf);
+ private SnapshotMetadata readSnapshotMetadata(File smf) throws IgniteCheckedException, IOException {
+ SnapshotMetadata meta = readFromFile(smf);
String smfName = smf.getName().substring(0, smf.getName().length() - SNAPSHOT_METAFILE_EXT.length());
- try (InputStream in = new BufferedInputStream(new FileInputStream(smf))) {
- SnapshotMetadata meta = marsh.unmarshal(in, U.resolveClassLoader(cctx.gridConfig()));
+ if (!U.maskForFileName(meta.consistentId()).equals(smfName)) {
+ throw new IgniteException(
+ "Error reading snapshot metadata [smfName=" + smfName + ", consId=" + U.maskForFileName(meta.consistentId())
+ );
+ }
- if (!U.maskForFileName(meta.consistentId()).equals(smfName)) {
- throw new IgniteException(
- "Error reading snapshot metadata [smfName=" + smfName + ", consId=" + U.maskForFileName(meta.consistentId())
- );
- }
+ return meta;
+ }
- return meta;
- }
- catch (IgniteCheckedException | IOException e) {
- throw new IgniteException("An error occurred during reading snapshot metadata file [file=" +
- smf.getAbsolutePath() + "]", e);
+ /**
+ * @param smf File to read.
+ * @return Read metadata.
+ * @param <T> Type of metadata.
+ */
+ public <T> T readFromFile(File smf) throws IgniteCheckedException, IOException {
+ if (!smf.exists())
+ throw new IgniteCheckedException("Snapshot metafile cannot be read due to it doesn't exist: " + smf);
+
+ try (InputStream in = new BufferedInputStream(Files.newInputStream(smf.toPath()))) {
+ return marsh.unmarshal(in, U.resolveClassLoader(cctx.gridConfig()));
}
}
@@ -1623,15 +2012,22 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
Map<String, SnapshotMetadata> metasMap = new HashMap<>();
SnapshotMetadata prev = null;
- for (File smf : smfs) {
- SnapshotMetadata curr = readSnapshotMetadata(smf);
+ try {
+ for (File smf : smfs) {
+ SnapshotMetadata curr = readSnapshotMetadata(smf);
- if (prev != null && !prev.sameSnapshot(curr))
- throw new IgniteException("Snapshot metadata files are from different snapshots [prev=" + prev + ", curr=" + curr);
+ if (prev != null && !prev.sameSnapshot(curr)) {
+ throw new IgniteException("Snapshot metadata files are from different snapshots " +
+ "[prev=" + prev + ", curr=" + curr + ']');
+ }
- metasMap.put(curr.consistentId(), curr);
+ metasMap.put(curr.consistentId(), curr);
- prev = curr;
+ prev = curr;
+ }
+ }
+ catch (IgniteCheckedException | IOException e) {
+ throw new IgniteException(e);
}
SnapshotMetadata currNodeSmf = metasMap.remove(cctx.localNode().consistentId().toString());
@@ -1649,9 +2045,40 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
}
}
+ /**
+ * @param snpName Snapshot name.
+ * @return Collection of incremental snapshots metafiles.
+ */
+ public Collection<IncrementalSnapshotMetadata> readIncrementalSnapshotMetadatas(String snpName) {
+ File[] incDirs = incrementalSnapshotsLocalRootDir(snpName, null)
+ .listFiles((dir, name) -> INC_SNP_NAME_PATTERN.matcher(name).matches());
+
+ if (incDirs == null)
+ return Collections.emptyList();
+
+ List<IncrementalSnapshotMetadata> metas = new ArrayList<>();
+
+ try {
+ for (File incDir: incDirs) {
+ for (File metaFile: incDir.listFiles((dir, name) -> name.endsWith(SNAPSHOT_METAFILE_EXT)))
+ metas.add(readFromFile(metaFile));
+ }
+ }
+ catch (IgniteCheckedException | IOException e) {
+ throw new IgniteException(e);
+ }
+
+ return metas;
+ }
+
/** {@inheritDoc} */
@Override public IgniteFuture<Void> createSnapshot(String name) {
- return createSnapshot(name, null);
+ return createSnapshot(name, null, false);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteFuture<Void> createIncrementalSnapshot(String name) {
+ return createSnapshot(name, null, true);
}
/**
@@ -1661,7 +2088,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
* @param snpPath Snapshot directory path.
* @return Future which will be completed when a process ends.
*/
- public IgniteFutureImpl<Void> createSnapshot(String name, @Nullable String snpPath) {
+ public IgniteFutureImpl<Void> createSnapshot(String name, @Nullable String snpPath, boolean incremental) {
A.notNullOrEmpty(name, "Snapshot name cannot be null or empty.");
A.ensure(U.alphanumericUnderscore(name), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_");
@@ -1693,12 +2120,13 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
return new IgniteSnapshotFutureImpl(cctx.kernalContext().closure()
.callAsync(
BALANCE,
- new CreateSnapshotCallable(name),
+ new CreateSnapshotCallable(name, incremental),
options(Collections.singletonList(crd)).withFailoverDisabled()
));
}
ClusterSnapshotFuture snpFut0;
+ int incIdx = -1;
synchronized (snpOpMux) {
if (clusterSnpFut != null && !clusterSnpFut.isDone()) {
@@ -1710,10 +2138,25 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
if (clusterSnpReq != null)
throw new IgniteException("Create snapshot request has been rejected. Parallel snapshot processes are not allowed.");
- if (localSnapshotNames().contains(name)) {
- throw new IgniteException(
- "Create snapshot request has been rejected. Snapshot with given name already exists on local node."
- );
+ boolean snpExists = localSnapshotNames(snpPath).contains(name);
+
+ if (!incremental && snpExists) {
+ throw new IgniteException("Create snapshot request has been rejected. " +
+ "Snapshot with given name already exists on local node.");
+ }
+
+ if (incremental) {
+ if (!cctx.gridConfig().getDataStorageConfiguration().isWalCompactionEnabled()) {
+ throw new IgniteException("Create incremental snapshot request has been rejected. " +
+ "WAL compaction must be enabled.");
+ }
+
+ if (!snpExists) {
+ throw new IgniteException("Create incremental snapshot request has been rejected. " +
+ "Base snapshot with given name doesn't exist on local node.");
+ }
+
+ incIdx = maxLocalIncrementSnapshot(name, snpPath) + 1;
}
if (isRestoring()) {
@@ -1722,10 +2165,14 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
);
}
- snpFut0 = new ClusterSnapshotFuture(UUID.randomUUID(), name);
+ snpFut0 = new ClusterSnapshotFuture(UUID.randomUUID(), name, incIdx);
clusterSnpFut = snpFut0;
- lastSeenSnpFut = snpFut0;
+
+ if (incremental)
+ lastSeenIncSnpFut = snpFut0;
+ else
+ lastSeenSnpFut = snpFut0;
}
List<String> grps = cctx.cache().persistentGroups().stream()
@@ -1747,10 +2194,21 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
Set<UUID> bltNodeIds =
new HashSet<>(F.viewReadOnly(srvNodes, F.node2id(), (node) -> CU.baselineNode(node, clusterState)));
- startSnpProc.start(snpFut0.rqId,
- new SnapshotOperationRequest(snpFut0.rqId, cctx.localNodeId(), name, snpPath, grps, bltNodeIds));
-
- String msg = "Cluster-wide snapshot operation started [snpName=" + name + ", grps=" + grps + ']';
+ startSnpProc.start(snpFut0.rqId, new SnapshotOperationRequest(
+ snpFut0.rqId,
+ cctx.localNodeId(),
+ name,
+ snpPath,
+ grps,
+ bltNodeIds,
+ incremental,
+ incIdx
+ ));
+
+ String msg =
+ "Cluster-wide snapshot operation started [snpName=" + name + ", grps=" + grps +
+ (incremental ? "" : (", incremental=true, incrementIndex=" + incIdx)) +
+ ']';
recordSnapshotEvent(name, msg, EVT_CLUSTER_SNAPSHOT_STARTED);
@@ -1764,15 +2222,58 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
U.error(log, SNAPSHOT_FAILED_MSG, e);
- lastSeenSnpFut = new ClusterSnapshotFuture(name, e);
+ ClusterSnapshotFuture errSnpFut = new ClusterSnapshotFuture(name, e);
+
+ if (incremental)
+ lastSeenIncSnpFut = errSnpFut;
+ else
+ lastSeenSnpFut = errSnpFut;
return new IgniteFinishedFutureImpl<>(e);
}
}
+ /** Writes a warning message if an incremental snapshot contains atomic caches. */
+ void warnAtomicCachesInIncrementalSnapshot(String snpName, int incIdx, Collection<String> cacheGrps) {
+ List<String> warnCaches = new ArrayList<>();
+
+ for (String cacheGrp: cacheGrps) {
+ CacheGroupContext cgctx = cctx.cache().cacheGroup(CU.cacheId(cacheGrp));
+
+ if (cgctx != null && cgctx.hasAtomicCaches()) {
+ for (GridCacheContext<?, ?> c : cgctx.caches()) {
+ CacheConfiguration<?, ?> ccfg = c.config();
+
+ if (ccfg.getAtomicityMode() == CacheAtomicityMode.ATOMIC && ccfg.getBackups() > 0)
+ warnCaches.add(ccfg.getName());
+ }
+ }
+ }
+
+ if (warnCaches.isEmpty())
+ return;
+
+ U.warn(log, "Incremental snapshot [snpName=" + snpName + ", incIdx=" + incIdx + "] contains ATOMIC caches with backups: "
+ + warnCaches + ". Please note, incremental snapshots doesn't guarantee consistency of restored atomic caches. " +
+ "It is highly recommended to verify these caches after restoring with the \"idle_verify\" command. " +
+ "If it is needed it's possible to repair inconsistent partitions with the \"consistency\" command. " +
+ "Please, check the \"Control Script\" section of Ignite docs for more information about these commands.");
+ }
+
/** {@inheritDoc} */
@Override public IgniteFuture<Void> restoreSnapshot(String name, @Nullable Collection<String> grpNames) {
- return restoreSnapshot(name, null, grpNames);
+ return restoreSnapshot(name, null, grpNames, 0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteFuture<Void> restoreSnapshot(
+ String name,
+ @Nullable Collection<String> grpNames,
+ int incIdx
+ ) {
+ A.ensure(incIdx > 0, "Incremental snapshot index must be greater than 0.");
+
+ return restoreSnapshot(name, null, grpNames, incIdx);
}
/**
@@ -1784,13 +2285,31 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
* @return Future which will be completed when restore operation finished.
*/
public IgniteFutureImpl<Void> restoreSnapshot(String name, @Nullable String snpPath, @Nullable Collection<String> grpNames) {
+ return restoreSnapshot(name, snpPath, grpNames, 0);
+ }
+
+ /**
+ * Restore cache group(s) from the snapshot.
+ *
+ * @param name Snapshot name.
+ * @param snpPath Snapshot directory path.
+ * @param grpNames Cache groups to be restored or {@code null} to restore all cache groups from the snapshot.
+ * @param incIdx Index of incremental snapshot.
+ * @return Future which will be completed when restore operation finished.
+ */
+ public IgniteFutureImpl<Void> restoreSnapshot(
+ String name,
+ @Nullable String snpPath,
+ @Nullable Collection<String> grpNames,
+ int incIdx
+ ) {
A.notNullOrEmpty(name, "Snapshot name cannot be null or empty.");
A.ensure(U.alphanumericUnderscore(name), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_");
A.ensure(grpNames == null || !grpNames.isEmpty(), "List of cache group names cannot be empty.");
cctx.kernalContext().security().authorize(ADMIN_SNAPSHOT);
- return restoreCacheGrpProc.start(name, snpPath, grpNames);
+ return restoreCacheGrpProc.start(name, snpPath, grpNames, incIdx);
}
/** {@inheritDoc} */
@@ -1813,7 +2332,11 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
String snpName = (String)metaStorage.read(SNP_RUNNING_KEY);
String snpDirName = snpName == null ? (String)metaStorage.read(SNP_RUNNING_DIR_KEY) : null;
- File snpDir = snpName != null ? snapshotLocalDir(snpName, null) : snpDirName != null ? new File(snpDirName) : null;
+ File snpDir = snpName != null
+ ? snapshotLocalDir(snpName, null)
+ : snpDirName != null
+ ? new File(snpDirName)
+ : null;
if (snpDir == null)
return;
@@ -1823,7 +2346,10 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
for (File tmp : snapshotTmpDir().listFiles())
U.delete(tmp);
- deleteSnapshot(snpDir, pdsSettings.folderName());
+ if (INC_SNP_NAME_PATTERN.matcher(snpDir.getName()).matches() && snpDir.getAbsolutePath().contains(INC_SNP_DIR))
+ U.delete(snpDir);
+ else
+ deleteSnapshot(snpDir, pdsSettings.folderName());
if (log.isInfoEnabled()) {
log.info("Previous attempt to create snapshot fail due to the local node crash. All resources " +
@@ -1848,6 +2374,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
SnapshotOperationRequest snpReq = clusterSnpReq;
+ if (snpReq.incremental())
+ return;
+
AbstractSnapshotFutureTask<?> task = locSnpTasks.get(snpReq.snapshotName());
if (task == null)
@@ -1919,7 +2448,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
for (AbstractSnapshotFutureTask<?> sctx : F.view(locSnpTasks.values(), t -> t instanceof SnapshotFutureTask)) {
Set<Integer> retain = new HashSet<>(grps);
- retain.retainAll(((SnapshotFutureTask)sctx).affectedCacheGroups());
+ retain.retainAll(sctx.affectedCacheGroups());
if (!retain.isEmpty()) {
sctx.acceptException(new IgniteCheckedException("Snapshot has been interrupted due to some of the required " +
@@ -1932,7 +2461,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
* @param consId Consistent node id.
* @return Snapshot metadata file name.
*/
- private static String snapshotMetaFileName(String consId) {
+ public static String snapshotMetaFileName(String consId) {
return U.maskForFileName(consId) + SNAPSHOT_METAFILE_EXT;
}
@@ -2079,8 +2608,10 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
}
/**
+ * Registers a local snapshot task.
+ *
* @param task Snapshot operation task to be executed.
- * @return Snapshot operation task which should be registered on checkpoint to run.
+ * @return Snapshot operation task which should be registered.
*/
private AbstractSnapshotFutureTask<?> registerTask(String rqId, AbstractSnapshotFutureTask<?> task) {
if (!busyLock.enterBusy()) {
@@ -2116,7 +2647,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
}
/** @return Current snapshot task. */
- private SnapshotFutureTask currentSnapshotTask() {
+ <T extends AbstractSnapshotFutureTask<?>> T currentSnapshotTask(Class<T> snpTaskCls) {
SnapshotOperationRequest req = clusterSnpReq;
if (req == null)
@@ -2124,10 +2655,10 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
AbstractSnapshotFutureTask<?> task = locSnpTasks.get(req.snapshotName());
- if (!(task instanceof SnapshotFutureTask))
+ if (task == null || task.getClass() != snpTaskCls)
return null;
- return (SnapshotFutureTask)task;
+ return (T)task;
}
/**
@@ -2163,6 +2694,24 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
rqId);
}
+ /** @param snpLocDir Snapshot local directory. */
+ public void writeSnapshotDirectoryToMetastorage(File snpLocDir) {
+ cctx.database().checkpointReadLock();
+
+ try {
+ assert metaStorage != null && metaStorage.read(SNP_RUNNING_DIR_KEY) == null :
+ "The previous snapshot hasn't been completed correctly";
+
+ metaStorage.write(SNP_RUNNING_DIR_KEY, snpLocDir.getAbsolutePath());
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ finally {
+ cctx.database().checkpointReadUnlock();
+ }
+ }
+
/** Snapshot finished successfully or already restored. Key can be removed. */
private void removeLastMetaStorageKey() throws IgniteCheckedException {
cctx.database().checkpointReadLock();
@@ -2176,6 +2725,56 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
}
}
+ /**
+ * Disables creation of incremental snapshots for the given cache group.
+ *
+ * @param metaStorage External metastorage, useful if the flag is set before cluster activation.
+ * @param grpId Group ID.
+ */
+ public void disableIncrementalSnapshotsCreation(MetaStorage metaStorage, int grpId) {
+ cctx.database().checkpointReadLock();
+
+ try {
+ metaStorage.write(incrementalSnapshotCreationDisabledKey(grpId), true);
+ }
+ catch (IgniteCheckedException e) {
+ log.error("Failed to disable incremental snapshot creation for the cache group: " + grpId, e);
+ }
+ finally {
+ cctx.database().checkpointReadUnlock();
+ }
+ }
+
+ /**
+ * Enables creation of incremental snapshots for the given cache groups.
+ *
+ * @param grpIds Group IDs.
+ */
+ private void enableIncrementalSnapshotsCreation(Collection<Integer> grpIds) {
+ cctx.database().checkpointReadLock();
+
+ try {
+ for (int g: grpIds)
+ metaStorage.remove(incrementalSnapshotCreationDisabledKey(g));
+ }
+ catch (IgniteCheckedException e) {
+ log.error("Failed to allow incremental snapshot creation for group: " + grpIds, e);
+ }
+ finally {
+ cctx.database().checkpointReadUnlock();
+ }
+ }
+
+ /**
+ * Convert cache group ID to key for {@link #INC_SNP_DISABLED_KEY_PREFIX} metastorage records.
+ *
+ * @param grpId Group ID.
+ * @return Key.
+ */
+ public static String incrementalSnapshotCreationDisabledKey(int grpId) {
+ return INC_SNP_DISABLED_KEY_PREFIX + grpId;
+ }
+
/**
* @param snpName Snapshot name event related to.
* @param msg Event message.
@@ -2328,14 +2927,116 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
}
/**
- * @param meta Snapshot metadata.
- * @return Snapshot view.
+ * Checks that incremental snapshot can be created for given full snapshot and current cluster state.
+ *
+ * @param name Full snapshot name.
+ * @param snpPath Snapshot path.
+ * @param meta Full snapshot metadata.
*/
- private SnapshotView snapshotViewSupplier(SnapshotMetadata meta) {
- List<File> dirs = snapshotCacheDirectories(meta.snapshotName(), null, meta.folderName(), name -> true);
- Collection<String> cacheGrps = F.viewReadOnly(dirs, FilePageStoreManager::cacheGroupName);
+ private void checkIncrementalCanBeCreated(
+ String name,
+ @Nullable String snpPath,
+ SnapshotMetadata meta
+ ) throws IgniteCheckedException, IOException {
+ File snpDir = snapshotLocalDir(name, snpPath);
+
+ IgniteWriteAheadLogManager wal = cctx.wal();
+
+ if (wal == null)
+ throw new IgniteCheckedException("Create incremental snapshot request has been rejected. WAL must be enabled.");
+
+ File archiveDir = wal.archiveDir();
+
+ if (archiveDir == null)
+ throw new IgniteCheckedException("Create incremental snapshot request has been rejected. WAL archive must be enabled.");
+
+ ensureHardLinkAvailable(archiveDir.toPath(), snpDir.toPath());
+
+ Set<String> aliveNodesConsIds = cctx.discovery().aliveServerNodes()
+ .stream()
+ .map(node -> node.consistentId().toString())
+ .collect(Collectors.toSet());
- return new SnapshotView(meta.snapshotName(), meta.consistentId(), F.concat(meta.baselineNodes(), ","), F.concat(cacheGrps, ","));
+ for (String consId : meta.baselineNodes()) {
+ if (!aliveNodesConsIds.contains(consId)) {
+ throw new IgniteCheckedException("Create incremental snapshot request has been rejected. " +
+ "Node from full snapshot offline [consistentId=" + consId + ']');
+ }
+ }
+
+ File rootSnpCachesDir = new File(snpDir, databaseRelativePath(meta.folderName()));
+
+ for (int grpId : meta.cacheGroupIds()) {
+ if (grpId == METASTORAGE_CACHE_ID)
+ continue;
+
+ if (metaStorage.read(incrementalSnapshotCreationDisabledKey(grpId)) != null) {
+ throw new IgniteCheckedException("Create incremental snapshot request has been rejected. " +
+ "WAL was disabled since previous snapshot for cache group [groupId=" + grpId + ']');
+ }
+
+ CacheGroupContext gctx = cctx.kernalContext().cache().cacheGroup(grpId);
+
+ if (gctx == null) {
+ throw new IgniteCheckedException("Create incremental snapshot request has been rejected. " +
+ "Cache group destroyed [groupId=" + grpId + ']');
+ }
+
+ if (gctx.config().isEncryptionEnabled()) {
+ throw new IgniteCheckedException("Create incremental snapshot request has been rejected. " +
+ "Encrypted cache groups not supported [groupId=" + grpId + ']');
+ }
+
+ List<File> snpCacheDir =
+ cacheDirectories(rootSnpCachesDir, grpName -> gctx.cacheOrGroupName().equals(grpName));
+
+ if (snpCacheDir.isEmpty()) {
+ throw new IgniteCheckedException("Create incremental snapshot request has been rejected. " +
+ "Cache group directory not found [groupId=" + grpId + ']');
+ }
+
+ assert snpCacheDir.size() == 1 : "Single snapshot cache directory must be found";
+
+ for (File snpDataFile : FilePageStoreManager.cacheDataFiles(snpCacheDir.get(0))) {
+ StoredCacheData snpCacheData = GridLocalConfigManager.readCacheData(
+ snpDataFile,
+ MarshallerUtils.jdkMarshaller(cctx.kernalContext().igniteInstanceName()),
+ cctx.kernalContext().config()
+ );
+
+ byte[] snpCacheDataBytes = Files.readAllBytes(snpDataFile.toPath());
+
+ File nodeDataFile = new File(snpDataFile.getAbsolutePath().replace(
+ rootSnpCachesDir.getAbsolutePath(),
+ pdsSettings.persistentStoreNodePath().getAbsolutePath()
+ ));
+
+ if (!nodeDataFile.exists()) {
+ throw new IgniteCheckedException("Create incremental snapshot request has been rejected. " +
+ "Cache destroyed [cacheId=" + snpCacheData.cacheId() +
+ ", cacheName=" + snpCacheData.config().getName() + ']');
+ }
+
+ byte[] nodeCacheDataBytes = Files.readAllBytes(nodeDataFile.toPath());
+
+ if (!Arrays.equals(snpCacheDataBytes, nodeCacheDataBytes)) {
+ throw new IgniteCheckedException(
+ cacheChangedException(snpCacheData.cacheId(), snpCacheData.config().getName())
+ );
+ }
+ }
+ }
+ }
+
+ /**
+ * Throw cache changed exception.
+ *
+ * @param cacheId Cache id.
+ * @param name Cache name.
+ */
+ public static String cacheChangedException(int cacheId, String name) {
+ return "Create incremental snapshot request has been rejected. " +
+ "Cache changed [cacheId=" + cacheId + ", cacheName=" + name + ']';
}
/** @return Snapshot handlers. */
@@ -2343,6 +3044,11 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
return handlers;
}
+ /** @return Current incremental snapshot ID. */
+ public @Nullable UUID incrementalSnapshotId() {
+ return incSnpId;
+ }
+
/** Snapshot operation handlers. */
protected static class SnapshotHandlers {
/** Snapshot operation handlers. */
@@ -3320,22 +4026,14 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
"[snpName=" + snpLocDir.getName() + ", absPath=" + dbDir.getAbsolutePath() + ']');
}
- cctx.database().checkpointReadLock();
+ writeSnapshotDirectoryToMetastorage(snpLocDir);
try {
- assert metaStorage != null && metaStorage.read(SNP_RUNNING_DIR_KEY) == null :
- "The previous snapshot hasn't been completed correctly";
-
- metaStorage.write(SNP_RUNNING_DIR_KEY, snpLocDir.getAbsolutePath());
-
U.ensureDirectory(dbDir, "snapshot work directory for a local snapshot sender", log);
}
catch (IgniteCheckedException e) {
throw new IgniteException(e);
}
- finally {
- cctx.database().checkpointReadUnlock();
- }
}
/** {@inheritDoc} */
@@ -3657,6 +4355,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/** Serial version UID. */
private static final long serialVersionUID = 0L;
+ /** */
+ private final boolean needExchange;
+
/**
* @param processId Unique process id.
* @param req Snapshot initial request.
@@ -3665,12 +4366,14 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
UUID processId,
SnapshotOperationRequest req
) {
- super(processId, START_SNAPSHOT, req);
+ super(processId, START_SNAPSHOT, req, req.incremental());
+
+ needExchange = !req.incremental();
}
/** {@inheritDoc} */
@Override public boolean needExchange() {
- return true;
+ return needExchange;
}
/** {@inheritDoc} */
@@ -3695,6 +4398,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/** Snapshot start time. */
final long startTime;
+ /** Incremental snapshot index. */
+ final @Nullable Integer incIdx;
+
/** Snapshot finish time. */
volatile long endTime;
@@ -3711,6 +4417,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
name = "";
startTime = 0;
endTime = 0;
+ incIdx = null;
}
/**
@@ -3724,15 +4431,17 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
startTime = U.currentTimeMillis();
endTime = 0;
rqId = null;
+ incIdx = null;
}
/**
* @param rqId Unique snapshot request id.
* @param name Snapshot name.
*/
- public ClusterSnapshotFuture(UUID rqId, String name) {
+ public ClusterSnapshotFuture(UUID rqId, String name, @Nullable Integer incIdx) {
this.rqId = rqId;
this.name = name;
+ this.incIdx = incIdx;
startTime = U.currentTimeMillis();
}
@@ -3758,6 +4467,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/** Snapshot name. */
private final String snpName;
+ /** Incremental flag. */
+ private final boolean incremental;
+
/** Auto-injected grid instance. */
@IgniteInstanceResource
private transient IgniteEx ignite;
@@ -3765,13 +4477,17 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/**
* @param snpName Snapshot name.
*/
- public CreateSnapshotCallable(String snpName) {
+ public CreateSnapshotCallable(String snpName, boolean incremental) {
this.snpName = snpName;
+ this.incremental = incremental;
}
/** {@inheritDoc} */
@Override public Void call() throws Exception {
- ignite.snapshot().createSnapshot(snpName).get();
+ if (incremental)
+ ignite.snapshot().createIncrementalSnapshot(snpName).get();
+ else
+ ignite.snapshot().createSnapshot(snpName).get();
return null;
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotAwareMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotAwareMessage.java
new file mode 100644
index 00000000000..26e58138a40
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotAwareMessage.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.cache.GridCacheMessage;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Message that holds a transaction message and incremental snapshot ID.
+ */
+public class IncrementalSnapshotAwareMessage extends GridCacheMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ public static final short TYPE_CODE = 400;
+
+ /** Original transaction message. */
+ private GridCacheMessage payload;
+
+ /** Incremental snapshot ID. */
+ private UUID id;
+
+ /** ID of the latest incremental snapshot after which this transaction committed. */
+ private @Nullable UUID txSnpId;
+
+ /** Incremental snapshot topology version. */
+ private long topVer;
+
+ /** */
+ public IncrementalSnapshotAwareMessage() {
+ }
+
+ /** */
+ public IncrementalSnapshotAwareMessage(
+ GridCacheMessage payload,
+ UUID id,
+ @Nullable UUID txSnpId,
+ long topVer
+ ) {
+ this.payload = payload;
+ this.id = id;
+ this.txSnpId = txSnpId;
+ this.topVer = topVer;
+ }
+
+ /** @return Incremental snapshot ID. */
+ public UUID id() {
+ return id;
+ }
+
+ /** ID of the latest incremental snapshot after which this transaction committed. */
+ public UUID txInrementalSnapshotId() {
+ return txSnpId;
+ }
+
+ /** */
+ public GridCacheMessage payload() {
+ return payload;
+ }
+
+ /** @return Incremental snapshot topology version. */
+ public long snapshotTopologyVersion() {
+ return topVer;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {
+ payload.prepareMarshal(ctx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr) throws IgniteCheckedException {
+ payload.finishUnmarshal(ctx, ldr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!super.writeTo(buf, writer))
+ return false;
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 3:
+ if (!writer.writeUuid("id", id))
+ return false;
+
+ writer.incrementState();
+
+ case 4:
+ if (!writer.writeMessage("payload", payload))
+ return false;
+
+ writer.incrementState();
+
+ case 5:
+ if (!writer.writeLong("topVer", topVer))
+ return false;
+
+ writer.incrementState();
+
+ case 6:
+ if (!writer.writeUuid("txSnpId", txSnpId))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ if (!super.readFrom(buf, reader))
+ return false;
+
+ switch (reader.state()) {
+ case 3:
+ id = reader.readUuid("id");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 4:
+ payload = reader.readMessage("payload");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 5:
+ topVer = reader.readLong("topVer");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 6:
+ txSnpId = reader.readUuid("txSnpId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(IncrementalSnapshotAwareMessage.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return TYPE_CODE;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 7;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean addDeploymentInfo() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int handlerId() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean cacheGroupMessage() {
+ return false;
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotFutureTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotFutureTask.java
new file mode 100644
index 00000000000..e9be26974f7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotFutureTask.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.BiConsumer;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.MarshallerContextImpl;
+import org.apache.ignite.internal.binary.BinaryUtils;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotFinishRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.ClusterSnapshotRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.MarshallerContextImpl.mappingFileStoreWorkDir;
+import static org.apache.ignite.internal.binary.BinaryUtils.METADATA_FILE_SUFFIX;
+import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.binaryWorkDir;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.incrementalSnapshotWalsDir;
+
+/** */
+class IncrementalSnapshotFutureTask extends AbstractSnapshotFutureTask<Void> implements BiConsumer<String, File> {
+ /** Index of incremental snapshot. */
+ private final int incIdx;
+
+ /** Snapshot path. */
+ private final @Nullable String snpPath;
+
+ /** Metadata of the full snapshot. */
+ private final Set<Integer> affectedCacheGrps;
+
+ /**
+ * Pointer to the previous snapshot record.
+ * In case first increment snapshot will point to the {@link ClusterSnapshotRecord}.
+ * For second and subsequent incements on the previous {@link IncrementalSnapshotFinishRecord}.
+ */
+ private final WALPointer lowPtr;
+
+ /** Future that completes with WAL pointer to {@link IncrementalSnapshotFinishRecord}. */
+ private final IgniteInternalFuture<WALPointer> highPtrFut;
+
+ /** */
+ public IncrementalSnapshotFutureTask(
+ GridCacheSharedContext<?, ?> cctx,
+ UUID srcNodeId,
+ UUID reqNodeId,
+ SnapshotMetadata meta,
+ @Nullable String snpPath,
+ int incIdx,
+ File tmpWorkDir,
+ FileIOFactory ioFactory,
+ WALPointer lowPtr,
+ IgniteInternalFuture<WALPointer> highPtrFut
+ ) {
+ super(
+ cctx,
+ srcNodeId,
+ reqNodeId,
+ meta.snapshotName(),
+ tmpWorkDir,
+ ioFactory,
+ new SnapshotSender(
+ cctx.logger(IncrementalSnapshotFutureTask.class),
+ cctx.kernalContext().pools().getSnapshotExecutorService()
+ ) {
+ @Override protected void init(int partsCnt) {
+ // No-op.
+ }
+
+ @Override protected void sendPart0(File part, String cacheDirName, GroupPartitionId pair, Long length) {
+ // No-op.
+ }
+
+ @Override protected void sendDelta0(File delta, String cacheDirName, GroupPartitionId pair) {
+ // No-op.
+ }
+ },
+ null
+ );
+
+ this.incIdx = incIdx;
+ this.snpPath = snpPath;
+ this.affectedCacheGrps = new HashSet<>(meta.cacheGroupIds());
+ this.lowPtr = lowPtr;
+ this.highPtrFut = highPtrFut;
+
+ cctx.cache().configManager().addConfigurationChangeListener(this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Set<Integer> affectedCacheGroups() {
+ return affectedCacheGrps;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean start() {
+ try {
+ File incSnpDir = cctx.snapshotMgr().incrementalSnapshotLocalDir(snpName, snpPath, incIdx);
+
+ if (!incSnpDir.mkdirs() && !incSnpDir.exists()) {
+ onDone(new IgniteException("Can't create snapshot directory [dir=" + incSnpDir.getAbsolutePath() + ']'));
+
+ return false;
+ }
+
+ highPtrFut.chain(fut -> {
+ if (fut.error() != null) {
+ onDone(fut.error());
+
+ return null;
+ }
+
+ try {
+ String folderName = cctx.kernalContext().pdsFolderResolver().resolveFolders().folderName();
+
+ copyWal(incrementalSnapshotWalsDir(incSnpDir, folderName), fut.result());
+
+ copyFiles(
+ MarshallerContextImpl.mappingFileStoreWorkDir(cctx.gridConfig().getWorkDirectory()),
+ mappingFileStoreWorkDir(incSnpDir.getAbsolutePath()),
+ BinaryUtils::notTmpFile
+ );
+
+ copyFiles(
+ binaryWorkDir(cctx.gridConfig().getWorkDirectory(), folderName),
+ binaryWorkDir(incSnpDir.getAbsolutePath(), folderName),
+ file -> file.getName().endsWith(METADATA_FILE_SUFFIX)
+ );
+
+ onDone();
+ }
+ catch (Throwable e) {
+ onDone(e);
+ }
+
+ return null;
+ }, cctx.kernalContext().pools().getSnapshotExecutorService());
+
+ return true;
+ }
+ finally {
+ cctx.cache().configManager().removeConfigurationChangeListener(this);
+ }
+ }
+
+ /**
+ * Copies WAL segments to the incremental snapshot directory.
+ *
+ * @param incSnpWalDir Incremental snapshot directory.
+ * @param highPtr High WAL pointer to copy.
+ * @throws IgniteInterruptedCheckedException If failed.
+ * @throws IOException If failed.
+ */
+ private void copyWal(File incSnpWalDir, WALPointer highPtr) throws IgniteInterruptedCheckedException, IOException {
+ // First increment must include low segment, because full snapshot knows nothing about WAL.
+ // All other begins from the next segment because lowPtr already saved inside previous increment.
+ long lowIdx = lowPtr.index() + (incIdx == 1 ? 0 : 1);
+ long highIdx = highPtr.index();
+
+ assert cctx.gridConfig().getDataStorageConfiguration().isWalCompactionEnabled() : "WAL Compaction must be enabled";
+ assert lowIdx <= highIdx;
+
+ if (log.isInfoEnabled())
+ log.info("Waiting for WAL segments compression [lowIdx=" + lowIdx + ", highIdx=" + highIdx + ']');
+
+ cctx.wal().awaitCompacted(highPtr.index());
+
+ if (log.isInfoEnabled())
+ log.info("Linking WAL segments into incremental snapshot [lowIdx=" + lowIdx + ", " + "highIdx=" + highIdx + ']');
+
+ if (!incSnpWalDir.mkdirs() && !incSnpWalDir.exists())
+ throw new IgniteException("Failed to create snapshot WAL directory [idx=" + incSnpWalDir + ']');
+
+ for (; lowIdx <= highIdx; lowIdx++) {
+ File seg = cctx.wal().compactedSegment(lowIdx);
+
+ if (!seg.exists())
+ throw new IgniteException("WAL segment not found in archive [idx=" + lowIdx + ']');
+
+ Path segLink = incSnpWalDir.toPath().resolve(seg.getName());
+
+ if (log.isDebugEnabled())
+ log.debug("Creaing segment link [path=" + segLink.toAbsolutePath() + ']');
+
+ Files.createLink(segLink, seg.toPath());
+ }
+ }
+
+ /**
+ * Copy files {@code fromDir} to {@code toDir}.
+ *
+ * @param fromDir From directory.
+ * @param toDir To directory.
+ * @param filter File filter.
+ */
+ private void copyFiles(File fromDir, File toDir, FileFilter filter) throws IOException {
+ assert fromDir.exists() && fromDir.isDirectory();
+
+ if (!toDir.isDirectory() && !toDir.mkdirs() && !toDir.exists())
+ throw new IgniteException("Target directory can't be created [target=" + toDir.getAbsolutePath() + ']');
+
+ for (File from : fromDir.listFiles(filter)) {
+ try {
+ Files.copy(from.toPath(), new File(toDir, from.getName()).toPath());
+ }
+ catch (FileAlreadyExistsException e) {
+ // Skip, file might exist in case the marshaller directory is shared between multiple Ignite nodes.
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void acceptException(Throwable th) {
+ cctx.cache().configManager().removeConfigurationChangeListener(this);
+
+ onDone(th);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void accept(String name, File file) {
+ onDone(new IgniteException(IgniteSnapshotManager.cacheChangedException(CU.cacheId(name), name)));
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotMarkWalFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotMarkWalFuture.java
new file mode 100644
index 00000000000..f0574f9f89a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotMarkWalFuture.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotFinishRecord;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotStartRecord;
+import org.apache.ignite.internal.pagemem.wal.record.RolloverType;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx.FinalizationStatus.RECOVERY_FINISH;
+import static org.apache.ignite.transactions.TransactionState.ACTIVE;
+import static org.apache.ignite.transactions.TransactionState.COMMITTED;
+import static org.apache.ignite.transactions.TransactionState.MARKED_ROLLBACK;
+import static org.apache.ignite.transactions.TransactionState.ROLLED_BACK;
+
+/**
+ * Incremental snapshot based on the Consistent Cut distributed algorithm that goal is to define a specific set of transactions.
+ * It guarantees that every transaction was included into the set on one node also included into the same set on every other
+ * node participated in the transaction. It means that Ignite nodes can safely recover themselves to the consistent state by applying
+ * transactions from this set.
+ */
+class IncrementalSnapshotMarkWalFuture extends GridFutureAdapter<WALPointer> {
+ /** Cache context. */
+ private final GridCacheSharedContext<?, ?> cctx;
+
+ /** Incremental snapshot ID. */
+ private final UUID id;
+
+ /** Incremental snapshot topology version. */
+ private final long topVer;
+
+ /** Logger. */
+ private final IgniteLogger log;
+
+ /** Set of checked transactions included into the incremental snapshot. */
+ @GridToStringInclude
+ private final Set<GridCacheVersion> included = ConcurrentHashMap.newKeySet();
+
+ /** Set of checked transactions excluded from the incremental snapshot. */
+ @GridToStringInclude
+ private final Set<GridCacheVersion> excluded = ConcurrentHashMap.newKeySet();
+
+ /** Collection of transactions removed from {@link IgniteTxManager#activeTransactions()}. */
+ private final Set<IgniteInternalFuture<IgniteInternalTx>> removedFromActive = ConcurrentHashMap.newKeySet();
+
+ /** */
+ IncrementalSnapshotMarkWalFuture(GridCacheSharedContext<?, ?> cctx, UUID id, long topVer) {
+ this.cctx = cctx;
+ this.id = id;
+ this.topVer = topVer;
+
+ log = cctx.logger(IncrementalSnapshotMarkWalFuture.class);
+
+ cctx.tm().onCommitCallback((tx) -> removedFromActive.add(tx.finishFuture()));
+ }
+
+ /** Inits the future: it prepares list of active transactions to check whether they belong the incremental snapshot. */
+ void init() {
+ try {
+ cctx.wal().log(new IncrementalSnapshotStartRecord(id));
+
+ GridCompoundFuture<Boolean, Boolean> checkFut = new GridCompoundFuture<>(CU.boolReducer());
+
+ Iterator<IgniteInternalFuture<IgniteInternalTx>> finFutIt = cctx.tm().activeTransactions().stream()
+ .filter(tx -> tx.state() != ACTIVE)
+ .map(IgniteInternalTx::finishFuture)
+ .iterator();
+
+ // Invoke sequentially over two iterators:
+ // 1. Iterators are weakly consistent.
+ // 2. We need a guarantee to handle `removedFromActive` after `activeTxs` to avoid missed transactions.
+ checkTransactions(finFutIt, checkFut);
+ checkTransactions(removedFromActive.iterator(), checkFut);
+
+ checkFut.markInitialized();
+
+ checkFut.listen(finish -> {
+ if (isDone())
+ return;
+
+ if (Boolean.FALSE.equals(finish.result())) {
+ onDone(new IgniteCheckedException("Incremental snapshot is inconsistent [id=" + id + ']'));
+
+ return;
+ }
+
+ WALPointer ptr = null;
+ IgniteCheckedException err = null;
+
+ cctx.database().checkpointReadLock();
+
+ try {
+ ptr = cctx.wal().log(new IncrementalSnapshotFinishRecord(id, included, excluded), RolloverType.CURRENT_SEGMENT);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to write IncrementalSnapshotFinishRecord to WAL for [id=" + id + ']', e);
+
+ err = e;
+ }
+ finally {
+ cctx.database().checkpointReadUnlock();
+ }
+
+ onDone(ptr, err);
+ });
+ }
+ catch (IgniteCheckedException e) {
+ onDone(e);
+
+ U.error(log, "Failed to init incremental snapshot [id=" + id + ']', e);
+ }
+ finally {
+ cctx.tm().onCommitCallback(null);
+ }
+ }
+
+ /**
+ * Checks active transactions - decides whether they belong to the incremental snapshot after they finished.
+ *
+ * @param activeTxFinFuts Active transactions to check.
+ * @param checkFut Compound future that reduces finishes of the checking transactions.
+ */
+ private void checkTransactions(
+ Iterator<IgniteInternalFuture<IgniteInternalTx>> activeTxFinFuts,
+ GridCompoundFuture<Boolean, Boolean> checkFut
+ ) {
+ while (activeTxFinFuts.hasNext()) {
+ IgniteInternalFuture<Boolean> txCheckFut = activeTxFinFuts.next().chain(txFut -> {
+ // The `txFut` never fails and always returns IgniteInternalTx.
+ IgniteInternalTx tx = txFut.result();
+
+ // ROLLED_BACK transactions don't change data then don't care.
+ if (tx.state() == ROLLED_BACK || tx.state() == MARKED_ROLLBACK)
+ return true;
+
+ if (tx.state() != COMMITTED) {
+ U.warn(log, "Incremental snapshot is inconsistent due to transaction is in unexpected state " +
+ "[id=" + id + ", tx=" + tx + ']');
+
+ return false;
+ }
+
+ if (tx.finalizationStatus() == RECOVERY_FINISH) {
+ U.warn(log, "Incremental snapshot is inconsistent due to transaction committed after recovery process " +
+ "[id=" + id + ", tx=" + tx + ']');
+
+ return false;
+ }
+
+ AffinityTopologyVersion txTopVer = tx.topologyVersionSnapshot();
+
+ if (txTopVer == null) {
+ U.warn(log, "Incremental snapshot is inconsistent due to transaction doesn't map to topology " +
+ "[id=" + id + ", tx=" + tx + ']');
+
+ return false;
+ }
+
+ if (id.equals(tx.incrementalSnapshotId()) || txTopVer.topologyVersion() > topVer)
+ excluded.add(tx.nearXidVersion());
+ else
+ included.add(tx.nearXidVersion());
+
+ return true;
+ });
+
+ checkFut.add(txCheckFut);
+ }
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotMetadata.java
new file mode 100644
index 00000000000..c01bfca45af
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotMetadata.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.io.Serializable;
+import java.util.UUID;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotFinishRecord;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Incremental snapshot metadata file.
+ *
+ * @see IgniteSnapshotManager#createIncrementalSnapshot(String)
+ */
+public class IncrementalSnapshotMetadata implements Serializable {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 0L;
+
+ /** Unique snapshot request id. */
+ @GridToStringInclude
+ private final UUID rqId;
+
+ /** Snapshot name. */
+ @GridToStringInclude
+ private final String snpName;
+
+ /** Increment index. */
+ @GridToStringInclude
+ private final int incIdx;
+
+ /** Consistent id of a node to which this metadata relates. */
+ @GridToStringInclude
+ private final String consId;
+
+ /** Name of the folder that contains snapshot data. */
+ private final String folderName;
+
+ /** WAL pointer to {@link IncrementalSnapshotFinishRecord}. */
+ private final WALPointer incSnpRec;
+
+ /**
+ * @param rqId Unique request id.
+ * @param snpName Snapshot name.
+ * @param incIdx Incremental snapshot index.
+ * @param consId Consistent id of a node to which this metadata relates.
+ * @param folderName Directory name which stores the data files.
+ * @param incSnpRec Pointer to {@link IncrementalSnapshotFinishRecord}.
+ */
+ public IncrementalSnapshotMetadata(
+ UUID rqId,
+ String snpName,
+ int incIdx,
+ String consId,
+ String folderName,
+ WALPointer incSnpRec
+ ) {
+ this.rqId = rqId;
+ this.snpName = snpName;
+ this.incIdx = incIdx;
+ this.consId = consId;
+ this.folderName = folderName;
+ this.incSnpRec = incSnpRec;
+ }
+
+ /** @return Snapshot request ID. */
+ public UUID requestId() {
+ return rqId;
+ }
+
+ /** @return Snapshot name. */
+ public String snapshotName() {
+ return snpName;
+ }
+
+ /** @return Consistent ID of a node to which this metadata relates. */
+ public String consistentId() {
+ return consId;
+ }
+
+ /** @return Pointer to {@link IncrementalSnapshotFinishRecord}. */
+ public WALPointer incrementalSnapshotPointer() {
+ return incSnpRec;
+ }
+
+ /** @return Incremental snapshot index. */
+ public int incrementIndex() {
+ return incIdx;
+ }
+
+ /** @return Name of the folder that contains snapshot data. */
+ public String folderName() {
+ return folderName;
+ }
+
+ /**
+ * Checks that incremental snapshot is based on this full snapshot.
+ *
+ * @param meta Full snapshot metadata to verify.
+ * @return {@code false} if given metadata doesn't match full snapshot.
+ */
+ public boolean matchBaseSnapshot(SnapshotMetadata meta) {
+ return snpName.equals(meta.snapshotName()) && consId.equals(meta.consistentId());
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IncrementalSnapshotMetadata.class, this);
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFutureTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFutureTask.java
index 7f2477d3331..7fc31dbdfb7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFutureTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFutureTask.java
@@ -67,6 +67,7 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStor
import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
import org.apache.ignite.internal.processors.compress.CompressionProcessor;
import org.apache.ignite.internal.processors.marshaller.MappedName;
@@ -98,7 +99,7 @@ import static org.apache.ignite.internal.processors.cache.persistence.snapshot.I
* If partitions for particular cache group are not provided that they will be collected and added
* on checkpoint under the write-lock.
*/
-class SnapshotFutureTask extends AbstractSnapshotFutureTask<Set<GroupPartitionId>> implements CheckpointListener {
+class SnapshotFutureTask extends AbstractSnapshotFutureTask<SnapshotFutureTaskResult> implements CheckpointListener {
/** File page store manager for accessing cache group associated files. */
private final FilePageStoreManager pageStore;
@@ -144,6 +145,9 @@ class SnapshotFutureTask extends AbstractSnapshotFutureTask<Set<GroupPartitionId
/** Future which will be completed when task requested to be closed. Will be executed on system pool. */
private volatile CompletableFuture<Void> closeFut;
+ /** Pointer to {@link ClusterSnapshotRecord}. */
+ private volatile @Nullable WALPointer snpPtr;
+
/** Flag indicates that task already scheduled on checkpoint. */
private final AtomicBoolean started = new AtomicBoolean();
@@ -193,13 +197,6 @@ class SnapshotFutureTask extends AbstractSnapshotFutureTask<Set<GroupPartitionId
this.locBuff = locBuff;
}
- /**
- * @return Set of cache groups included into snapshot operation.
- */
- public Set<Integer> affectedCacheGroups() {
- return parts.keySet();
- }
-
/**
* @param th An exception which occurred during snapshot processing.
*/
@@ -217,7 +214,7 @@ class SnapshotFutureTask extends AbstractSnapshotFutureTask<Set<GroupPartitionId
}
/** {@inheritDoc} */
- @Override public boolean onDone(@Nullable Set<GroupPartitionId> res, @Nullable Throwable err) {
+ @Override public boolean onDone(@Nullable SnapshotFutureTaskResult res, @Nullable Throwable err) {
for (PageStoreSerialWriter writer : partDeltaWriters.values())
U.closeQuiet(writer);
@@ -357,7 +354,7 @@ class SnapshotFutureTask extends AbstractSnapshotFutureTask<Set<GroupPartitionId
// 1. Checkpoint holds write acquire lock and Snapshot holds PME. Then there are not any concurrent updates.
// 2. This record is written before the related CheckpointRecord, and is flushed with CheckpointRecord or instead it.
if (cctx.wal() != null) {
- cctx.wal().log(new ClusterSnapshotRecord(snpName));
+ snpPtr = cctx.wal().log(new ClusterSnapshotRecord(snpName));
ctx.walFlush(true);
}
@@ -634,7 +631,7 @@ class SnapshotFutureTask extends AbstractSnapshotFutureTask<Set<GroupPartitionId
.map(Map.Entry::getKey)
.collect(Collectors.toSet());
- closeFut = CompletableFuture.runAsync(() -> onDone(taken, err0),
+ closeFut = CompletableFuture.runAsync(() -> onDone(new SnapshotFutureTaskResult(taken, snpPtr), err0),
cctx.kernalContext().pools().getSystemExecutorService());
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFutureTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFutureTaskResult.java
new file mode 100644
index 00000000000..0f2acac059e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFutureTaskResult.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.util.Collections;
+import java.util.Set;
+import org.apache.ignite.internal.pagemem.wal.record.delta.ClusterSnapshotRecord;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Represents result of {@link SnapshotFutureTask}.
+ */
+class SnapshotFutureTaskResult {
+ /** Partitions for which snapshot was created. */
+ private final Set<GroupPartitionId> parts;
+
+ /** Pointer to {@link ClusterSnapshotRecord} in WAL. */
+ private final @Nullable WALPointer snpPtr;
+
+ /** */
+ SnapshotFutureTaskResult(Set<GroupPartitionId> parts, @Nullable WALPointer snpPtr) {
+ this.parts = Collections.unmodifiableSet(parts);
+ this.snpPtr = snpPtr;
+ }
+
+ /** */
+ Set<GroupPartitionId> parts() {
+ return parts;
+ }
+
+ /** */
+ @Nullable WALPointer snapshotPointer() {
+ return snpPtr;
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java
index afc72f98f46..958dc9211fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.cache.persistence.snapshot;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -144,7 +145,7 @@ public class SnapshotHandlerRestoreTask extends AbstractSnapshotVerificationTask
return snpMgr.handlers().invokeAll(SnapshotHandlerType.RESTORE,
new SnapshotHandlerContext(meta, grps, ignite.localNode(), snpDir, false));
}
- catch (IgniteCheckedException e) {
+ catch (IgniteCheckedException | IOException e) {
throw new IgniteException(e);
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java
index 713ab2e46c0..d1998ddb751 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java
@@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.metric.MetricRegistry;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.mxbean.SnapshotMXBean;
+import org.apache.ignite.spi.metric.IntMetric;
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotRestoreProcess.SNAPSHOT_RESTORE_METRICS;
@@ -50,7 +51,15 @@ public class SnapshotMXBeanImpl implements SnapshotMXBean {
/** {@inheritDoc} */
@Override public void createSnapshot(String snpName, String snpPath) {
- IgniteFuture<Void> fut = mgr.createSnapshot(snpName, F.isEmpty(snpPath) ? null : snpPath);
+ IgniteFuture<Void> fut = mgr.createSnapshot(snpName, F.isEmpty(snpPath) ? null : snpPath, false);
+
+ if (fut.isDone())
+ fut.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void createIncrementalSnapshot(String fullSnapshot, String fullSnapshotPath) {
+ IgniteFuture<Void> fut = mgr.createSnapshot(fullSnapshot, F.isEmpty(fullSnapshotPath) ? null : fullSnapshotPath, true);
if (fut.isDone())
fut.get();
@@ -77,6 +86,17 @@ public class SnapshotMXBeanImpl implements SnapshotMXBean {
fut.get();
}
+ /** {@inheritDoc} */
+ @Override public void restoreSnapshot(String name, String path, String grpNames, int incIdx) {
+ Set<String> grpNamesSet = F.isEmpty(grpNames) ? null :
+ Arrays.stream(grpNames.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toSet());
+
+ IgniteFuture<Void> fut = mgr.restoreSnapshot(name, F.isEmpty(path) ? null : path, grpNamesSet, incIdx);
+
+ if (fut.isDone())
+ fut.get();
+ }
+
/** {@inheritDoc} */
@Override public void cancelSnapshotRestore(String name) {
mgr.cancelSnapshotRestore(name).get();
@@ -88,6 +108,8 @@ public class SnapshotMXBeanImpl implements SnapshotMXBean {
if (req != null) {
return "Create snapshot operation is in progress [name=" + req.snapshotName() +
+ ", incremental=" + req.incremental() +
+ (req.incremental() ? (", incrementIndex=" + req.incrementIndex()) : "") +
", id=" + req.requestId() + ']';
}
@@ -95,9 +117,13 @@ public class SnapshotMXBeanImpl implements SnapshotMXBean {
MetricRegistry mreg = metricMgr.registry(SNAPSHOT_RESTORE_METRICS);
String name = mreg.findMetric("snapshotName").getAsString();
+ int incIdx = mreg.<IntMetric>findMetric("incrementIndex").value();
String id = mreg.findMetric("requestId").getAsString();
- return "Restore snapshot operation is in progress [name=" + name + ", id=" + id + ']';
+ boolean incremental = incIdx > 0;
+
+ return "Restore snapshot operation is in progress [name=" + name + ", incremental=" + incremental +
+ (incremental ? ", incrementIndex=" + incIdx : "") + ", id=" + id + ']';
}
return "There is no create or restore snapshot operation in progress.";
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadata.java
index fd68e9a3219..cd0517ff790 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadata.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadata.java
@@ -30,7 +30,9 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
+import org.apache.ignite.internal.pagemem.wal.record.delta.ClusterSnapshotRecord;
import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -48,9 +50,11 @@ public class SnapshotMetadata implements Serializable {
private final UUID rqId;
/** Snapshot name. */
+ @GridToStringInclude
private final String snpName;
/** Consistent id of a node to which this metadata relates. */
+ @GridToStringInclude
private final String consId;
/**
@@ -70,6 +74,9 @@ public class SnapshotMetadata implements Serializable {
@GridToStringInclude
private final Set<String> bltNodes;
+ /** WAL pointer to {@link ClusterSnapshotRecord} if exists. */
+ private final @Nullable WALPointer snpRecPtr;
+
/**
* Map of cache group partitions from which snapshot has been taken on the local node. This map can be empty
* since for instance, due to the node filter there is no cache data on node.
@@ -92,12 +99,14 @@ public class SnapshotMetadata implements Serializable {
private boolean hasComprGrps;
/**
- * F@param snpName Snapshot name.
+ * @param rqId Unique request id.
+ * @param snpName Snapshot name.
* @param consId Consistent id of a node to which this metadata relates.
* @param folderName Directory name which stores the data files.
* @param pageSize Page size of stored snapshot data.
* @param grpIds The list of cache groups ids which were included into snapshot.
* @param bltNodes The set of affected by snapshot baseline nodes.
+ * @param snpRecPtr WAL pointer to {@link ClusterSnapshotRecord} if exists.
* @param masterKeyDigest Master key digest for encrypted caches.
*/
public SnapshotMetadata(
@@ -110,6 +119,7 @@ public class SnapshotMetadata implements Serializable {
Collection<Integer> compGrpIds,
Set<String> bltNodes,
Set<GroupPartitionId> pairs,
+ @Nullable WALPointer snpRecPtr,
@Nullable byte[] masterKeyDigest
) {
this.rqId = rqId;
@@ -119,6 +129,7 @@ public class SnapshotMetadata implements Serializable {
this.pageSize = pageSize;
this.grpIds = grpIds;
this.bltNodes = bltNodes;
+ this.snpRecPtr = snpRecPtr;
this.masterKeyDigest = masterKeyDigest;
if (!F.isEmpty(compGrpIds)) {
@@ -199,6 +210,13 @@ public class SnapshotMetadata implements Serializable {
return hasComprGrps;
}
+ /**
+ * @return WAL pointer to {@link ClusterSnapshotRecord} if exists.
+ */
+ public @Nullable WALPointer snapshotRecordPointer() {
+ return snpRecPtr;
+ }
+
/** Save the state of this <tt>HashMap</tt> partitions and cache groups to a stream. */
private void writeObject(java.io.ObjectOutputStream s)
throws java.io.IOException {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java
deleted file mode 100644
index 76fb7b31750..00000000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence.snapshot;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.compute.ComputeJob;
-import org.apache.ignite.compute.ComputeJobAdapter;
-import org.apache.ignite.compute.ComputeJobResult;
-import org.apache.ignite.compute.ComputeJobResultPolicy;
-import org.apache.ignite.compute.ComputeTaskAdapter;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.task.GridInternal;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-/** Snapshot task to collect snapshot metadata from the baseline nodes for given snapshot name. */
-@GridInternal
-public class SnapshotMetadataCollectorTask
- extends ComputeTaskAdapter<SnapshotMetadataCollectorTaskArg, Map<ClusterNode, List<SnapshotMetadata>>> {
- /** Serial version uid. */
- private static final long serialVersionUID = 0L;
-
- /** {@inheritDoc} */
- @Override public @NotNull Map<? extends ComputeJob, ClusterNode> map(
- List<ClusterNode> subgrid,
- SnapshotMetadataCollectorTaskArg arg
- ) throws IgniteException {
- Map<ComputeJob, ClusterNode> map = U.newHashMap(subgrid.size());
-
- for (ClusterNode node : subgrid) {
- map.put(new ComputeJobAdapter() {
- @IgniteInstanceResource
- private transient IgniteEx ignite;
-
- @Override public List<SnapshotMetadata> execute() throws IgniteException {
- return ignite.context().cache().context().snapshotMgr()
- .readSnapshotMetadatas(arg.snapshotName(), arg.snapshotPath());
- }
- }, node);
- }
-
- return map;
- }
-
- /** {@inheritDoc} */
- @Override public @Nullable Map<ClusterNode, List<SnapshotMetadata>> reduce(
- List<ComputeJobResult> results
- ) throws IgniteException {
- Map<ClusterNode, List<SnapshotMetadata>> reduceRes = new HashMap<>();
- Map<ClusterNode, Exception> exs = new HashMap<>();
-
- SnapshotMetadata first = null;
-
- for (ComputeJobResult res: results) {
- if (res.getException() != null) {
- exs.put(res.getNode(), res.getException());
-
- continue;
- }
-
- List<SnapshotMetadata> metas = res.getData();
-
- for (SnapshotMetadata meta : metas) {
- if (first == null)
- first = meta;
-
- if (!first.sameSnapshot(meta)) {
- exs.put(res.getNode(),
- new IgniteException("An error occurred during comparing snapshot metadata from cluster nodes " +
- "[first=" + first + ", meta=" + meta + ", nodeId=" + res.getNode().id() + ']'));
-
- continue;
- }
-
- reduceRes.computeIfAbsent(res.getNode(), n -> new ArrayList<>())
- .add(meta);
- }
- }
-
- if (exs.isEmpty())
- return reduceRes;
- else
- throw new IgniteSnapshotVerifyException(exs);
- }
-
- /** {@inheritDoc} */
- @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) throws IgniteException {
- // Handle all exceptions during the `reduce` operation.
- return ComputeJobResultPolicy.WAIT;
- }
-}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java
new file mode 100644
index 00000000000..1695ad692ef
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
+import org.apache.ignite.compute.ComputeTaskAdapter;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.incrementalSnapshotWalsDir;
+
+/** Snapshot task to verify snapshot metadata on the baseline nodes for given snapshot name. */
+@GridInternal
+public class SnapshotMetadataVerificationTask
+ extends ComputeTaskAdapter<SnapshotMetadataVerificationTaskArg, SnapshotMetadataVerificationTaskResult> {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public @NotNull Map<? extends ComputeJob, ClusterNode> map(
+ List<ClusterNode> subgrid,
+ SnapshotMetadataVerificationTaskArg arg
+ ) throws IgniteException {
+ Map<ComputeJob, ClusterNode> map = U.newHashMap(subgrid.size());
+
+ for (ClusterNode node : subgrid)
+ map.put(new MetadataVerificationJob(arg), node);
+
+ return map;
+ }
+
+ /** Job that verifies snapshot on an Ignite node. */
+ private static class MetadataVerificationJob extends ComputeJobAdapter {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ @IgniteInstanceResource
+ private transient IgniteEx ignite;
+
+ /** */
+ @LoggerResource
+ private transient IgniteLogger log;
+
+ /** */
+ private final SnapshotMetadataVerificationTaskArg arg;
+
+ /** */
+ public MetadataVerificationJob(SnapshotMetadataVerificationTaskArg arg) {
+ this.arg = arg;
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<SnapshotMetadata> execute() throws IgniteException {
+ IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr();
+
+ List<SnapshotMetadata> snpMeta = snpMgr.readSnapshotMetadatas(arg.snapshotName(), arg.snapshotPath());
+
+ if (arg.incrementIndex() > 0) {
+ List<SnapshotMetadata> metas = snpMeta.stream()
+ .filter(m -> m.consistentId().equals(ignite.localNode().consistentId()))
+ .collect(Collectors.toList());
+
+ if (metas.size() != 1) {
+ throw new IgniteException("Failed to find snapshot metafile [metas=" + metas +
+ ", snpName=" + arg.snapshotName() + ", snpPath=" + arg.snapshotPath() + ']');
+ }
+
+ checkIncrementalSnapshots(metas.get(0), arg);
+ }
+
+ return snpMeta;
+ }
+
+ /** Checks that all incremental snapshots are present, contain correct metafile and WAL segments. */
+ public void checkIncrementalSnapshots(SnapshotMetadata fullMeta, SnapshotMetadataVerificationTaskArg arg) {
+ try {
+ IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr();
+
+ // Incremental snapshot must contain ClusterSnapshotRecord.
+ long startSeg = fullMeta.snapshotRecordPointer().index();
+
+ for (int inc = 1; inc <= arg.incrementIndex(); inc++) {
+ File incSnpDir = snpMgr.incrementalSnapshotLocalDir(arg.snapshotName(), arg.snapshotPath(), inc);
+
+ if (!incSnpDir.exists()) {
+ throw new IgniteException("No incremental snapshot found " +
+ "[snpName=" + arg.snapshotName() + ", snpPath=" + arg.snapshotPath() + ", incrementIndex=" + inc + ']');
+ }
+
+ String metaFileName = IgniteSnapshotManager.snapshotMetaFileName(ignite.localNode().consistentId().toString());
+
+ File metafile = incSnpDir.toPath().resolve(metaFileName).toFile();
+
+ IncrementalSnapshotMetadata incMeta = snpMgr.readFromFile(metafile);
+
+ if (!incMeta.matchBaseSnapshot(fullMeta)) {
+ throw new IgniteException("Incremental snapshot doesn't match full snapshot " +
+ "[incMeta=" + incMeta + ", fullMeta=" + fullMeta + ']');
+ }
+
+ if (incMeta.incrementIndex() != inc) {
+ throw new IgniteException(
+ "Incremental snapshot meta has wrong index [expectedIdx=" + inc + ", meta=" + incMeta + ']');
+ }
+
+ checkWalSegments(incMeta, startSeg, incrementalSnapshotWalsDir(incSnpDir, incMeta.folderName()));
+
+ // Incremental snapshots must not cross each other.
+ startSeg = incMeta.incrementalSnapshotPointer().index() + 1;
+ }
+ }
+ catch (IgniteCheckedException | IOException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** Check that incremental snapshot contains all required WAL segments. Throws {@link IgniteException} in case of any errors. */
+ private void checkWalSegments(IncrementalSnapshotMetadata meta, long startWalSeg, File incSnpWalDir) {
+ IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
+
+ List<FileDescriptor> walSeg = factory.resolveWalFiles(
+ new IgniteWalIteratorFactory.IteratorParametersBuilder()
+ .filesOrDirs(incSnpWalDir.listFiles(file ->
+ FileWriteAheadLogManager.WAL_SEGMENT_FILE_COMPACTED_PATTERN.matcher(file.getName()).matches())));
+
+ if (walSeg.isEmpty())
+ throw new IgniteException("No WAL segments found for incremental snapshot [dir=" + incSnpWalDir + ']');
+
+ long actFirstSeg = walSeg.get(0).idx();
+
+ if (actFirstSeg != startWalSeg) {
+ throw new IgniteException("Missed WAL segment [expectFirstSegment=" + startWalSeg
+ + ", actualFirstSegment=" + actFirstSeg + ", meta=" + meta + ']');
+ }
+
+ long expLastSeg = meta.incrementalSnapshotPointer().index();
+ long actLastSeg = walSeg.get(walSeg.size() - 1).idx();
+
+ if (actLastSeg != expLastSeg) {
+ throw new IgniteException("Missed WAL segment [expectLastSegment=" + startWalSeg
+ + ", actualLastSegment=" + actFirstSeg + ", meta=" + meta + ']');
+ }
+
+ List<?> walSegGaps = factory.hasGaps(walSeg);
+
+ if (!walSegGaps.isEmpty())
+ throw new IgniteException("Missed WAL segments [misses=" + walSegGaps + ", meta=" + meta + ']');
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public @Nullable SnapshotMetadataVerificationTaskResult reduce(List<ComputeJobResult> results) throws IgniteException {
+ Map<ClusterNode, List<SnapshotMetadata>> reduceRes = new HashMap<>();
+ Map<ClusterNode, Exception> exs = new HashMap<>();
+
+ SnapshotMetadata first = null;
+
+ for (ComputeJobResult res: results) {
+ if (res.getException() != null) {
+ exs.put(res.getNode(), res.getException());
+
+ continue;
+ }
+
+ List<SnapshotMetadata> metas = res.getData();
+
+ for (SnapshotMetadata meta : metas) {
+ if (first == null)
+ first = meta;
+
+ if (!first.sameSnapshot(meta)) {
+ exs.put(res.getNode(),
+ new IgniteException("An error occurred during comparing snapshot metadata from cluster nodes " +
+ "[first=" + first + ", meta=" + meta + ", nodeId=" + res.getNode().id() + ']'));
+
+ continue;
+ }
+
+ reduceRes.computeIfAbsent(res.getNode(), n -> new ArrayList<>())
+ .add(meta);
+ }
+ }
+
+ return new SnapshotMetadataVerificationTaskResult(reduceRes, exs);
+ }
+
+ /** {@inheritDoc} */
+ @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) throws IgniteException {
+ // Handle all exceptions during the `reduce` operation.
+ return ComputeJobResultPolicy.WAIT;
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTaskArg.java
similarity index 78%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTaskArg.java
index 70d416428fd..9fa3d55a51f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTaskArg.java
@@ -25,9 +25,9 @@ import org.apache.ignite.internal.visor.VisorDataTransferObject;
import org.jetbrains.annotations.Nullable;
/**
- * Input parameters for checking snapshot partitions consistency task.
+ * Input parameters for checking snapshot metadata.
*/
-public class SnapshotMetadataCollectorTaskArg extends VisorDataTransferObject {
+public class SnapshotMetadataVerificationTaskArg extends VisorDataTransferObject {
/** Serial version UID. */
private static final long serialVersionUID = 0L;
@@ -37,8 +37,11 @@ public class SnapshotMetadataCollectorTaskArg extends VisorDataTransferObject {
/** Snapshot directory path. */
private String snpPath;
+ /** Incremental snapshot index. */
+ private int incIdx;
+
/** Default constructor. */
- public SnapshotMetadataCollectorTaskArg() {
+ public SnapshotMetadataVerificationTaskArg() {
// No-op.
}
@@ -46,9 +49,10 @@ public class SnapshotMetadataCollectorTaskArg extends VisorDataTransferObject {
* @param snpName Snapshot name.
* @param snpPath Snapshot directory path.
*/
- public SnapshotMetadataCollectorTaskArg(String snpName, @Nullable String snpPath) {
+ public SnapshotMetadataVerificationTaskArg(String snpName, @Nullable String snpPath, int incIdx) {
this.snpName = snpName;
this.snpPath = snpPath;
+ this.incIdx = incIdx;
}
/**
@@ -65,20 +69,29 @@ public class SnapshotMetadataCollectorTaskArg extends VisorDataTransferObject {
return snpPath;
}
+ /**
+ * @return Incremental snapshot index.
+ */
+ public int incrementIndex() {
+ return incIdx;
+ }
+
/** {@inheritDoc} */
@Override protected void writeExternalData(ObjectOutput out) throws IOException {
U.writeString(out, snpName);
U.writeString(out, snpPath);
+ out.writeInt(incIdx);
}
/** {@inheritDoc} */
@Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
snpName = U.readString(in);
snpPath = U.readString(in);
+ incIdx = in.readInt();
}
/** {@inheritDoc} */
@Override public String toString() {
- return S.toString(SnapshotMetadataCollectorTaskArg.class, this);
+ return S.toString(SnapshotMetadataVerificationTaskArg.class, this);
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTaskResult.java
new file mode 100644
index 00000000000..212f615ce1a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTaskResult.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.dto.IgniteDataTransferObject;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/** */
+public class SnapshotMetadataVerificationTaskResult extends IgniteDataTransferObject {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Full snapshot metadata. */
+ private Map<ClusterNode, List<SnapshotMetadata>> meta;
+
+ /** Errors happened during snapshot metadata verification. */
+ private Map<ClusterNode, Exception> exceptions;
+
+ /** */
+ public SnapshotMetadataVerificationTaskResult(
+ Map<ClusterNode, List<SnapshotMetadata>> meta,
+ Map<ClusterNode, Exception> exceptions
+ ) {
+ this.meta = Collections.unmodifiableMap(meta);
+ this.exceptions = Collections.unmodifiableMap(exceptions);
+ }
+
+ /** */
+ public SnapshotMetadataVerificationTaskResult() {
+ }
+
+ /** @return Errors happened during snapshot metadata verification. */
+ public Map<ClusterNode, Exception> exceptions() {
+ return Collections.unmodifiableMap(exceptions);
+ }
+
+ /** @return Full snapshot metadata. */
+ public Map<ClusterNode, List<SnapshotMetadata>> meta() {
+ return Collections.unmodifiableMap(meta);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+ U.writeMap(out, meta);
+ U.writeMap(out, exceptions);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+ meta = U.readMap(in);
+ exceptions = U.readMap(in);
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java
index 552105c4dbd..d04f0206831 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java
@@ -79,6 +79,12 @@ public class SnapshotOperationRequest implements Serializable {
/** Operation start time. */
private final long startTime;
+ /** If {@code true} then incremental snapshot requested. */
+ private final boolean incremental;
+
+ /** Index of incremental snapshot. */
+ private final int incIdx;
+
/**
* @param reqId Request ID.
* @param opNodeId Operational node ID.
@@ -86,6 +92,8 @@ public class SnapshotOperationRequest implements Serializable {
* @param snpPath Snapshot directory path.
* @param grps List of cache group names.
* @param nodes Baseline node IDs that must be alive to complete the operation.
+ * @param incremental {@code True} if incremental snapshot requested.
+ * @param incIdx Incremental snapshot index.
*/
public SnapshotOperationRequest(
UUID reqId,
@@ -93,7 +101,9 @@ public class SnapshotOperationRequest implements Serializable {
String snpName,
String snpPath,
@Nullable Collection<String> grps,
- Set<UUID> nodes
+ Set<UUID> nodes,
+ boolean incremental,
+ int incIdx
) {
this.reqId = reqId;
this.opNodeId = opNodeId;
@@ -101,6 +111,8 @@ public class SnapshotOperationRequest implements Serializable {
this.grps = grps;
this.nodes = nodes;
this.snpPath = snpPath;
+ this.incremental = incremental;
+ this.incIdx = incIdx;
startTime = U.currentTimeMillis();
}
@@ -160,6 +172,16 @@ public class SnapshotOperationRequest implements Serializable {
this.err = err;
}
+ /** @return {@code True} if incremental snapshot requested. */
+ public boolean incremental() {
+ return incremental;
+ }
+
+ /** @return Incremental index. */
+ public int incrementIndex() {
+ return incIdx;
+ }
+
/** @return Start time. */
public long startTime() {
return startTime;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java
index d083e9afd1d..243ecca85a5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.cache.persistence.snapshot;
import java.io.File;
+import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -119,7 +120,7 @@ public class SnapshotPartitionsVerifyTask extends AbstractSnapshotVerificationTa
return new SnapshotPartitionsVerifyHandler(cctx)
.invoke(new SnapshotHandlerContext(meta, rqGrps, ignite.localNode(), snpDir, false));
}
- catch (IgniteCheckedException e) {
+ catch (IgniteCheckedException | IOException e) {
throw new IgniteException(e);
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
index ca09ee198e7..1067a9faf99 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
@@ -26,8 +26,10 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -41,6 +43,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.LongAdder;
import java.util.function.BiPredicate;
import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
@@ -64,16 +67,33 @@ import org.apache.ignite.internal.binary.BinaryUtils;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.managers.discovery.DiscoCache;
import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotFinishRecord;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotStartRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.ClusterSnapshotRecord;
import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.GridLocalConfigManager;
import org.apache.ignite.internal.processors.cache.StoredCacheData;
import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.persistence.CacheStripedExecutor;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.file.FileVersionCheckingFactory;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.ClusterSnapshotFuture;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
import org.apache.ignite.internal.processors.compress.CompressionProcessor;
import org.apache.ignite.internal.processors.metric.MetricRegistry;
@@ -86,7 +106,9 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.lang.IgniteUuid;
import org.jetbrains.annotations.Nullable;
@@ -94,6 +116,10 @@ import static java.util.Optional.ofNullable;
import static org.apache.ignite.internal.IgniteFeatures.SNAPSHOT_RESTORE_CACHE_GROUP;
import static org.apache.ignite.internal.MarshallerContextImpl.mappingFileStoreWorkDir;
import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CLUSTER_SNAPSHOT;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_RECORD_V2;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.INCREMENTAL_SNAPSHOT_FINISH_RECORD;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.INCREMENTAL_SNAPSHOT_START_RECORD;
import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.binaryWorkDir;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_GRP_DIR_PREFIX;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheGroupName;
@@ -101,10 +127,14 @@ import static org.apache.ignite.internal.processors.cache.persistence.file.FileP
import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_NAME;
import static org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId.getTypeByPartId;
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.databaseRelativePath;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.incrementalSnapshotWalsDir;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER;
import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PRELOAD;
import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE;
import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK;
import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_START;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_STOP;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_INCREMENTAL_SNAPSHOT_START;
/**
* Distributed process to restore cache group from the snapshot.
@@ -137,6 +167,12 @@ public class SnapshotRestoreProcess {
/** Cache group restore cache start phase. */
private final DistributedProcess<UUID, Boolean> cacheStartProc;
+ /** Cache group restore cache stop phase. */
+ private final DistributedProcess<UUID, Boolean> cacheStopProc;
+
+ /** Incremental snapshot restore phase. */
+ private final DistributedProcess<UUID, Boolean> incSnpRestoreProc;
+
/** Cache group restore rollback phase. */
private final DistributedProcess<UUID, Boolean> rollbackRestoreProc;
@@ -175,6 +211,12 @@ public class SnapshotRestoreProcess {
cacheStartProc = new DistributedProcess<>(
ctx, RESTORE_CACHE_GROUP_SNAPSHOT_START, this::cacheStart, this::finishCacheStart);
+ cacheStopProc = new DistributedProcess<>(
+ ctx, RESTORE_CACHE_GROUP_SNAPSHOT_STOP, this::cacheStop, this::finishCacheStop);
+
+ incSnpRestoreProc = new DistributedProcess<>(
+ ctx, RESTORE_INCREMENTAL_SNAPSHOT_START, this::incrementalSnapshotRestore, this::finishIncrementalSnapshotRestore);
+
rollbackRestoreProc = new DistributedProcess<>(
ctx, RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK, this::rollback, this::finishRollback);
}
@@ -211,6 +253,8 @@ public class SnapshotRestoreProcess {
"The system time when the restore operation of a cluster snapshot on this node ended.");
mreg.register("snapshotName", () -> lastOpCtx.snpName, String.class,
"The snapshot name of the last running cluster snapshot restore operation on this node.");
+ mreg.register("incrementIndex", () -> lastOpCtx.incIdx,
+ "The index of incremental snapshot of the last snapshot restore operation on this node.");
mreg.register("requestId", () -> Optional.ofNullable(lastOpCtx.reqId).map(UUID::toString).orElse(""),
String.class, "The request ID of the last running cluster snapshot restore operation on this node.");
mreg.register("error", () -> Optional.ofNullable(lastOpCtx.err.get()).map(Throwable::toString).orElse(""),
@@ -219,6 +263,12 @@ public class SnapshotRestoreProcess {
"The total number of partitions to be restored on this node.");
mreg.register("processedPartitions", () -> lastOpCtx.processedParts.get(),
"The number of processed partitions on this node.");
+ mreg.register("totalWalSegments", () -> lastOpCtx.totalWalSegments,
+ "The total number of WAL segments in the incremental snapshot to be restored on this node.");
+ mreg.register("processedWalSegments", () -> lastOpCtx.processedWalSegments,
+ "The number of processed WAL segments in the incremental snapshot on this node.");
+ mreg.register("processedWalEntries", () -> Optional.ofNullable(lastOpCtx.processedWalEntries).map(LongAdder::sum).orElse(-1L),
+ "The number of processed entries from incremental snapshot on this node.");
}
/**
@@ -227,9 +277,10 @@ public class SnapshotRestoreProcess {
* @param snpName Snapshot name.
* @param snpPath Snapshot directory path.
* @param cacheGrpNames Cache groups to be restored or {@code null} to restore all cache groups from the snapshot.
+ * @param incIdx Index of incremental snapshot.
* @return Future that will be completed when the restore operation is complete and the cache groups are started.
*/
- public IgniteFutureImpl<Void> start(String snpName, @Nullable String snpPath, @Nullable Collection<String> cacheGrpNames) {
+ public IgniteFutureImpl<Void> start(String snpName, @Nullable String snpPath, @Nullable Collection<String> cacheGrpNames, int incIdx) {
IgniteSnapshotManager snpMgr = ctx.cache().context().snapshotMgr();
ClusterSnapshotFuture fut0;
@@ -255,7 +306,7 @@ public class SnapshotRestoreProcess {
if (restoringSnapshotName() != null)
throw new IgniteException(OP_REJECT_MSG + "The previous snapshot restore operation was not completed.");
- fut = new ClusterSnapshotFuture(UUID.randomUUID(), snpName);
+ fut = new ClusterSnapshotFuture(UUID.randomUUID(), snpName, incIdx);
fut0 = fut;
}
@@ -288,14 +339,14 @@ public class SnapshotRestoreProcess {
});
String msg = "Cluster-wide snapshot restore operation started [reqId=" + fut0.rqId + ", snpName=" + snpName +
- (cacheGrpNames == null ? "" : ", caches=" + cacheGrpNames) + ']';
+ (cacheGrpNames == null ? "" : ", caches=" + cacheGrpNames) + (incIdx > 0 ? ", incrementIndex=" + incIdx : "") + ']';
if (log.isInfoEnabled())
log.info(msg);
snpMgr.recordSnapshotEvent(snpName, msg, EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED);
- snpMgr.checkSnapshot(snpName, snpPath, cacheGrpNames, true).listen(f -> {
+ snpMgr.checkSnapshot(snpName, snpPath, cacheGrpNames, true, incIdx).listen(f -> {
if (f.error() != null) {
finishProcess(fut0.rqId, f.error());
@@ -350,7 +401,15 @@ public class SnapshotRestoreProcess {
Collection<UUID> bltNodes = F.viewReadOnly(ctx.discovery().discoCache().aliveBaselineNodes(), F.node2id());
SnapshotOperationRequest req = new SnapshotOperationRequest(
- fut0.rqId, F.first(dataNodes), snpName, snpPath, cacheGrpNames, new HashSet<>(bltNodes));
+ fut0.rqId,
+ F.first(dataNodes),
+ snpName,
+ snpPath,
+ cacheGrpNames,
+ new HashSet<>(bltNodes),
+ false,
+ incIdx
+ );
prepareRestoreProc.start(req.requestId(), req);
});
@@ -914,11 +973,15 @@ public class SnapshotRestoreProcess {
try {
SnapshotMetadata meta = F.first(opCtx0.metasPerNode.get(opCtx0.opNodeId));
- File binDir = binaryWorkDir(snpDir.getAbsolutePath(), meta.folderName());
+ File dir = opCtx0.incIdx > 0 ?
+ ctx.cache().context().snapshotMgr()
+ .incrementalSnapshotLocalDir(opCtx0.snpName, opCtx0.snpPath, opCtx0.incIdx)
+ : snpDir;
- ctx.cacheObjects().updateMetadata(binDir, opCtx0.stopChecker);
+ File binDir = binaryWorkDir(dir.getAbsolutePath(), meta.folderName());
+ File marshallerDir = mappingFileStoreWorkDir(dir.getAbsolutePath());
- File marshallerDir = mappingFileStoreWorkDir(snpDir.getAbsolutePath());
+ ctx.cacheObjects().updateMetadata(binDir, opCtx0.stopChecker);
restoreMappings(marshallerDir, opCtx0.stopChecker);
}
@@ -1242,6 +1305,13 @@ public class SnapshotRestoreProcess {
orElse(checkNodeLeft(opCtx0.nodes(), res.keySet()));
if (failure == null) {
+ if (opCtx0.incIdx > 0) {
+ if (U.isLocalNodeCoordinator(ctx.discovery()))
+ incSnpRestoreProc.start(reqId, reqId);
+
+ return;
+ }
+
finishProcess(reqId, null);
return;
@@ -1253,6 +1323,423 @@ public class SnapshotRestoreProcess {
rollbackRestoreProc.start(reqId, reqId);
}
+ /**
+ * @param reqId Request ID.
+ * @return Result future.
+ */
+ private IgniteInternalFuture<Boolean> cacheStop(UUID reqId) {
+ if (!U.isLocalNodeCoordinator(ctx.discovery()))
+ return new GridFinishedFuture<>();
+
+ SnapshotRestoreContext opCtx0 = opCtx;
+
+ Collection<String> stopCaches = opCtx0.cfgs.values()
+ .stream()
+ .map(c -> c.config().getName())
+ .collect(Collectors.toSet());
+
+ if (log.isInfoEnabled())
+ log.info("Stopping caches [reqId=" + opCtx0.reqId + ", caches=" + stopCaches + ']');
+
+ // Skip deleting cache files as they will be removed during rollback.
+ return ctx.cache().dynamicDestroyCaches(stopCaches, false, false)
+ .chain(fut -> {
+ if (fut.error() != null)
+ throw F.wrap(fut.error());
+ else
+ return true;
+ });
+ }
+
+ /**
+ * @param reqId Request ID.
+ * @param res Results.
+ * @param errs Errors.
+ */
+ private void finishCacheStop(UUID reqId, Map<UUID, Boolean> res, Map<UUID, Throwable> errs) {
+ if (ctx.clientNode())
+ return;
+
+ if (!errs.isEmpty()) {
+ SnapshotRestoreContext opCtx0 = opCtx;
+
+ log.error("Failed to stop caches during a snapshot rollback routine [reqId=" + opCtx0.reqId + ", err=" + errs + ']');
+ }
+
+ if (U.isLocalNodeCoordinator(ctx.discovery()))
+ rollbackRestoreProc.start(reqId, reqId);
+ }
+
+ /**
+ * Inits restoring incremental snapshot.
+ *
+ * @param reqId Request ID.
+ * @return Result future.
+ */
+ private IgniteInternalFuture<Boolean> incrementalSnapshotRestore(UUID reqId) {
+ if (ctx.clientNode())
+ return new GridFinishedFuture<>();
+
+ SnapshotRestoreContext opCtx0 = opCtx;
+
+ if (log.isInfoEnabled()) {
+ log.info("Starting incremental snapshot restore operation " +
+ "[reqId=" + opCtx0.reqId + ", snpName=" + opCtx0.snpName + ", incrementIndex=" + opCtx0.incIdx +
+ ", caches=" + F.viewReadOnly(opCtx0.cfgs, c -> c.config().getName()) + ']');
+ }
+
+ GridFutureAdapter<Boolean> res = new GridFutureAdapter<>();
+
+ ctx.pools().getSnapshotExecutorService().submit(() -> {
+ try {
+ Set<Integer> cacheIds = opCtx0.cfgs.keySet();
+
+ walEnabled(false, cacheIds);
+
+ restoreIncrementalSnapshot(opCtx0.snpName, opCtx0.snpPath, cacheIds, opCtx0.incIdx);
+
+ walEnabled(true, cacheIds);
+
+ CheckpointProgress cp = ctx.cache().context().database()
+ .forceNewCheckpoint("Incremental snapshot restored.", (fut) -> {
+ if (fut.error() != null)
+ res.onDone(fut.error());
+ else
+ res.onDone(true);
+ });
+
+ if (cp == null)
+ res.onDone(new IgniteCheckedException("Node is stopping."));
+ }
+ catch (Throwable e) {
+ res.onDone(e);
+ }
+ });
+
+ return res;
+ }
+
+ /**
+ * Restore incremental snapshot.
+ *
+ * @param snpName Base snapshot name.
+ * @param snpPath Base snapshot path.
+ * @param cacheIds Restoring cache IDs.
+ * @param incIdx Index of incremental snapshot.
+ */
+ private void restoreIncrementalSnapshot(
+ String snpName,
+ String snpPath,
+ Set<Integer> cacheIds,
+ int incIdx
+ ) throws IgniteCheckedException, IOException {
+ SnapshotRestoreContext opCtx0 = opCtx;
+
+ IncrementalSnapshotMetadata meta = ctx.cache().context().snapshotMgr()
+ .readIncrementalSnapshotMetadata(snpName, snpPath, incIdx);
+
+ File[] segments = walSegments(snpName, snpPath, incIdx, meta.folderName());
+
+ opCtx0.totalWalSegments = segments.length;
+
+ UUID incSnpId = meta.requestId();
+
+ File lastSeg = Arrays.stream(segments)
+ .map(File::toPath)
+ .max(Comparator.comparingLong(FileWriteAheadLogManager::segmentIndex))
+ .orElseThrow(() -> new IgniteCheckedException("Last WAL segment wasn't found [snpName=" + snpName + ']'))
+ .toFile();
+
+ IncrementalSnapshotFinishRecord incSnpFinRec = readFinishRecord(lastSeg, incSnpId);
+
+ if (incSnpFinRec == null) {
+ throw new IgniteCheckedException("System WAL record for incremental snapshot wasn't found " +
+ "[id=" + incSnpId + ", walSegFile=" + lastSeg + ']');
+ }
+
+ CacheStripedExecutor exec = new CacheStripedExecutor(ctx.pools().getStripedExecutorService());
+
+ long start = U.currentTimeMillis();
+
+ LongAdder applied = new LongAdder();
+
+ opCtx0.processedWalEntries = applied;
+ opCtx0.processedWalSegments = 0;
+
+ Set<WALRecord.RecordType> recTypes = new HashSet<>(F.asList(
+ CLUSTER_SNAPSHOT,
+ INCREMENTAL_SNAPSHOT_START_RECORD,
+ INCREMENTAL_SNAPSHOT_FINISH_RECORD,
+ DATA_RECORD_V2));
+
+ // Create a single WAL iterator for 2 steps: finding ClusterSnapshotRecord and applying incremental snapshots.
+ // TODO: Fix it after resolving https://issues.apache.org/jira/browse/IGNITE-18718.
+ try (WALIterator it = walIter(log, recTypes, segments)) {
+ long startIdx = -1;
+
+ // Step 1. Skips applying WAL until base snapshot record has been reached.
+ while (it.hasNext()) {
+ IgniteBiTuple<WALPointer, WALRecord> walRec = it.next();
+
+ WALRecord rec = walRec.getValue();
+
+ if (rec.type() == CLUSTER_SNAPSHOT) {
+ if (((ClusterSnapshotRecord)rec).clusterSnapshotName().equals(snpName)) {
+ startIdx = walRec.getKey().index();
+
+ break;
+ }
+ }
+ }
+
+ if (startIdx < 0) {
+ throw new IgniteCheckedException("System WAL record for full snapshot wasn't found " +
+ "[snpName=" + snpName + ", walSegFile=" + segments[0] + ']');
+ }
+
+ UUID prevIncSnpId = incIdx > 1
+ ? ctx.cache().context().snapshotMgr().readIncrementalSnapshotMetadata(snpName, snpPath, incIdx - 1).requestId()
+ : null;
+
+ IgnitePredicate<GridCacheVersion> txVerFilter = prevIncSnpId != null
+ ? txVer -> true : txVer -> !incSnpFinRec.excluded().contains(txVer);
+
+ long lastProcessedIdx = 0;
+
+ // Step 2. Apply incremental snapshots.
+ while (it.hasNext()) {
+ IgniteBiTuple<WALPointer, WALRecord> walRec = it.next();
+
+ long curIdx = walRec.getKey().index();
+
+ if (curIdx != lastProcessedIdx) {
+ opCtx0.processedWalSegments = (int)(curIdx - startIdx);
+
+ lastProcessedIdx = curIdx;
+ }
+
+ WALRecord rec = walRec.getValue();
+
+ if (rec.type() == INCREMENTAL_SNAPSHOT_START_RECORD) {
+ IncrementalSnapshotStartRecord startRec = (IncrementalSnapshotStartRecord)rec;
+
+ if (startRec.id().equals(incSnpFinRec.id()))
+ txVerFilter = v -> incSnpFinRec.included().contains(v);
+ }
+ else if (rec.type() == INCREMENTAL_SNAPSHOT_FINISH_RECORD) {
+ IncrementalSnapshotFinishRecord finRec = (IncrementalSnapshotFinishRecord)rec;
+
+ if (finRec.id().equals(prevIncSnpId))
+ txVerFilter = txVer -> !incSnpFinRec.excluded().contains(txVer);
+ }
+ else if (rec.type() == DATA_RECORD_V2) {
+ DataRecord data = (DataRecord)rec;
+
+ for (DataEntry e: data.writeEntries()) {
+ // That is OK to restore only part of transaction related to a specified cache group,
+ // because a full snapshot restoring does the same.
+ if (!cacheIds.contains(e.cacheId()) || !txVerFilter.apply(e.nearXidVersion()))
+ continue;
+
+ GridCacheContext<?, ?> cacheCtx = ctx.cache().context().cacheContext(e.cacheId());
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ctx.cache().context().database();
+
+ exec.submit(() -> {
+ try {
+ applyDataEntry(dbMgr, cacheCtx, e);
+
+ applied.increment();
+ }
+ catch (IgniteCheckedException err) {
+ U.error(log, "Failed to apply data entry, dataEntry=" + e + ", ptr=" + data.position());
+
+ exec.onError(err);
+ }
+ }, cacheCtx.groupId(), e.partitionId());
+ }
+ }
+ }
+ }
+
+ opCtx0.processedWalSegments += 1;
+
+ exec.awaitApplyComplete();
+
+ // Close partition counter gaps that can exists due to some transactions excluded from incremental snapshot.
+ for (int cacheId: cacheIds) {
+ GridCacheContext<?, ?> cacheCtx = ctx.cache().context().cacheContext(cacheId);
+
+ for (int part = 0; part < cacheCtx.topology().partitions(); part++) {
+ int partId = part;
+
+ exec.submit(() -> {
+ GridDhtLocalPartition locPart = cacheCtx.topology().localPartition(partId);
+
+ if (locPart != null)
+ locPart.finalizeUpdateCounters();
+ }, cacheCtx.groupId(), part);
+ }
+ }
+
+ exec.awaitApplyComplete();
+
+ if (log.isInfoEnabled()) {
+ log.info("Finished restore incremental snapshot [snpName=" + snpName + ", incrementIndex=" + incIdx +
+ ", id=" + incSnpId + ", updatesApplied=" + applied.longValue() + ", time=" + (U.currentTimeMillis() - start) + " ms]");
+ }
+ }
+
+ /** @return {@link IncrementalSnapshotFinishRecord} for specified snapshot, or {@code null} if not found. */
+ private @Nullable IncrementalSnapshotFinishRecord readFinishRecord(File segment, UUID incSnpId) throws IgniteCheckedException {
+ try (WALIterator it = walIter(log, Collections.singleton(INCREMENTAL_SNAPSHOT_FINISH_RECORD), segment)) {
+ while (it.hasNext()) {
+ IncrementalSnapshotFinishRecord finRec = (IncrementalSnapshotFinishRecord)it.next().getValue();
+
+ if (finRec.id().equals(incSnpId))
+ return finRec;
+ }
+ }
+
+ return null;
+ }
+
+ /** @return WAL segments to restore for specified incremental index since the base snapshot. */
+ private File[] walSegments(String snpName, String snpPath, int incIdx, String folderName) throws IgniteCheckedException {
+ File[] segments = null;
+
+ for (int i = 1; i <= incIdx; i++) {
+ File incSnpDir = ctx.cache().context().snapshotMgr().incrementalSnapshotLocalDir(snpName, snpPath, i);
+
+ if (!incSnpDir.exists())
+ throw new IgniteCheckedException("Incremental snapshot doesn't exists [dir=" + incSnpDir + ']');
+
+ File incSnpWalDir = incrementalSnapshotWalsDir(incSnpDir, folderName);
+
+ if (!incSnpWalDir.exists())
+ throw new IgniteCheckedException("Incremental snapshot WAL directory doesn't exists [dir=" + incSnpWalDir + ']');
+
+ File[] incSegs = incSnpWalDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER);
+
+ if (incSegs == null)
+ throw new IgniteCheckedException("Failed to list WAL segments from snapshot directory [dir=" + incSnpDir + ']');
+
+ if (segments == null)
+ segments = incSegs;
+ else {
+ int segLen = segments.length;
+
+ segments = Arrays.copyOf(segments, segLen + incSegs.length);
+
+ System.arraycopy(incSegs, 0, segments, segLen, incSegs.length);
+ }
+ }
+
+ if (F.isEmpty(segments)) {
+ throw new IgniteCheckedException("No WAL segments found for incremental snapshot " +
+ "[snpName=" + snpName + ", snpPath=" + snpPath + ", incrementIndex=" + incIdx + ']');
+ }
+
+ return segments;
+ }
+
+ /**
+ * Enable or disable WAL while restoring incremental snapshot.
+ *
+ * @param enabled Enable or disable WAL for caches.
+ * @param cacheIds Restoring cache IDs.
+ */
+ private void walEnabled(boolean enabled, Set<Integer> cacheIds) {
+ for (Integer cacheId: cacheIds) {
+ int grpId = ctx.cache().cacheDescriptor(cacheId).groupId();
+
+ CacheGroupContext grp = ctx.cache().cacheGroup(grpId);
+
+ assert grp != null : "cacheId=" + cacheId + " grpId=" + grpId;
+
+ grp.localWalEnabled(enabled, false);
+
+ if (!enabled)
+ ctx.cache().context().database().lastCheckpointInapplicableForWalRebalance(grpId);
+ }
+ }
+
+ /**
+ * @param dbMgr Database manager.
+ * @param cacheCtx Cache context to apply an update.
+ * @param dataEntry Data entry to apply.
+ * @throws IgniteCheckedException If failed to apply entry.
+ */
+ private void applyDataEntry(
+ GridCacheDatabaseSharedManager dbMgr,
+ GridCacheContext<?, ?> cacheCtx,
+ DataEntry dataEntry
+ ) throws IgniteCheckedException {
+ int partId = dataEntry.partitionId();
+
+ if (partId == -1) {
+ U.warn(log, "Partition isn't set for read data entry [entry=" + dataEntry + ']');
+
+ partId = cacheCtx.affinity().partition(dataEntry.key());
+ }
+
+ // Local partitions has already created after caches started.
+ GridDhtLocalPartition locPart = cacheCtx.topology().localPartition(partId);
+
+ assert locPart != null : "Missed local partition " + partId;
+
+ if (dbMgr.applyDataEntry(cacheCtx, locPart, dataEntry))
+ cacheCtx.offheap().dataStore(locPart).updateCounter(dataEntry.partitionCounter() - 1, 1);
+ }
+
+ /**
+ * @param log Ignite logger.
+ * @param types WAL record types to read.
+ * @param segments WAL segments.
+ * @return Iterator over WAL segments.
+ */
+ public WALIterator walIter(IgniteLogger log, Set<WALRecord.RecordType> types, File... segments) throws IgniteCheckedException {
+ return new IgniteWalIteratorFactory(log)
+ .iterator(new IgniteWalIteratorFactory.IteratorParametersBuilder()
+ .filter((recType, recPtr) -> types.contains(recType))
+ .sharedContext(ctx.cache().context())
+ .filesOrDirs(segments));
+ }
+
+ /**
+ * @param reqId Request ID.
+ * @param res Results.
+ * @param errs Errors.
+ */
+ private void finishIncrementalSnapshotRestore(UUID reqId, Map<UUID, Boolean> res, Map<UUID, Throwable> errs) {
+ if (ctx.clientNode())
+ return;
+
+ SnapshotRestoreContext opCtx0 = opCtx;
+
+ Throwable failure = errs.values().stream().findFirst().
+ orElse(checkNodeLeft(opCtx0.nodes(), res.keySet()));
+
+ if (failure == null) {
+ if (fut != null) { // Call on originated node only.
+ Set<String> cacheGrps = opCtx0.cfgs.keySet().stream()
+ .map(cacheId -> CU.cacheOrGroupName(ctx.cache().cacheDescriptor(cacheId).cacheConfiguration()))
+ .collect(Collectors.toSet());
+
+ ctx.cache().context().snapshotMgr()
+ .warnAtomicCachesInIncrementalSnapshot(opCtx0.snpName, opCtx0.incIdx, cacheGrps);
+ }
+
+ finishProcess(reqId, null);
+
+ return;
+ }
+
+ opCtx0.err.compareAndSet(null, failure);
+
+ if (U.isLocalNodeCoordinator(ctx.discovery()))
+ cacheStopProc.start(reqId, reqId);
+ }
+
/**
* @param metas Map of snapshot metadata distribution across the cluster.
* @return Map of cache partitions per each node.
@@ -1534,6 +2021,9 @@ public class SnapshotRestoreProcess {
/** Operational node id. */
private final UUID opNodeId;
+ /** Index of incremental snapshot, {@code 0} if no incremental snapshot to restore. */
+ private final int incIdx;
+
/**
* Set of restored cache groups path on local node. Collected when all cache configurations received
* from the <tt>prepare</tt> distributed process.
@@ -1573,6 +2063,15 @@ public class SnapshotRestoreProcess {
/** Operation end time. */
private volatile long endTime;
+ /** Total number of WAL segments in incremental snapshot to be restored. */
+ private volatile int totalWalSegments = -1;
+
+ /** Number of processed WAL segments in incremental snapshot. */
+ private volatile int processedWalSegments = -1;
+
+ /** Number of processed entries in incremental snapshot. */
+ private volatile LongAdder processedWalEntries;
+
/** Creates an empty context. */
protected SnapshotRestoreContext() {
reqId = null;
@@ -1581,6 +2080,7 @@ public class SnapshotRestoreProcess {
opNodeId = null;
discoCache = null;
snpPath = null;
+ incIdx = 0;
}
/**
@@ -1597,6 +2097,7 @@ public class SnapshotRestoreProcess {
snpName = req.snapshotName();
snpPath = req.snapshotPath();
opNodeId = req.operationalNodeId();
+ incIdx = req.incrementIndex();
startTime = U.currentTimeMillis();
this.discoCache = discoCache;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
index eeaab94b87d..2f34b575bbf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
@@ -24,7 +24,7 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactor
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.file.UnzipFileIO;
import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
-import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
import static java.nio.file.StandardOpenOption.READ;
@@ -36,9 +36,6 @@ public class FileDescriptor implements Comparable<FileDescriptor>, AbstractWalRe
/** file extension of WAL segment. */
private static final String WAL_SEGMENT_FILE_EXT = ".wal";
- /** Length of WAL segment file name. */
- private static final int WAL_SEGMENT_FILE_NAME_LENGTH = 16;
-
/** File represented by this class. */
protected final File file;
@@ -67,7 +64,7 @@ public class FileDescriptor implements Comparable<FileDescriptor>, AbstractWalRe
assert fileName.contains(WAL_SEGMENT_FILE_EXT);
- this.idx = idx == null ? Long.parseLong(fileName.substring(0, WAL_SEGMENT_FILE_NAME_LENGTH)) : idx;
+ this.idx = idx == null ? U.fixedLengthFileNumber(fileName) : idx;
}
/**
@@ -77,16 +74,7 @@ public class FileDescriptor implements Comparable<FileDescriptor>, AbstractWalRe
* @return Segment file name.
*/
public static String fileName(long idx) {
- SB b = new SB();
-
- String segmentStr = Long.toString(idx);
-
- for (int i = segmentStr.length(); i < WAL_SEGMENT_FILE_NAME_LENGTH; i++)
- b.a('0');
-
- b.a(segmentStr).a(WAL_SEGMENT_FILE_EXT);
-
- return b.toString();
+ return U.fixedLengthNumberName(idx, WAL_SEGMENT_FILE_EXT);
}
/** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 3b98be3688e..edbc38d5dc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -32,6 +32,7 @@ import java.nio.ByteOrder;
import java.nio.channels.ClosedByInterruptException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.sql.Time;
import java.util.ArrayList;
import java.util.Arrays;
@@ -176,10 +177,10 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
private static final byte[] FILL_BUF = new byte[1024 * 1024];
/** Pattern for segment file names. */
- public static final Pattern WAL_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal");
+ public static final Pattern WAL_NAME_PATTERN = U.fixedLengthNumberNamePattern(".wal");
/** Pattern for WAL temp files - these files will be cleared at startup. */
- public static final Pattern WAL_TEMP_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal\\.tmp");
+ public static final Pattern WAL_TEMP_NAME_PATTERN = U.fixedLengthNumberNamePattern(".wal.tmp");
/** WAL segment file filter, see {@link #WAL_NAME_PATTERN} */
public static final FileFilter WAL_SEGMENT_FILE_FILTER = file -> !file.isDirectory() &&
@@ -190,7 +191,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
WAL_TEMP_NAME_PATTERN.matcher(file.getName()).matches();
/** */
- public static final Pattern WAL_SEGMENT_FILE_COMPACTED_PATTERN = Pattern.compile("\\d{16}\\.wal\\.zip");
+ public static final Pattern WAL_SEGMENT_FILE_COMPACTED_PATTERN = U.fixedLengthNumberNamePattern(".wal.zip");
/** WAL segment file filter, see {@link #WAL_NAME_PATTERN} */
public static final FileFilter WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER = file -> !file.isDirectory() &&
@@ -198,7 +199,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
WAL_SEGMENT_FILE_COMPACTED_PATTERN.matcher(file.getName()).matches());
/** */
- private static final Pattern WAL_SEGMENT_TEMP_FILE_COMPACTED_PATTERN = Pattern.compile("\\d{16}\\.wal\\.zip\\.tmp");
+ private static final Pattern WAL_SEGMENT_TEMP_FILE_COMPACTED_PATTERN = U.fixedLengthNumberNamePattern(".wal.zip.tmp");
/** */
private static final FileFilter WAL_SEGMENT_FILE_COMPACTED_FILTER = file -> !file.isDirectory() &&
@@ -658,6 +659,11 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
return !new File(dsCfg.getWalArchivePath()).equals(new File(dsCfg.getWalPath()));
}
+ /** {@inheritDoc} */
+ @Override public @Nullable File archiveDir() {
+ return walArchiveDir;
+ }
+
/**
* Collects WAL segments from the archive only if they are all present.
* Will wait for the last segment to be archived if it is not.
@@ -678,10 +684,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
List<File> res = new ArrayList<>();
for (long i = low.index(); i < high.index(); i++) {
- String segmentName = fileName(i);
-
- File file = new File(walArchiveDir, segmentName);
- File fileZip = new File(walArchiveDir, segmentName + ZIP_SUFFIX);
+ File file = archiveSegment(i, null);
+ File fileZip = compactedSegment(i);
if (file.exists())
res.add(file);
@@ -1122,10 +1126,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
* @return {@code True} exists.
*/
private boolean hasIndex(long absIdx) {
- String segmentName = fileName(absIdx);
-
- boolean inArchive = new File(walArchiveDir, segmentName).exists() ||
- new File(walArchiveDir, segmentName + ZIP_SUFFIX).exists();
+ boolean inArchive = archiveSegment(absIdx, null).exists() ||
+ compactedSegment(absIdx).exists();
if (inArchive)
return true;
@@ -1282,6 +1284,16 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
return lastIdx;
}
+ /**
+ * @param segment WAL segment file.
+ * @return Segment index.
+ */
+ public static long segmentIndex(Path segment) {
+ String fn = segment.getFileName().toString();
+
+ return Long.parseLong(fn.substring(0, fn.indexOf('.')));
+ }
+
/**
* @param file File to read.
* @param ioFactory IO factory.
@@ -2110,11 +2122,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
File origFile = new File(walWorkDir, fileName(segIdx));
- String name = fileName(absIdx);
-
- File dstTmpFile = new File(walArchiveDir, name + TMP_SUFFIX);
-
- File dstFile = new File(walArchiveDir, name);
+ File dstTmpFile = FileWriteAheadLogManager.this.archiveSegment(absIdx, TMP_SUFFIX);
+ File dstFile = FileWriteAheadLogManager.this.archiveSegment(absIdx, null);
if (log.isInfoEnabled()) {
log.info("Starting to copy WAL segment [absIdx=" + absIdx + ", segIdx=" + segIdx +
@@ -2394,13 +2403,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if ((segIdx = tryReserveNextSegmentOrWait()) == -1)
continue;
- String segmentFileName = fileName(segIdx);
-
- File tmpZip = new File(walArchiveDir, segmentFileName + ZIP_SUFFIX + TMP_SUFFIX);
-
- File zip = new File(walArchiveDir, segmentFileName + ZIP_SUFFIX);
-
- File raw = new File(walArchiveDir, segmentFileName);
+ File tmpZip = archiveSegment(segIdx, ZIP_SUFFIX + TMP_SUFFIX);
+ File zip = compactedSegment(segIdx);
+ File raw = archiveSegment(segIdx, null);
long currSize = 0;
long reservedSize = raw.length();
@@ -2555,12 +2560,45 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (segmentReservedOrLocked(desc.idx))
return;
- if (desc.idx < lastCheckpointPtr.index() && duplicateIndices.contains(desc.idx))
- segmentAware.addSize(desc.idx, -deleteArchiveFiles(desc.file));
+ if (desc.idx < lastCheckpointPtr.index() && duplicateIndices.contains(desc.idx)) {
+ long sz = deleteArchiveFiles(desc.file);
+
+ segmentAware.addSize(desc.idx, -sz);
+ }
}
}
}
+ /** {@inheritDoc} */
+ @Override public File compactedSegment(long idx) {
+ return archiveSegment(idx, ZIP_SUFFIX);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void awaitCompacted(long idx) throws IgniteInterruptedCheckedException {
+ segmentAware.awaitSegmentCompressed(idx);
+ }
+
+ /** */
+ private File archiveSegment(long idx, @Nullable String ext) {
+ return archiveSegment(walArchiveDir, idx, ext);
+ }
+
+ /**
+ * @param walArchiveDir WAL archive directory.
+ * @param idx Segment index.
+ * @param ext Optional extension
+ * @return Path to archive segment.
+ */
+ public static File archiveSegment(File walArchiveDir, long idx, String ext) {
+ String fileName = fileName(idx);
+
+ if (ext != null)
+ fileName += ext;
+
+ return new File(walArchiveDir, fileName);
+ }
+
/**
* Responsible for decompressing previously compressed segments of WAL archive if they are needed for replay.
*/
@@ -2605,11 +2643,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (segmentToDecompress == -1)
continue;
- String segmentFileName = fileName(segmentToDecompress);
-
- File zip = new File(walArchiveDir, segmentFileName + ZIP_SUFFIX);
- File unzipTmp = new File(walArchiveDir, segmentFileName + TMP_SUFFIX);
- File unzip = new File(walArchiveDir, segmentFileName);
+ File zip = compactedSegment(segmentToDecompress);
+ File unzipTmp = archiveSegment(segmentToDecompress, TMP_SUFFIX);
+ File unzip = archiveSegment(segmentToDecompress, null);
long currSize = 0;
long reservedSize = U.uncompressedSize(zip);
@@ -2686,7 +2722,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (decompressionFutures.containsKey(idx))
return decompressionFutures.get(idx);
- File f = new File(walArchiveDir, fileName(idx));
+ File f = archiveSegment(idx, null);
if (f.exists())
return new GridFinishedFuture<>();
@@ -2962,8 +2998,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
AbstractFileDescriptor currDesc = desc;
if (!desc.file().exists()) {
- FileDescriptor zipFile = new FileDescriptor(
- new File(walArchiveDir, fileName(desc.idx()) + ZIP_SUFFIX));
+ FileDescriptor zipFile = new FileDescriptor(archiveSegment(walArchiveDir, desc.idx(), ZIP_SUFFIX));
if (!zipFile.file.exists()) {
throw new FileNotFoundException("Both compressed and raw segment files are missing in archive " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java
index 12a1e5f2fdc..3ca1ab82a73 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java
@@ -65,9 +65,9 @@ class SegmentArchivedStorage extends SegmentObservable {
}
/**
- * Method will wait activation of particular WAL segment index.
+ * Method will wait archivation of particular WAL segment index.
*
- * @param awaitIdx absolute index {@link #lastArchivedAbsoluteIndex()} to become true.
+ * @param awaitIdx absolute index {@link #lastArchivedAbsoluteIndex()} to become true.
* @throws IgniteInterruptedCheckedException if interrupted.
*/
synchronized void awaitSegmentArchived(long awaitIdx) throws IgniteInterruptedCheckedException {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
index cb1e7ed9422..2e468da4970 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
@@ -130,15 +130,25 @@ public class SegmentAware {
}
/**
- * Method will wait activation of particular WAL segment index.
+ * Method will wait archivation of particular WAL segment index.
*
- * @param awaitIdx absolute index {@link #lastArchivedAbsoluteIndex()} to become true.
+ * @param awaitIdx absolute index {@link #lastArchivedAbsoluteIndex()} to become true.
* @throws IgniteInterruptedCheckedException if interrupted.
*/
public void awaitSegmentArchived(long awaitIdx) throws IgniteInterruptedCheckedException {
segmentArchivedStorage.awaitSegmentArchived(awaitIdx);
}
+ /**
+ * Method will wait activation of particular WAL segment index.
+ *
+ * @param awaitIdx absolute index {@link #lastCompressedIdx()} to become true.
+ * @throws IgniteInterruptedCheckedException if interrupted.
+ */
+ public void awaitSegmentCompressed(long awaitIdx) throws IgniteInterruptedCheckedException {
+ segmentCompressStorage.awaitSegmentCompressed(awaitIdx);
+ }
+
/**
* Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation. Waits if
* there's no segment to archive right now.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
index 2272c0ec123..85a9257f0b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
@@ -79,10 +79,31 @@ class SegmentCompressStorage {
else
this.lastCompressedIdx = lastMaxCompressedIdx;
+ notifyAll();
+
if (compressedIdx > lastEnqueuedToCompressIdx)
lastEnqueuedToCompressIdx = compressedIdx;
}
+ /**
+ * Method will wait activation of particular WAL segment index.
+ *
+ * @param awaitIdx absolute index {@link #lastCompressedIdx()}} to become true.
+ * @throws IgniteInterruptedCheckedException if interrupted.
+ */
+ public synchronized void awaitSegmentCompressed(long awaitIdx) throws IgniteInterruptedCheckedException {
+ while (lastCompressedIdx() < awaitIdx && !interrupted) {
+ try {
+ wait(2000);
+ }
+ catch (InterruptedException e) {
+ throw new IgniteInterruptedCheckedException(e);
+ }
+ }
+
+ checkInterrupted();
+ }
+
/**
* @return Last compressed segment.
*/
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentReservationStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentReservationStorage.java
index ee81bff4b34..adc4fbfe670 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentReservationStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentReservationStorage.java
@@ -122,7 +122,9 @@ class SegmentReservationStorage extends SegmentObservable {
Long oldMin = oldMinE == null ? null : oldMinE.getKey();
Long newMin = newMinE == null ? null : newMinE.getKey();
- return Objects.equals(oldMin, newMin) ? null : newMin == null ? -1 : newMin;
+ return Objects.equals(oldMin, newMin)
+ ? null
+ : newMin == null ? -1 : newMin;
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
index 1f79234ee09..f902295dcc9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
@@ -24,9 +24,11 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
@@ -39,6 +41,8 @@ import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
import org.apache.ignite.internal.pagemem.wal.record.EncryptedRecord;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotFinishRecord;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotStartRecord;
import org.apache.ignite.internal.pagemem.wal.record.IndexRenameRootPageRecord;
import org.apache.ignite.internal.pagemem.wal.record.LazyDataEntry;
import org.apache.ignite.internal.pagemem.wal.record.MasterKeyChangeRecordV2;
@@ -556,6 +560,12 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
case CLUSTER_SNAPSHOT:
return 4 + ((ClusterSnapshotRecord)record).clusterSnapshotName().getBytes().length;
+ case INCREMENTAL_SNAPSHOT_START_RECORD:
+ return 16;
+
+ case INCREMENTAL_SNAPSHOT_FINISH_RECORD:
+ return ((IncrementalSnapshotFinishRecord)record).dataSize();
+
default:
throw new UnsupportedOperationException("Type: " + record.type());
}
@@ -1293,6 +1303,25 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
break;
+ case INCREMENTAL_SNAPSHOT_START_RECORD:
+ long mst = in.readLong();
+ long lst = in.readLong();
+
+ res = new IncrementalSnapshotStartRecord(new UUID(mst, lst));
+
+ break;
+
+ case INCREMENTAL_SNAPSHOT_FINISH_RECORD:
+ long mstSignBits = in.readLong();
+ long lstSignBits = in.readLong();
+
+ Set<GridCacheVersion> included = readVersions(in);
+ Set<GridCacheVersion> excluded = readVersions(in);
+
+ res = new IncrementalSnapshotFinishRecord(new UUID(mstSignBits, lstSignBits), included, excluded);
+
+ break;
+
default:
throw new UnsupportedOperationException("Type: " + type);
}
@@ -1938,6 +1967,32 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
break;
+ case INCREMENTAL_SNAPSHOT_START_RECORD:
+ IncrementalSnapshotStartRecord startRec = (IncrementalSnapshotStartRecord)rec;
+
+ buf.putLong(startRec.id().getMostSignificantBits());
+ buf.putLong(startRec.id().getLeastSignificantBits());
+
+ break;
+
+ case INCREMENTAL_SNAPSHOT_FINISH_RECORD:
+ IncrementalSnapshotFinishRecord incSnpFinRec = (IncrementalSnapshotFinishRecord)rec;
+
+ buf.putLong(incSnpFinRec.id().getMostSignificantBits());
+ buf.putLong(incSnpFinRec.id().getLeastSignificantBits());
+
+ buf.putInt(incSnpFinRec.included().size());
+
+ for (GridCacheVersion v: incSnpFinRec.included())
+ putVersion(buf, v, false);
+
+ buf.putInt(incSnpFinRec.excluded().size());
+
+ for (GridCacheVersion v: incSnpFinRec.excluded())
+ putVersion(buf, v, false);
+
+ break;
+
case CLUSTER_SNAPSHOT:
byte[] snpName = ((ClusterSnapshotRecord)rec).clusterSnapshotName().getBytes();
@@ -2254,6 +2309,26 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
}
}
+ /**
+ * Read set of versions.
+ *
+ * @param in Data input to read from.
+ * @return Read set of cache versions.
+ */
+ private Set<GridCacheVersion> readVersions(ByteBufferBackedDataInput in) throws IOException {
+ int txsSize = in.readInt();
+
+ Set<GridCacheVersion> txs = new HashSet<>();
+
+ for (int i = 0; i < txsSize; i++) {
+ GridCacheVersion v = readVersion(in, false);
+
+ txs.add(v);
+ }
+
+ return txs;
+ }
+
/**
* @param dataRec Data record to serialize.
* @return Full data record size.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
index 8846b81f233..403950cfa14 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
@@ -665,6 +665,16 @@ public interface IgniteInternalTx {
*/
public MvccSnapshot mvccSnapshot();
+ /**
+ * @return ID of incremental snapshot after which this transaction commits, {@code null} if snapshot isn't running.
+ */
+ public @Nullable UUID incrementalSnapshotId();
+
+ /**
+ * @param id ID of incremental snapshot after which this transaction commits, {@code null} if snapshot isn't running.
+ */
+ public void incrementalSnapshotId(@Nullable UUID id);
+
/**
* @return Transaction counters.
* @param createIfAbsent {@code True} if non-null instance is needed.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 0fe1da5b8e7..c63aafe4a30 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -267,6 +267,19 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
@GridToStringInclude
protected volatile MvccSnapshot mvccSnapshot;
+ /** Incremental snapshot ID. */
+ private @Nullable UUID incSnpId;
+
+ /** {@inheritDoc} */
+ @Override public UUID incrementalSnapshotId() {
+ return incSnpId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void incrementalSnapshotId(UUID id) {
+ incSnpId = id;
+ }
+
/** {@code True} if tx should skip adding itself to completed version map on finish. */
private boolean skipCompletedVers;
@@ -2084,6 +2097,16 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
return null;
}
+ /** {@inheritDoc} */
+ @Override public UUID incrementalSnapshotId() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void incrementalSnapshotId(UUID id) {
+ throw new IllegalStateException("Deserialized transaction can only be used as read-only.");
+ }
+
/** {@inheritDoc} */
@Override public boolean localResult() {
return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 4e9d37007ce..506b589b8e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -78,6 +78,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPr
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareResponse;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxRemote;
import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.IncrementalSnapshotAwareMessage;
import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -133,6 +134,9 @@ import static org.apache.ignite.transactions.TransactionState.ROLLING_BACK;
* Isolated logic to process cache messages.
*/
public class IgniteTxHandler {
+ /** */
+ private static final int TX_MSG_HND_ID = 0;
+
/** Logger. */
private IgniteLogger log;
@@ -218,73 +222,154 @@ public class IgniteTxHandler {
txPrepareMsgLog = ctx.logger(CU.TX_MSG_PREPARE_LOG_CATEGORY);
txFinishMsgLog = ctx.logger(CU.TX_MSG_FINISH_LOG_CATEGORY);
- ctx.io().addCacheHandler(0, GridNearTxPrepareRequest.class, new CI2<UUID, GridCacheMessage>() {
+ ctx.io().addCacheHandler(TX_MSG_HND_ID, GridNearTxPrepareRequest.class, new CI2<UUID, GridCacheMessage>() {
@Override public void apply(UUID nodeId, GridCacheMessage msg) {
processNearTxPrepareRequest(nodeId, (GridNearTxPrepareRequest)msg);
}
});
- ctx.io().addCacheHandler(0, GridNearTxPrepareResponse.class, new CI2<UUID, GridCacheMessage>() {
+ ctx.io().addCacheHandler(TX_MSG_HND_ID, GridNearTxPrepareResponse.class, new CI2<UUID, GridCacheMessage>() {
@Override public void apply(UUID nodeId, GridCacheMessage msg) {
processNearTxPrepareResponse(nodeId, (GridNearTxPrepareResponse)msg);
}
});
- ctx.io().addCacheHandler(0, GridNearTxFinishRequest.class, new CI2<UUID, GridCacheMessage>() {
+ ctx.io().addCacheHandler(TX_MSG_HND_ID, GridNearTxFinishRequest.class, new CI2<UUID, GridCacheMessage>() {
@Override public void apply(UUID nodeId, GridCacheMessage msg) {
processNearTxFinishRequest(nodeId, (GridNearTxFinishRequest)msg);
}
});
- ctx.io().addCacheHandler(0, GridNearTxFinishResponse.class, new CI2<UUID, GridCacheMessage>() {
+ ctx.io().addCacheHandler(TX_MSG_HND_ID, GridNearTxFinishResponse.class, new CI2<UUID, GridCacheMessage>() {
@Override public void apply(UUID nodeId, GridCacheMessage msg) {
processNearTxFinishResponse(nodeId, (GridNearTxFinishResponse)msg);
}
});
- ctx.io().addCacheHandler(0, GridDhtTxPrepareRequest.class, new CI2<UUID, GridCacheMessage>() {
+ ctx.io().addCacheHandler(TX_MSG_HND_ID, GridDhtTxPrepareRequest.class, new CI2<UUID, GridCacheMessage>() {
@Override public void apply(UUID nodeId, GridCacheMessage msg) {
processDhtTxPrepareRequest(nodeId, (GridDhtTxPrepareRequest)msg);
}
});
- ctx.io().addCacheHandler(0, GridDhtTxPrepareResponse.class, new CI2<UUID, GridCacheMessage>() {
+ ctx.io().addCacheHandler(TX_MSG_HND_ID, GridDhtTxPrepareResponse.class, new CI2<UUID, GridCacheMessage>() {
@Override public void apply(UUID nodeId, GridCacheMessage msg) {
processDhtTxPrepareResponse(nodeId, (GridDhtTxPrepareResponse)msg);
}
});
- ctx.io().addCacheHandler(0, GridDhtTxFinishRequest.class, new CI2<UUID, GridCacheMessage>() {
+ ctx.io().addCacheHandler(TX_MSG_HND_ID, GridDhtTxFinishRequest.class, new CI2<UUID, GridCacheMessage>() {
@Override public void apply(UUID nodeId, GridCacheMessage msg) {
processDhtTxFinishRequest(nodeId, (GridDhtTxFinishRequest)msg);
}
});
- ctx.io().addCacheHandler(0, GridDhtTxOnePhaseCommitAckRequest.class, new CI2<UUID, GridCacheMessage>() {
+ ctx.io().addCacheHandler(TX_MSG_HND_ID, GridDhtTxOnePhaseCommitAckRequest.class, new CI2<UUID, GridCacheMessage>() {
@Override public void apply(UUID nodeId, GridCacheMessage msg) {
processDhtTxOnePhaseCommitAckRequest(nodeId, (GridDhtTxOnePhaseCommitAckRequest)msg);
}
});
- ctx.io().addCacheHandler(0, GridDhtTxFinishResponse.class, new CI2<UUID, GridCacheMessage>() {
+ ctx.io().addCacheHandler(TX_MSG_HND_ID, GridDhtTxFinishResponse.class, new CI2<UUID, GridCacheMessage>() {
@Override public void apply(UUID nodeId, GridCacheMessage msg) {
processDhtTxFinishResponse(nodeId, (GridDhtTxFinishResponse)msg);
}
});
- ctx.io().addCacheHandler(0, GridCacheTxRecoveryRequest.class,
+ ctx.io().addCacheHandler(TX_MSG_HND_ID, GridCacheTxRecoveryRequest.class,
new CI2<UUID, GridCacheTxRecoveryRequest>() {
@Override public void apply(UUID nodeId, GridCacheTxRecoveryRequest req) {
processCheckPreparedTxRequest(nodeId, req);
}
});
- ctx.io().addCacheHandler(0, GridCacheTxRecoveryResponse.class,
+ ctx.io().addCacheHandler(TX_MSG_HND_ID, GridCacheTxRecoveryResponse.class,
new CI2<UUID, GridCacheTxRecoveryResponse>() {
@Override public void apply(UUID nodeId, GridCacheTxRecoveryResponse res) {
processCheckPreparedTxResponse(nodeId, res);
}
});
+
+ ctx.io().addCacheHandler(TX_MSG_HND_ID, IncrementalSnapshotAwareMessage.class,
+ new CI2<UUID, IncrementalSnapshotAwareMessage>() {
+ @Override public void apply(UUID nodeId, IncrementalSnapshotAwareMessage msg) {
+ processIncrementalSnapshotAwareMessage(nodeId, msg);
+ }
+ });
+ }
+
+ /** */
+ private void processIncrementalSnapshotAwareMessage(UUID nodeId, IncrementalSnapshotAwareMessage msg) {
+ // Skip if node has joined after incremental snapshot started.
+ if (ctx.localNode().order() <= msg.snapshotTopologyVersion()) {
+ ctx.snapshotMgr().handleIncrementalSnapshotId(msg.id(), msg.snapshotTopologyVersion());
+
+ setIncrementalSnapshotIdIfRequired(msg);
+ }
+
+ GridCacheMessage cacheMsg = msg.payload();
+
+ ctx.io()
+ .cacheHandler(TX_MSG_HND_ID, cacheMsg.getClass())
+ .apply(nodeId, cacheMsg);
+ }
+
+ /**
+ * Set received incremental snapshot ID to transaction if specified.
+ *
+ * @param msg Finish message signed with incremental snapshot ID.
+ */
+ private void setIncrementalSnapshotIdIfRequired(IncrementalSnapshotAwareMessage msg) {
+ if (msg.txInrementalSnapshotId() != null) {
+ IgniteInternalTx tx = findTransactionByMessage(msg.payload());
+
+ if (tx != null)
+ tx.incrementalSnapshotId(msg.txInrementalSnapshotId());
+ }
+ }
+
+ /**
+ * Finds transaction related to the specified message.
+ *
+ * @param msg Transaction message.
+ * @return Transaction, or {@code null} if not found.
+ */
+ private @Nullable IgniteInternalTx findTransactionByMessage(GridCacheMessage msg) {
+ if (msg instanceof GridNearTxFinishRequest) {
+ GridNearTxFinishRequest req = (GridNearTxFinishRequest)msg;
+
+ GridCacheVersion dhtVer = ctx.tm().mappedVersion(req.version());
+
+ if (dhtVer != null)
+ return ctx.tm().tx(dhtVer);
+ }
+ else if (msg instanceof GridDhtTxFinishRequest) {
+ GridDhtTxFinishRequest req = (GridDhtTxFinishRequest)msg;
+
+ return ctx.tm().tx(req.version());
+ }
+ else if (msg instanceof GridDhtTxPrepareResponse) {
+ GridDhtTxPrepareResponse res = (GridDhtTxPrepareResponse)msg;
+
+ GridDhtTxPrepareFuture fut =
+ (GridDhtTxPrepareFuture)ctx.mvcc().versionedFuture(res.version(), res.futureId());
+
+ if (fut != null)
+ return fut.tx();
+ }
+ else if (msg instanceof GridNearTxPrepareResponse) {
+ GridNearTxPrepareResponse res = (GridNearTxPrepareResponse)msg;
+
+ GridNearTxPrepareFutureAdapter fut =
+ (GridNearTxPrepareFutureAdapter)ctx.mvcc().versionedFuture(res.version(), res.futureId());
+
+ if (fut != null)
+ return fut.tx();
+ }
+
+ assert false : "Unexpected message is wrapped into IncrementalSnapshotAwareMessage: " + msg;
+
+ return null;
}
/**
@@ -1046,6 +1131,9 @@ public class IgniteTxHandler {
tx.nearFinishMiniId(req.miniId());
tx.storeEnabled(req.storeEnabled());
+ if (locTx != null)
+ tx.incrementalSnapshotId(locTx.incrementalSnapshotId());
+
if (!tx.markFinalizing(USER_FINISH)) {
if (log.isDebugEnabled())
log.debug("Will not finish transaction (it is handled by another thread) [commit=" + req.commit() + ", tx=" + tx + ']');
@@ -1211,6 +1299,7 @@ public class IgniteTxHandler {
if (dhtTx != null) {
dhtTx.onePhaseCommit(true);
dhtTx.needReturnValue(req.needReturnValue());
+ dhtTx.incrementalSnapshotId(ctx.snapshotMgr().incrementalSnapshotId());
finish(dhtTx, req);
}
@@ -1531,8 +1620,10 @@ public class IgniteTxHandler {
GridDhtTxRemote dhtTx,
GridNearTxRemote nearTx) {
try {
+ IgniteTxAdapter tx = dhtTx != null ? dhtTx : nearTx;
+
// Reply back to sender.
- ctx.io().send(nodeId, res, req.policy());
+ ctx.tm().sendTransactionMessage(nodeId, res, tx, req.policy());
if (txPrepareMsgLog.isDebugEnabled()) {
txPrepareMsgLog.debug("Sent dht prepare response [txId=" + req.nearXidVersion() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index be8636e1768..dcd3092a593 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -36,6 +36,8 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
import java.util.function.Predicate;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteClientDisconnectedException;
@@ -311,6 +313,12 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
/** Distributed transaction configuration. */
private DistributedTransactionConfiguration distributedTransactionConfiguration;
+ /** Transforms transaction message. */
+ private volatile BiFunction<GridCacheMessage, IgniteInternalTx, GridCacheMessage> txMsgTransform;
+
+ /** Callback invoked on transaction commit. */
+ private volatile Consumer<IgniteInternalTx> onCommitCallback;
+
/** {@inheritDoc} */
@Override protected void onKernalStop0(boolean cancel) {
cctx.gridIO().removeMessageListener(TOPIC_TX);
@@ -1608,6 +1616,11 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
", tx=" + tx.getClass().getSimpleName() + ']');
}
+ Consumer<IgniteInternalTx> onCommitCallback0 = onCommitCallback;
+
+ if (onCommitCallback0 != null)
+ onCommitCallback0.accept(tx);
+
ConcurrentMap<GridCacheVersion, IgniteInternalTx> txIdMap = transactionMap(tx);
if (txIdMap.remove(tx.xidVersion(), tx)) {
@@ -3132,6 +3145,45 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
return distributedTransactionConfiguration;
}
+ /** @param transform Transaction message transformer. */
+ public void txMessageTransformer(BiFunction<GridCacheMessage, IgniteInternalTx, GridCacheMessage> transform) {
+ txMsgTransform = transform;
+ }
+
+ /** @param callback Callback invoked on transaction commit. */
+ public void onCommitCallback(Consumer<IgniteInternalTx> callback) {
+ onCommitCallback = callback;
+ }
+
+ /**
+ * Sends transaction message after transforming it.
+ *
+ * @param nodeId Node ID to send message.
+ * @param msg Original message to transform.
+ * @param tx Transaction.
+ * @param plc IO policy.
+ */
+ public void sendTransactionMessage(UUID nodeId, GridCacheMessage msg, IgniteInternalTx tx, byte plc) throws IgniteCheckedException {
+ BiFunction<GridCacheMessage, IgniteInternalTx, GridCacheMessage> transform = txMsgTransform;
+
+ if (transform != null)
+ msg = transform.apply(msg, tx);
+
+ cctx.io().send(nodeId, msg, plc);
+ }
+
+ /**
+ * Sends transaction message after transforming it.
+ *
+ * @param n Node to send message.
+ * @param msg Original message to transform.
+ * @param tx Transaction.
+ * @param plc IO policy.
+ */
+ public void sendTransactionMessage(ClusterNode n, GridCacheMessage msg, IgniteInternalTx tx, byte plc) throws IgniteCheckedException {
+ sendTransactionMessage(n.id(), msg, tx, plc);
+ }
+
/** Tx key collisions info holder. */
private final class KeyCollisionsHolder {
/** Stripes count. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 135e1119401..1cb2506d536 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -400,6 +400,9 @@ public abstract class IgniteUtils {
/** Alphanumeric with underscore regexp pattern. */
private static final Pattern ALPHANUMERIC_UNDERSCORE_PATTERN = Pattern.compile("^[a-zA-Z_0-9]+$");
+ /** Length of numbered file name. */
+ public static final int NUMBER_FILE_NAME_LENGTH = 16;
+
/** Ignite package. */
public static final String IGNITE_PKG = "org.apache.ignite.";
@@ -3366,6 +3369,50 @@ public abstract class IgniteUtils {
return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
}
+ /**
+ * Generates file name from index.
+ *
+ * @param num Number to generate file name.
+ * @param ext Optional extension
+ * @return File name.
+ */
+ public static String fixedLengthNumberName(long num, @Nullable String ext) {
+ SB b = new SB();
+
+ String segmentStr = Long.toString(num);
+
+ for (int i = segmentStr.length(); i < NUMBER_FILE_NAME_LENGTH; i++)
+ b.a('0');
+
+ b.a(segmentStr);
+
+ if (ext != null)
+ b.a(ext);
+
+ return b.toString();
+ }
+
+ /**
+ * @param fileName File name.
+ * @return Number of this file.
+ */
+ public static long fixedLengthFileNumber(String fileName) {
+ return Long.parseLong(fileName.substring(0, NUMBER_FILE_NAME_LENGTH));
+ }
+
+ /**
+ * @param ext Optional extension.
+ * @return Pattern to match numbered file name with the specific extension.
+ */
+ public static Pattern fixedLengthNumberNamePattern(@Nullable String ext) {
+ String pattern = "\\d{" + NUMBER_FILE_NAME_LENGTH + "}";
+
+ if (ext != null)
+ pattern += ext.replaceAll("\\.", "\\\\\\.");
+
+ return Pattern.compile(pattern);
+ }
+
/**
* Verifier always returns successful result for any host.
*/
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
index ad0a84d2f42..6d02e66a5a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
@@ -98,7 +98,7 @@ public class DistributedProcess<I extends Serializable, R extends Serializable>
Function<I, IgniteInternalFuture<R>> exec,
CI3<UUID, Map<UUID, R>, Map<UUID, Throwable>> finish
) {
- this(ctx, type, exec, finish, (id, req) -> new InitMessage<>(id, type, req));
+ this(ctx, type, exec, finish, (id, req) -> new InitMessage<>(id, type, req, false));
}
/**
@@ -142,6 +142,7 @@ public class DistributedProcess<I extends Serializable, R extends Serializable>
}
p.crdId = crd.id();
+ p.waitClnRes = msg.waitClientResults();
if (crd.isLocal())
initCoordinator(p, topVer);
@@ -155,7 +156,7 @@ public class DistributedProcess<I extends Serializable, R extends Serializable>
else
p.resFut.onDone(f.result());
- if (!ctx.clientNode()) {
+ if (!ctx.clientNode() || p.waitClnRes) {
assert crd != null;
sendSingleMessage(p);
@@ -229,7 +230,7 @@ public class DistributedProcess<I extends Serializable, R extends Serializable>
if (crd.isLocal())
initCoordinator(p, discoCache.version());
- if (!ctx.clientNode())
+ if (!ctx.clientNode() || p.waitClnRes)
p.resFut.listen(f -> sendSingleMessage(p));
}
else if (F.eq(ctx.localNodeId(), p.crdId)) {
@@ -276,7 +277,9 @@ public class DistributedProcess<I extends Serializable, R extends Serializable>
assert p.remaining.isEmpty();
- p.remaining.addAll(F.viewReadOnly(ctx.discovery().serverNodes(topVer), F.node2id()));
+ p.remaining.addAll(F.viewReadOnly(
+ p.waitClnRes ? ctx.discovery().nodes(topVer) : ctx.discovery().serverNodes(topVer),
+ F.node2id()));
p.initCrdFut.onDone();
}
@@ -394,6 +397,9 @@ public class DistributedProcess<I extends Serializable, R extends Serializable>
/** Remaining nodes ids to received single nodes result. */
private final Set<UUID> remaining = new GridConcurrentHashSet<>();
+ /** If {@code true} it waits client nodes results, otherwise only server nodes results are awaited. */
+ private volatile boolean waitClnRes;
+
/** Future for a local action result. */
private final GridFutureAdapter<R> resFut = new GridFutureAdapter<>();
@@ -469,9 +475,19 @@ public class DistributedProcess<I extends Serializable, R extends Serializable>
*/
RESTORE_CACHE_GROUP_SNAPSHOT_START,
+ /**
+ * Cache group restore cache stop phase.
+ */
+ RESTORE_CACHE_GROUP_SNAPSHOT_STOP,
+
/**
* Cache group restore rollback phase.
*/
- RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK
+ RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK,
+
+ /**
+ * Incremental snapshot restore start phase.
+ */
+ RESTORE_INCREMENTAL_SNAPSHOT_START
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/InitMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/InitMessage.java
index 4e43f523200..4be44dfb72e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/InitMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/InitMessage.java
@@ -52,15 +52,19 @@ public class InitMessage<I extends Serializable> implements DiscoveryCustomMessa
/** Request. */
private final I req;
+ /** Whether coordinator waits client nodes results. */
+ private final boolean waitClnRes;
+
/**
* @param processId Process id.
* @param type Process type.
* @param req Request.
*/
- public InitMessage(UUID processId, DistributedProcessType type, I req) {
+ public InitMessage(UUID processId, DistributedProcessType type, I req, boolean waitClnRes) {
this.processId = processId;
this.type = type.ordinal();
this.req = req;
+ this.waitClnRes = waitClnRes;
}
/** {@inheritDoc} */
@@ -99,6 +103,11 @@ public class InitMessage<I extends Serializable> implements DiscoveryCustomMessa
return req;
}
+ /** @return Whether coordinator waits client nodes results. */
+ public boolean waitClientResults() {
+ return waitClnRes;
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(InitMessage.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
index 0b4d4863597..8fb3c4d8a78 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
@@ -3012,7 +3012,7 @@ public class GridFunc {
V v2 = m2.get(e.getKey());
if (v1 == v2)
- return true;
+ continue;
if (v1 == null || v2 == null)
return false;
@@ -3027,7 +3027,7 @@ public class GridFunc {
return false;
}
else {
- if (!eq(v1, v2))
+ if (!(v1.getClass().isArray() ? arrayEq(v1, v2) : eq(v1, v2)))
return false;
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cdc/VisorCdcDeleteLostSegmentsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cdc/VisorCdcDeleteLostSegmentsTask.java
index b63e18d757a..d6d8aaec036 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cdc/VisorCdcDeleteLostSegmentsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cdc/VisorCdcDeleteLostSegmentsTask.java
@@ -32,7 +32,6 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.compute.ComputeJobResult;
import org.apache.ignite.internal.cdc.CdcFileLockHolder;
-import org.apache.ignite.internal.cdc.CdcMain;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -110,10 +109,10 @@ public class VisorCdcDeleteLostSegmentsTask extends VisorMultiNodeTask<Void, Voi
cdcFiles
.filter(p -> WAL_SEGMENT_FILE_FILTER.accept(p.toFile()))
- .sorted(Comparator.comparingLong(CdcMain::segmentIndex)
+ .sorted(Comparator.comparingLong(FileWriteAheadLogManager::segmentIndex)
.reversed()) // Sort by segment index.
.forEach(path -> {
- long idx = CdcMain.segmentIndex(path);
+ long idx = FileWriteAheadLogManager.segmentIndex(path);
if (lastSgmnt.get() == -1 || lastSgmnt.get() - idx == 1) {
lastSgmnt.set(idx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTask.java
index ec08b4cf469..4d86d5f30c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTask.java
@@ -52,8 +52,11 @@ public class VisorSnapshotCreateTask extends VisorSnapshotOneNodeTask<VisorSnaps
/** {@inheritDoc} */
@Override protected String run(VisorSnapshotCreateTaskArg arg) throws IgniteException {
- IgniteFutureImpl<Void> fut =
- ignite.context().cache().context().snapshotMgr().createSnapshot(arg.snapshotName(), arg.snapshotPath());
+ IgniteFutureImpl<Void> fut = ignite.context().cache().context().snapshotMgr().createSnapshot(
+ arg.snapshotName(),
+ arg.snapshotPath(),
+ arg.incremental()
+ );
IgniteSnapshotManager.ClusterSnapshotFuture snpFut =
fut.internalFuture() instanceof IgniteSnapshotManager.ClusterSnapshotFuture ?
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTaskArg.java
index a828cf9ec9d..eadfab88551 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTaskArg.java
@@ -32,6 +32,9 @@ public class VisorSnapshotCreateTaskArg extends VisorSnapshotCheckTaskArg {
/** Synchronous execution flag. */
private boolean sync;
+ /** Incremental snapshot flag. */
+ private boolean inc;
+
/** Default constructor. */
public VisorSnapshotCreateTaskArg() {
// No-op.
@@ -41,11 +44,13 @@ public class VisorSnapshotCreateTaskArg extends VisorSnapshotCheckTaskArg {
* @param snpName Snapshot name.
* @param snpPath Snapshot directory path.
* @param sync Synchronous execution flag.
+ * @param inc Incremental snapshot flag.
*/
- public VisorSnapshotCreateTaskArg(String snpName, String snpPath, boolean sync) {
+ public VisorSnapshotCreateTaskArg(String snpName, String snpPath, boolean sync, boolean inc) {
super(snpName, snpPath);
this.sync = sync;
+ this.inc = inc;
}
/** @return Synchronous execution flag. */
@@ -53,11 +58,17 @@ public class VisorSnapshotCreateTaskArg extends VisorSnapshotCheckTaskArg {
return sync;
}
+ /** @return Incremental snapshot flag. */
+ public boolean incremental() {
+ return inc;
+ }
+
/** {@inheritDoc} */
@Override protected void writeExternalData(ObjectOutput out) throws IOException {
super.writeExternalData(out);
out.writeBoolean(sync);
+ out.writeBoolean(inc);
}
/** {@inheritDoc} */
@@ -65,6 +76,7 @@ public class VisorSnapshotCreateTaskArg extends VisorSnapshotCheckTaskArg {
super.readExternalData(ver, in);
sync = in.readBoolean();
+ inc = in.readBoolean();
}
/** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTask.java
index 10ff00c9c3f..fad625cf3fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTask.java
@@ -68,7 +68,7 @@ public class VisorSnapshotRestoreTask extends VisorSnapshotOneNodeTask<VisorSnap
/** {@inheritDoc} */
@Override protected String run(VisorSnapshotRestoreTaskArg arg) throws IgniteException {
IgniteFutureImpl<Void> fut = ignite.context().cache().context().snapshotMgr()
- .restoreSnapshot(arg.snapshotName(), arg.snapshotPath(), arg.groupNames());
+ .restoreSnapshot(arg.snapshotName(), arg.snapshotPath(), arg.groupNames(), arg.incrementIndex());
IgniteSnapshotManager.ClusterSnapshotFuture snpFut =
fut.internalFuture() instanceof IgniteSnapshotManager.ClusterSnapshotFuture ?
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTaskArg.java
index 86c07c4740a..07c18fe59c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTaskArg.java
@@ -38,6 +38,9 @@ public class VisorSnapshotRestoreTaskArg extends VisorSnapshotCreateTaskArg {
/** Snapshot restore operation management action. */
private VisorSnapshotRestoreTaskAction action;
+ /** Incremental snapshot index. */
+ private int incIdx;
+
/** Default constructor. */
public VisorSnapshotRestoreTaskArg() {
// No-op.
@@ -45,6 +48,8 @@ public class VisorSnapshotRestoreTaskArg extends VisorSnapshotCreateTaskArg {
/**
* @param snpName Snapshot name.
+ * @param snpPath Snapshot path.
+ * @param incIdx Incremental snapshot index, {@code null} if no increments should be restored.
* @param sync Synchronous execution flag.
* @param action Snapshot restore operation management action.
* @param grpNames Cache group names.
@@ -52,14 +57,16 @@ public class VisorSnapshotRestoreTaskArg extends VisorSnapshotCreateTaskArg {
public VisorSnapshotRestoreTaskArg(
String snpName,
String snpPath,
+ @Nullable Integer incIdx,
boolean sync,
VisorSnapshotRestoreTaskAction action,
@Nullable Collection<String> grpNames
) {
- super(snpName, snpPath, sync);
+ super(snpName, snpPath, sync, false);
this.action = action;
this.grpNames = grpNames;
+ this.incIdx = incIdx == null ? 0 : incIdx;
}
/** @return Cache group names. */
@@ -72,12 +79,17 @@ public class VisorSnapshotRestoreTaskArg extends VisorSnapshotCreateTaskArg {
return action;
}
+ /** @return Incremental snapshot index. */
+ public int incrementIndex() {
+ return incIdx;
+ }
/** {@inheritDoc} */
@Override protected void writeExternalData(ObjectOutput out) throws IOException {
super.writeExternalData(out);
U.writeEnum(out, action);
U.writeCollection(out, grpNames);
+ out.writeInt(incIdx);
}
/** {@inheritDoc} */
@@ -85,6 +97,7 @@ public class VisorSnapshotRestoreTaskArg extends VisorSnapshotCreateTaskArg {
super.readExternalData(ver, in);
action = U.readEnum(in, VisorSnapshotRestoreTaskAction.class);
grpNames = U.readCollection(in);
+ incIdx = in.readInt();
}
/** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotStatusTask.java
index d49136986e8..0309f65738a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotStatusTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotStatusTask.java
@@ -31,7 +31,7 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.Snapshot
import org.apache.ignite.internal.processors.metric.MetricRegistry;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.T5;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.visor.VisorJob;
import org.apache.ignite.internal.visor.VisorMultiNodeTask;
@@ -85,11 +85,11 @@ public class VisorSnapshotStatusTask extends VisorMultiNodeTask<Void, VisorSnaps
res = F.view(res, s -> s.requestId.equals(s0.requestId));
// Merge nodes progress.
- Map<UUID, T2<Long, Long>> progress = new HashMap<>();
+ Map<UUID, T5<Long, Long, Long, Long, Long>> progress = new HashMap<>();
res.forEach(s -> progress.putAll(s.progress));
- return new VisorSnapshotTaskResult(new SnapshotStatus(s0.op, s0.name, s0.requestId, s0.startTime, progress), null);
+ return new VisorSnapshotTaskResult(new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.requestId, s0.startTime, progress), null);
}
/** */
@@ -115,20 +115,26 @@ public class VisorSnapshotStatusTask extends VisorMultiNodeTask<Void, VisorSnaps
SnapshotOperationRequest req = snpMgr.currentCreateRequest();
if (req != null) {
- MetricRegistry mreg = ignite.context().metric().registry(SNAPSHOT_METRICS);
+ T5<Long, Long, Long, Long, Long> metrics;
+
+ if (req.incremental())
+ metrics = new T5<>(-1L, -1L, -1L, -1L, -1L);
+ else {
+ MetricRegistry mreg = ignite.context().metric().registry(SNAPSHOT_METRICS);
+
+ metrics = new T5<>(
+ mreg.<LongMetric>findMetric("CurrentSnapshotProcessedSize").value(),
+ mreg.<LongMetric>findMetric("CurrentSnapshotTotalSize").value(),
+ -1L, -1L, -1L);
+ }
return new SnapshotStatus(
SnapshotOperation.CREATE,
req.snapshotName(),
+ req.incrementIndex(),
req.requestId().toString(),
req.startTime(),
- F.asMap(
- ignite.localNode().id(),
- new T2<>(
- mreg.<LongMetric>findMetric("CurrentSnapshotProcessedSize").value(),
- mreg.<LongMetric>findMetric("CurrentSnapshotTotalSize").value()
- )
- )
+ F.asMap(ignite.localNode().id(), metrics)
);
}
@@ -140,13 +146,17 @@ public class VisorSnapshotStatusTask extends VisorMultiNodeTask<Void, VisorSnaps
return new SnapshotStatus(
SnapshotOperation.RESTORE,
mreg.findMetric("snapshotName").getAsString(),
+ mreg.<IntMetric>findMetric("incrementIndex").value(),
mreg.findMetric("requestId").getAsString(),
mreg.<LongMetric>findMetric("startTime").value(),
F.asMap(
ignite.localNode().id(),
- new T2<>(
+ new T5<>(
(long)mreg.<IntMetric>findMetric("processedPartitions").value(),
- (long)mreg.<IntMetric>findMetric("totalPartitions").value()
+ (long)mreg.<IntMetric>findMetric("totalPartitions").value(),
+ (long)mreg.<IntMetric>findMetric("processedWalSegments").value(),
+ (long)mreg.<IntMetric>findMetric("totalWalSegments").value(),
+ mreg.<LongMetric>findMetric("processedWalEntries").value()
)
)
);
@@ -167,6 +177,9 @@ public class VisorSnapshotStatusTask extends VisorMultiNodeTask<Void, VisorSnaps
/** Snapshot name. */
private final String name;
+ /** Incremental snapshot index. */
+ private final int incIdx;
+
/** Request ID. */
private final String requestId;
@@ -174,13 +187,20 @@ public class VisorSnapshotStatusTask extends VisorMultiNodeTask<Void, VisorSnaps
private final long startTime;
/** Progress of operation on nodes. */
- private final Map<UUID, T2<Long, Long>> progress;
+ private final Map<UUID, T5<Long, Long, Long, Long, Long>> progress;
/** */
- public SnapshotStatus(SnapshotOperation op, String name, String requestId, long startTime,
- Map<UUID, T2<Long, Long>> progress) {
+ public SnapshotStatus(
+ SnapshotOperation op,
+ String name,
+ int incIdx,
+ String requestId,
+ long startTime,
+ Map<UUID, T5<Long, Long, Long, Long, Long>> progress
+ ) {
this.op = op;
this.name = name;
+ this.incIdx = incIdx;
this.requestId = requestId;
this.startTime = startTime;
this.progress = Collections.unmodifiableMap(progress);
@@ -196,6 +216,11 @@ public class VisorSnapshotStatusTask extends VisorMultiNodeTask<Void, VisorSnaps
return name;
}
+ /** @return Incremental snapshot index. */
+ public int incrementIndex() {
+ return incIdx;
+ }
+
/** @return Request ID. */
public String requestId() {
return requestId;
@@ -207,7 +232,7 @@ public class VisorSnapshotStatusTask extends VisorMultiNodeTask<Void, VisorSnaps
}
/** @return Progress of operation on nodes. */
- public Map<UUID, T2<Long, Long>> progress() {
+ public Map<UUID, T5<Long, Long, Long, Long, Long>> progress() {
return progress;
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java
index d8b06eaccd9..b677f55574c 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java
@@ -40,6 +40,21 @@ public interface SnapshotMXBean {
String snpPath
);
+ /**
+ * Create the cluster-wide incremental snapshot for the given full snapshot.
+ *
+ * @param fullSnapshot Full snapshot name to attach incremental snapshot to.
+ * @param fullSnapshotPath Full snapshot directory path.
+ * @see IgniteSnapshot#createSnapshot(String)
+ * @see IgniteSnapshot#createIncrementalSnapshot(String)
+ */
+ public void createIncrementalSnapshot(
+ @MXBeanParameter(name = "fullSnapshot", description = "Snapshot name.")
+ String fullSnapshot,
+ @MXBeanParameter(name = "fullSnapshotPath", description = "Optional snapshot directory path.")
+ String fullSnapshotPath
+ );
+
/**
* Cancel previously started snapshot operation on the node initiator.
*
@@ -76,6 +91,28 @@ public interface SnapshotMXBean {
String cacheGroupNames
);
+ /**
+ * Restore cluster-wide snapshot and its increments. Snapshot is restored first and after that all increments
+ * are restored sequentially from the {@code 1} to the specified {@code incIdx}.
+ *
+ * @param name Snapshot name.
+ * @param path Snapshot directory path.
+ * @param cacheGroupNames Optional comma-separated list of cache group names.
+ * @param incIdx Incremental snapshot index.
+ * @see IgniteSnapshot#restoreSnapshot(String, Collection)
+ */
+ @MXBeanDescription("Restore cluster-wide snapshot and its increments.")
+ public void restoreSnapshot(
+ @MXBeanParameter(name = "snpName", description = "Snapshot name.")
+ String name,
+ @MXBeanParameter(name = "snpPath", description = "Optional snapshot directory path.")
+ String path,
+ @MXBeanParameter(name = "cacheGroupNames", description = "Optional comma-separated list of cache group names.")
+ String cacheGroupNames,
+ @MXBeanParameter(name = "incIdx", description = "Incremental snapshot index.")
+ int incIdx
+ );
+
/**
* Cancel previously started snapshot restore operation.
*
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/SnapshotView.java b/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/SnapshotView.java
index 3b17ca28bdd..a7078204bcd 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/SnapshotView.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/SnapshotView.java
@@ -17,7 +17,12 @@
package org.apache.ignite.spi.systemview.view;
+import java.util.Collection;
import org.apache.ignite.internal.managers.systemview.walker.Order;
+import org.apache.ignite.internal.pagemem.wal.record.delta.ClusterSnapshotRecord;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.IncrementalSnapshotMetadata;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadata;
+import org.apache.ignite.internal.util.typedef.F;
/**
* Snapshot representation for a {@link SystemView}.
@@ -41,22 +46,45 @@ public class SnapshotView {
/** Cache group names that were included in the snapshot. */
private final String cacheGrps;
+ /** WAL segment that contains {@link ClusterSnapshotRecord} if exists. */
+ private final Long snpRecSeg;
+
+ /** Full or incremental. */
+ private final SnapshotType type;
+
+ /** Incremental snapshot index. */
+ private final Integer incIdx;
+
/**
- * @param name Snapshot name.
- * @param consistentId Node consistent ID.
- * @param baselineNodes Baseline nodes affected by the snapshot.
+ * @param meta Snapshot metadata.
* @param cacheGrps Cache group names that were included in the snapshot.
*/
public SnapshotView(
- String name,
- String consistentId,
- String baselineNodes,
- String cacheGrps
+ SnapshotMetadata meta,
+ Collection<String> cacheGrps
) {
- this.name = name;
- this.consistentId = consistentId;
- this.baselineNodes = baselineNodes;
- this.cacheGrps = cacheGrps;
+ type = SnapshotType.FULL;
+ name = meta.snapshotName();
+ consistentId = meta.consistentId();
+ baselineNodes = F.concat(meta.baselineNodes(), ",");
+ snpRecSeg = meta.snapshotRecordPointer() == null ? null : meta.snapshotRecordPointer().index();
+ incIdx = null;
+
+ this.cacheGrps = F.concat(cacheGrps, ",");
+ }
+
+ /**
+ * @param incMeta Incremental snapshot metadata.
+ */
+ public SnapshotView(IncrementalSnapshotMetadata incMeta) {
+ type = SnapshotType.INCREMENTAL;
+ name = incMeta.snapshotName();
+ consistentId = incMeta.consistentId();
+ snpRecSeg = incMeta.incrementalSnapshotPointer().index();
+ incIdx = incMeta.incrementIndex();
+
+ baselineNodes = null;
+ cacheGrps = null;
}
/**
@@ -90,4 +118,37 @@ public class SnapshotView {
public String cacheGroups() {
return cacheGrps;
}
+
+ /**
+ * @return WAL segment that contains {@link ClusterSnapshotRecord} if exists.
+ */
+ @Order(4)
+ public Long snapshotRecordSegment() {
+ return snpRecSeg;
+ }
+
+ /**
+ * @return Incremental snapshot index, {@code null} for full snapshot.
+ */
+ @Order(5)
+ public Integer incrementIndex() {
+ return incIdx;
+ }
+
+ /**
+ * @return Snapshot type.
+ */
+ @Order(6)
+ public String type() {
+ return type.name();
+ }
+
+ /** Snapshot types. */
+ private enum SnapshotType {
+ /** Full snapshot. */
+ FULL,
+
+ /** Incremental snapshot. */
+ INCREMENTAL
+ }
}
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 2167ba26569..d509b8248b2 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -1146,9 +1146,9 @@ org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotHandler
org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotHandlerResult
org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotHandlerType
org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadata
-org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadataCollectorTask
-org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadataCollectorTask$1
-org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadataCollectorTaskArg
+org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadataVerificationTask
+org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadataVerificationTask$1
+org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadataVerificationTaskArg
org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperation
org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperationRequest
org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotPartitionsVerifyTask
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/metric/SystemViewSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/metric/SystemViewSelfTest.java
index 243d95e6861..20f60392f3b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/metric/SystemViewSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/metric/SystemViewSelfTest.java
@@ -20,7 +20,6 @@ package org.apache.ignite.internal.metric;
import java.lang.reflect.Field;
import java.sql.Connection;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -91,6 +90,7 @@ import org.apache.ignite.internal.processors.service.DummyService;
import org.apache.ignite.internal.util.GridTestClockTimer;
import org.apache.ignite.internal.util.StripedExecutor;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiPredicate;
@@ -2098,39 +2098,43 @@ public class SystemViewSelfTest extends GridCommonAbstractTest {
.setDataStorageConfiguration(
new DataStorageConfiguration().setDefaultDataRegionConfiguration(
new DataRegionConfiguration().setName("pds").setPersistenceEnabled(true)
- )))
+ ).setWalCompactionEnabled(true)))
) {
ignite.cluster().state(ClusterState.ACTIVE);
ignite.cache(DEFAULT_CACHE_NAME).put(1, 1);
ignite.snapshot().createSnapshot(testSnap0).get();
+ ignite.snapshot().createSnapshot(testSnap1).get(getTestTimeout());
+ ignite.snapshot().createIncrementalSnapshot(testSnap1).get(getTestTimeout());
+ ignite.snapshot().createIncrementalSnapshot(testSnap1).get(getTestTimeout());
SystemView<SnapshotView> views = ignite.context().systemView().view(SNAPSHOT_SYS_VIEW);
- assertEquals(1, views.size());
-
- SnapshotView view = F.first(views);
-
- assertEquals(testSnap0, view.name());
- assertEquals(ignite.localNode().consistentId().toString(), view.consistentId());
-
- Collection<?> constIds = F.nodeConsistentIds(ignite.cluster().nodes());
+ List<T2<String, Integer>> exp = Lists.newArrayList(
+ new T2<>(testSnap0, null),
+ new T2<>(testSnap1, null),
+ new T2<>(testSnap1, 1),
+ new T2<>(testSnap1, 2));
- assertEquals(F.concat(constIds, ","), view.baselineNodes());
- assertEquals(F.concat(Arrays.asList(dfltCacheGrp, METASTORAGE_CACHE_NAME), ","), view.cacheGroups());
+ assertEquals(4, views.size());
- ignite.createCache("testCache");
+ for (SnapshotView v: views) {
+ assertTrue(exp.remove(new T2<>(v.name(), v.incrementIndex())));
- ignite.snapshot().createSnapshot(testSnap1).get();
+ assertEquals(ignite.localNode().consistentId().toString(), v.consistentId());
+ assertNotNull(v.snapshotRecordSegment());
- views = ignite.context().systemView().view(SNAPSHOT_SYS_VIEW);
+ Integer incIdx = v.incrementIndex();
- assertEquals(2, views.size());
-
- List<String> exp = Lists.newArrayList(testSnap0, testSnap1);
-
- views.forEach(v -> assertTrue(exp.remove(v.name())));
+ if (incIdx == null) {
+ assertEquals(ignite.localNode().consistentId().toString(), v.baselineNodes());
+ assertEquals(String.join(",", dfltCacheGrp, METASTORAGE_CACHE_NAME), v.cacheGroups());
+ assertEquals("FULL", v.type());
+ }
+ else
+ assertEquals("INCREMENTAL", v.type());
+ }
assertTrue(exp.isEmpty());
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index 1dac0437bf7..03418defa38 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+import java.io.File;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
@@ -201,4 +202,19 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
@Override public void startAutoReleaseSegments() {
// No-op.
}
+
+ /** {@inheritDoc} */
+ @Override public @Nullable File archiveDir() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public @Nullable File compactedSegment(long idx) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void awaitCompacted(long idx) {
+ // No-op.
+ }
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java
index 0f2aef16714..5a83873f2e9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java
@@ -70,11 +70,17 @@ import org.apache.ignite.internal.TestRecordingCommunicationSpi;
import org.apache.ignite.internal.encryption.AbstractEncryptionTest;
import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotFinishRecord;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotStartRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
import org.apache.ignite.internal.processors.marshaller.MappedName;
import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -82,6 +88,7 @@ import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteFutureCancelledException;
import org.apache.ignite.lang.IgnitePredicate;
@@ -108,7 +115,9 @@ import static org.apache.ignite.internal.processors.cache.persistence.file.FileP
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_PREFIX;
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.CP_SNAPSHOT_REASON;
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.DFLT_SNAPSHOT_TMP_DIR;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.incrementalSnapshotWalsDir;
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.resolveSnapshotWorkDirectory;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.snapshotMetaFileName;
import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
@@ -190,7 +199,8 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
.setMaxSize(512L * 1024 * 1024)
.setPersistenceEnabled(persistence))
.setCheckpointFrequency(3000)
- .setPageSize(PAGE_SIZE))
+ .setPageSize(PAGE_SIZE)
+ .setWalCompactionEnabled(true))
.setClusterStateOnStart(INACTIVE)
.setIncludeEventTypes(EVTS_CLUSTER_SNAPSHOT)
.setDiscoverySpi(discoSpi);
@@ -662,6 +672,57 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
return snpFutTask;
}
+ /** Checks incremental snapshot exists. */
+ protected boolean checkIncremental(IgniteEx node, String snpName, String snpPath, int incIdx) {
+ File incSnpDir = snp(node).incrementalSnapshotLocalDir(snpName, snpPath, incIdx);
+
+ if (incSnpDir.exists()) {
+ checkIncrementalSnapshotWalRecords(node, incSnpDir);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /** */
+ private void checkIncrementalSnapshotWalRecords(IgniteEx node, File incSnpDir) {
+ try {
+ IncrementalSnapshotMetadata incSnpMeta = snp(node).readFromFile(
+ new File(incSnpDir, snapshotMetaFileName(node.localNode().consistentId().toString())));
+
+ File incSnpWalDir = incrementalSnapshotWalsDir(incSnpDir, incSnpMeta.folderName());
+
+ WALIterator it = new IgniteWalIteratorFactory(log).iterator(
+ new IgniteWalIteratorFactory.IteratorParametersBuilder().filesOrDirs(incSnpWalDir));
+
+ boolean started = false;
+ boolean finished = false;
+
+ for (IgniteBiTuple<WALPointer, WALRecord> entry: it) {
+ assertFalse("IncrementalSnapshotFinishRecord must be the last record in snapshot", finished);
+
+ WALRecord rec = entry.getValue();
+
+ if (rec.type() == WALRecord.RecordType.INCREMENTAL_SNAPSHOT_START_RECORD) {
+ if (((IncrementalSnapshotStartRecord)rec).id().equals(incSnpMeta.requestId()))
+ started = true;
+ }
+
+ if (rec.type() == WALRecord.RecordType.INCREMENTAL_SNAPSHOT_FINISH_RECORD) {
+ if (((IncrementalSnapshotFinishRecord)rec).id().equals(incSnpMeta.requestId()))
+ finished = true;
+ }
+ }
+
+ assertTrue(started);
+ assertTrue(finished);
+ }
+ catch (IOException | IgniteCheckedException | IllegalArgumentException e) {
+ assert false : "Unexpected exception while checking segments: " + e;
+ }
+ }
+
/**
* @param ignite Ignite instance to resolve discovery spi to.
* @return BlockingCustomMessageDiscoverySpi instance.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java
index af574b63b97..c9832ae4ecc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java
@@ -622,7 +622,7 @@ public class IgniteClusterSnapshotCheckTest extends AbstractSnapshotSelfTest {
corruptPartitionFile(ignite, SNAPSHOT_NAME, ccfg1, PART_ID);
- return snp(ignite).checkSnapshot(SNAPSHOT_NAME, null, cachesToCheck, false).get(TIMEOUT);
+ return snp(ignite).checkSnapshot(SNAPSHOT_NAME, null, cachesToCheck, false, 0).get(TIMEOUT);
}
/**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java
index a3b4600b5b0..13edc33a16b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java
@@ -394,7 +394,7 @@ public class IgniteClusterSnapshotHandlerTest extends IgniteClusterSnapshotResto
IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr();
- snpMgr.createSnapshot(snpName, snpDir.getAbsolutePath()).get(TIMEOUT);
+ snpMgr.createSnapshot(snpName, snpDir.getAbsolutePath(), false).get(TIMEOUT);
ignite.destroyCache(DEFAULT_CACHE_NAME);
awaitPartitionMapExchange();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java
index bb237a7d235..5574d01c059 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java
@@ -138,7 +138,9 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR
for (int i = 0; i < CACHE_KEYS_RANGE; i++)
ignite.cache(DEFAULT_CACHE_NAME).put(i, i);
- ignite.context().cache().context().snapshotMgr().createSnapshot(SNAPSHOT_NAME, snpDir.toString()).get(TIMEOUT);
+ ignite.context().cache().context().snapshotMgr()
+ .createSnapshot(SNAPSHOT_NAME, snpDir.toString(), false)
+ .get(TIMEOUT);
// Check snapshot.
IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, snpDir.getAbsolutePath()).get(TIMEOUT)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java
index 06a97542ce1..44f156e7975 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java
@@ -102,6 +102,7 @@ import static org.apache.ignite.internal.processors.cache.persistence.snapshot.I
import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+import static org.junit.Assume.assumeFalse;
/**
* Cluster-wide snapshot test.
@@ -467,7 +468,7 @@ public class IgniteClusterSnapshotSelfTest extends AbstractSnapshotSelfTest {
IgniteFuture<Void> fut = ignite.snapshot().createSnapshot(SNAPSHOT_NAME);
- spi.waitBlocked(10_000L);
+ spi.waitBlocked(TIMEOUT);
// Creating of new caches should not be blocked.
ignite.getOrCreateCache(dfltCacheCfg.setName("default2"))
@@ -503,7 +504,7 @@ public class IgniteClusterSnapshotSelfTest extends AbstractSnapshotSelfTest {
IgniteFuture<Void> fut = ignite.snapshot().createSnapshot(SNAPSHOT_NAME);
- spi.waitBlocked(10_000L);
+ spi.waitBlocked(TIMEOUT);
// Not baseline node joins successfully.
String grid4Dir = folderName(startGrid(4));
@@ -528,21 +529,34 @@ public class IgniteClusterSnapshotSelfTest extends AbstractSnapshotSelfTest {
/** @throws Exception If fails. */
@Test
public void testClusterSnapshotExOnInitiatorLeft() throws Exception {
- IgniteEx ignite = startGridsWithCache(2, dfltCacheCfg, CACHE_KEYS_RANGE);
+ for (boolean inc : new boolean[] {false, true}) {
+ IgniteEx ignite = startGridsWithCache(2, dfltCacheCfg, CACHE_KEYS_RANGE);
- BlockingCustomMessageDiscoverySpi spi = discoSpi(ignite);
- spi.block((msg) -> msg instanceof FullMessage);
+ if (inc) {
+ assumeFalse("https://issues.apache.org/jira/browse/IGNITE-17819", encryption);
- IgniteFuture<Void> fut = ignite.snapshot().createSnapshot(SNAPSHOT_NAME);
+ ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+ }
- spi.waitBlocked(10_000L);
+ BlockingCustomMessageDiscoverySpi spi = discoSpi(ignite);
+ spi.block((msg) -> msg instanceof FullMessage);
- ignite.close();
+ IgniteFuture<Void> fut = inc
+ ? ignite.snapshot().createIncrementalSnapshot(SNAPSHOT_NAME)
+ : ignite.snapshot().createSnapshot(SNAPSHOT_NAME);
- assertThrowsAnyCause(log,
- fut::get,
- NodeStoppingException.class,
- SNP_NODE_STOPPING_ERR_MSG);
+ spi.waitBlocked(TIMEOUT);
+
+ ignite.close();
+
+ assertThrowsAnyCause(log,
+ fut::get,
+ NodeStoppingException.class,
+ SNP_NODE_STOPPING_ERR_MSG);
+
+ stopAllGrids();
+ cleanPersistenceDir();
+ }
}
/** @throws Exception If fails. */
@@ -612,7 +626,7 @@ public class IgniteClusterSnapshotSelfTest extends AbstractSnapshotSelfTest {
assertTrue("Snapshot directory must be empty for node 1 due to snapshot future fail: " + dirNameIgnite1,
!searchDirectoryRecursively(locSnpDir.toPath(), dirNameIgnite1).isPresent());
- List<String> allSnapshots = snp(ignite).localSnapshotNames();
+ List<String> allSnapshots = snp(ignite).localSnapshotNames(null);
assertTrue("Snapshot directory must be empty due to snapshot fail: " + allSnapshots,
allSnapshots.isEmpty());
@@ -669,7 +683,10 @@ public class IgniteClusterSnapshotSelfTest extends AbstractSnapshotSelfTest {
awaitPartitionMapExchange();
- assertTrue("Snapshot directory must be empty", grid2.context().cache().context().snapshotMgr().localSnapshotNames().isEmpty());
+ assertTrue(
+ "Snapshot directory must be empty",
+ grid2.context().cache().context().snapshotMgr().localSnapshotNames(null).isEmpty()
+ );
ignite.snapshot().createSnapshot(SNAPSHOT_NAME)
.get();
@@ -748,7 +765,7 @@ public class IgniteClusterSnapshotSelfTest extends AbstractSnapshotSelfTest {
ignite.cache(DEFAULT_CACHE_NAME).put(i, i);
ignite.context().cache().context().snapshotMgr()
- .createSnapshot(SNAPSHOT_NAME, cfgPath ? null : snpDir.getAbsolutePath()).get();
+ .createSnapshot(SNAPSHOT_NAME, cfgPath ? null : snpDir.getAbsolutePath(), false).get();
stopAllGrids();
@@ -774,7 +791,9 @@ public class IgniteClusterSnapshotSelfTest extends AbstractSnapshotSelfTest {
GridKernalContext kctx = grid(idx).context();
assertThrowsAnyCause(log,
- () -> kctx.cache().context().snapshotMgr().createSnapshot(SNAPSHOT_NAME, invalidPath).get(TIMEOUT),
+ () -> kctx.cache().context().snapshotMgr()
+ .createSnapshot(SNAPSHOT_NAME, invalidPath, false)
+ .get(TIMEOUT),
IgniteCheckedException.class,
invalidPath);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotStreamerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotStreamerTest.java
index f0d6141dca8..ddb59e26f4c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotStreamerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotStreamerTest.java
@@ -421,11 +421,11 @@ public class IgniteClusterSnapshotStreamerTest extends AbstractSnapshotSelfTest
if (create) {
if (expWrn == null)
- snp(snpHnd).createSnapshot(SNAPSHOT_NAME, null).get();
+ snp(snpHnd).createSnapshot(SNAPSHOT_NAME, null, false).get();
else {
Throwable snpWrn = assertThrows(
null,
- () -> snp(snpHnd).createSnapshot(SNAPSHOT_NAME, null).get(),
+ () -> snp(snpHnd).createSnapshot(SNAPSHOT_NAME, null, false).get(),
IgniteException.class,
expWrn
);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotWalRecordTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotWalRecordTest.java
index cbd70523f13..6db4f0ad006 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotWalRecordTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotWalRecordTest.java
@@ -21,9 +21,12 @@ import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.configuration.CacheConfiguration;
@@ -42,9 +45,13 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.spi.systemview.view.SnapshotView;
+import org.apache.ignite.spi.systemview.view.SystemView;
import org.apache.ignite.testframework.GridTestUtils;
import org.junit.Test;
+import static org.apache.ignite.spi.systemview.view.SnapshotView.SNAPSHOT_SYS_VIEW;
+
/** */
public class IgniteClusterSnapshotWalRecordTest extends AbstractSnapshotSelfTest {
/**
@@ -58,23 +65,33 @@ public class IgniteClusterSnapshotWalRecordTest extends AbstractSnapshotSelfTest
CountDownLatch loadStopLatch = new CountDownLatch(1);
// Start changing data concurrently with performing the ClusterSnapshot operation.
- IgniteInternalFuture<?> loadFut = GridTestUtils.runMultiThreadedAsync(() -> {
- Random r = new Random();
+ IgniteInternalFuture<?> loadFut = null;
- while (loadStopLatch.getCount() > 0) {
- int key = r.nextInt(CACHE_KEYS_RANGE);
+ try {
+ loadFut = GridTestUtils.runMultiThreadedAsync(() -> {
+ Random r = new Random();
- Account acc = new Account(r.nextInt(), r.nextInt());
+ while (loadStopLatch.getCount() > 0 && !Thread.interrupted()) {
+ int key = r.nextInt(CACHE_KEYS_RANGE);
- ign.cache(DEFAULT_CACHE_NAME).put(key, acc);
- }
- }, 5, "cache-loader-");
+ Account acc = new Account(r.nextInt(), r.nextInt());
- snp(ign).createSnapshot(SNAPSHOT_NAME).get();
+ ign.cache(DEFAULT_CACHE_NAME).put(key, acc);
+ }
+ }, 5, "cache-loader-");
- loadStopLatch.countDown();
+ snp(ign).createSnapshot(SNAPSHOT_NAME).get();
- loadFut.get();
+ loadStopLatch.countDown();
+
+ loadFut.get();
+ }
+ catch (Throwable err) {
+ if (loadFut != null)
+ loadFut.cancel();
+
+ throw err;
+ }
T2<Map<Integer, Account>, Map<Integer, Account>> data = parseWalCacheState(ign, SNAPSHOT_NAME);
@@ -93,6 +110,58 @@ public class IgniteClusterSnapshotWalRecordTest extends AbstractSnapshotSelfTest
assertCacheKeys(snpIgn.cache(DEFAULT_CACHE_NAME), snpData);
}
+ /** */
+ @Test
+ public void testClusterSnapshotRecordIsWrittenToSnapshotMetadata() throws Exception {
+ int nodes = 3;
+ int snapshots = 10;
+
+ startGridsWithCache(nodes, 1, key -> new Account(key, key),
+ new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+ );
+
+ for (int i = 0; i < snapshots; i++) {
+ // Start changing data concurrently with performing the ClusterSnapshot operation.
+ snp(grid(0)).createSnapshot(SNAPSHOT_NAME + i).get();
+ }
+
+ for (int i = 0; i < nodes; i++) {
+ IgniteEx ign = grid(i);
+
+ ign.context().cache().context().wal().flush(null, true);
+
+ WALIterator walIt = wal(grid(i));
+
+ long snpCnt = 0;
+
+ SystemView<SnapshotView> snpView = ign.context().systemView().view(SNAPSHOT_SYS_VIEW);
+
+ for (IgniteBiTuple<WALPointer, WALRecord> tuple: walIt) {
+ WALRecord rec = tuple.getValue();
+
+ if (rec.type() == WALRecord.RecordType.CLUSTER_SNAPSHOT) {
+ SnapshotMetadata metadata = snp(grid(i)).readSnapshotMetadata(
+ snp(grid(i)).snapshotLocalDir(SNAPSHOT_NAME + snpCnt),
+ (String)grid(i).configuration().getConsistentId());
+
+ assertEquals(tuple.getKey(), metadata.snapshotRecordPointer());
+
+ List<SnapshotView> snpNodesView = StreamSupport.stream(snpView.spliterator(), false)
+ .filter(v -> v.name().equals(metadata.snapshotName()))
+ .filter(v -> v.consistentId().equals(ign.localNode().consistentId()))
+ .filter(v -> v.snapshotRecordSegment().equals(tuple.getKey().index()))
+ .collect(Collectors.toList());
+
+ assertEquals(1, snpNodesView.size());
+
+ snpCnt++;
+ }
+ }
+
+ assertEquals(snapshots, snpCnt);
+ }
+ }
+
/**
* Parsing WAL files and dumping cache states: fisrst is before {@link ClusterSnapshotRecord} was written, and second
* is after all load operations stopped.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java
index 01e25c398b2..74c9d4e9293 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java
@@ -24,6 +24,7 @@ import javax.management.MBeanException;
import javax.management.ReflectionException;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
@@ -42,6 +43,7 @@ import static org.apache.ignite.internal.util.distributed.DistributedProcess.Dis
import static org.apache.ignite.testframework.GridTestUtils.assertContains;
import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+import static org.junit.Assume.assumeFalse;
/**
* Tests {@link SnapshotMXBean}.
@@ -73,6 +75,15 @@ public class IgniteSnapshotMXBeanTest extends AbstractSnapshotSelfTest {
assertTrue("Waiting for snapshot operation failed.",
GridTestUtils.waitForCondition(() -> (long)getMetric("LastSnapshotEndTime", snpMBean) > 0, TIMEOUT));
+ if (!encryption) {
+ mxBean.createIncrementalSnapshot(SNAPSHOT_NAME, "");
+
+ assertTrue(
+ "Waiting for incremental snapshot failed",
+ GridTestUtils.waitForCondition(() -> checkIncremental(ignite, SNAPSHOT_NAME, null, 1), TIMEOUT)
+ );
+ }
+
stopAllGrids();
IgniteEx snp = startGridsFromSnapshot(2, SNAPSHOT_NAME);
@@ -116,6 +127,41 @@ public class IgniteSnapshotMXBeanTest extends AbstractSnapshotSelfTest {
assertCacheKeys(ignite.cache(DEFAULT_CACHE_NAME), CACHE_KEYS_RANGE);
}
+ /** @throws Exception If fails. */
+ @Test
+ public void testRestoreIncrementalSnapshot() throws Exception {
+ assumeFalse("https://issues.apache.org/jira/browse/IGNITE-17819", encryption);
+
+ IgniteEx ignite = startGridsWithCache(2, CACHE_KEYS_RANGE, valueBuilder(), dfltCacheCfg);
+
+ ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ try (IgniteDataStreamer<Integer, Object> ds = ignite.dataStreamer(dfltCacheCfg.getName())) {
+ for (int i = CACHE_KEYS_RANGE; i < 2 * CACHE_KEYS_RANGE; i++)
+ ds.addData(i, valueBuilder().apply(i));
+ }
+
+ ignite.snapshot().createIncrementalSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ ignite.cache(dfltCacheCfg.getName()).destroy();
+
+ awaitPartitionMapExchange();
+
+ DynamicMBean mReg0 = metricRegistry(grid(0).name(), null, SNAPSHOT_RESTORE_METRICS);
+ DynamicMBean mReg1 = metricRegistry(grid(1).name(), null, SNAPSHOT_RESTORE_METRICS);
+
+ assertEquals(0, (long)getMetric("endTime", mReg0));
+ assertEquals(0, (long)getMetric("endTime", mReg1));
+
+ getMxBean(ignite.name(), SNAPSHOT_GROUP, SnapshotMXBeanImpl.class, SnapshotMXBean.class)
+ .restoreSnapshot(SNAPSHOT_NAME, "", "", 1);
+
+ assertTrue(GridTestUtils.waitForCondition(() -> (long)getMetric("endTime", mReg0) > 0, getTestTimeout()));
+ assertTrue(GridTestUtils.waitForCondition(() -> (long)getMetric("endTime", mReg1) > 0, getTestTimeout()));
+
+ assertCacheKeys(ignite.cache(DEFAULT_CACHE_NAME), 2 * CACHE_KEYS_RANGE);
+ }
+
/** @throws Exception If fails. */
@Test
public void testCancelRestoreSnapshot() throws Exception {
@@ -172,8 +218,9 @@ public class IgniteSnapshotMXBeanTest extends AbstractSnapshotSelfTest {
public void testStatus() throws Exception {
IgniteEx srv = startGridsWithCache(2, dfltCacheCfg, CACHE_KEYS_RANGE);
- checkSnapshotStatus(false, false, null);
+ checkSnapshotStatus(false, false, false, null);
+ // Create full snapshot.
TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(grid(1));
spi.blockMessages((node, msg) -> msg instanceof SingleNodeMessage);
@@ -182,14 +229,33 @@ public class IgniteSnapshotMXBeanTest extends AbstractSnapshotSelfTest {
spi.waitForBlocked();
- checkSnapshotStatus(true, false, SNAPSHOT_NAME);
+ checkSnapshotStatus(true, false, false, SNAPSHOT_NAME);
spi.stopBlock();
fut.get(getTestTimeout());
- checkSnapshotStatus(false, false, null);
+ checkSnapshotStatus(false, false, false, null);
+
+ // Create incremental snapshot.
+ // TODO: remove condition after resolving IGNITE-17819.
+ if (!encryption) {
+ spi.blockMessages((node, msg) -> msg instanceof SingleNodeMessage);
+
+ fut = srv.snapshot().createIncrementalSnapshot(SNAPSHOT_NAME);
+
+ spi.waitForBlocked();
+
+ checkSnapshotStatus(true, false, true, SNAPSHOT_NAME);
+
+ spi.stopBlock();
+
+ fut.get(getTestTimeout());
+
+ checkSnapshotStatus(false, false, false, null);
+ }
+ // Restore full snapshot.
srv.destroyCache(DEFAULT_CACHE_NAME);
spi.blockMessages((node, msg) -> msg instanceof SingleNodeMessage);
@@ -198,21 +264,42 @@ public class IgniteSnapshotMXBeanTest extends AbstractSnapshotSelfTest {
spi.waitForBlocked();
- checkSnapshotStatus(false, true, SNAPSHOT_NAME);
+ checkSnapshotStatus(false, true, false, SNAPSHOT_NAME);
spi.stopBlock();
fut.get(getTestTimeout());
- checkSnapshotStatus(false, false, null);
+ checkSnapshotStatus(false, false, false, null);
+
+ // Restore incremental snapshot.
+ // TODO: remove condition after resolving IGNITE-17819.
+ if (!encryption) {
+ srv.destroyCache(DEFAULT_CACHE_NAME);
+
+ spi.blockMessages((node, msg) -> msg instanceof SingleNodeMessage);
+
+ fut = srv.snapshot().restoreSnapshot(SNAPSHOT_NAME, F.asList(DEFAULT_CACHE_NAME), 1);
+
+ spi.waitForBlocked();
+
+ checkSnapshotStatus(false, true, true, SNAPSHOT_NAME);
+
+ spi.stopBlock();
+
+ fut.get(getTestTimeout());
+
+ checkSnapshotStatus(false, false, false, null);
+ }
}
/**
* @param isCreating {@code True} if create snapshot operation is in progress.
* @param isRestoring {@code True} if restore snapshot operation is in progress.
+ * @param isIncremental {@code True} if incremental snapshot operation.
* @param expName Expected snapshot name.
*/
- private void checkSnapshotStatus(boolean isCreating, boolean isRestoring, String expName) throws Exception {
+ private void checkSnapshotStatus(boolean isCreating, boolean isRestoring, boolean isIncremental, String expName) throws Exception {
assertTrue(waitForCondition(() -> G.allGrids().stream().allMatch(
ignite -> {
IgniteSnapshotManager mgr = ((IgniteEx)ignite).context().cache().context().snapshotMgr();
@@ -238,6 +325,10 @@ public class IgniteSnapshotMXBeanTest extends AbstractSnapshotSelfTest {
assertContains(log, status, "Restore snapshot operation is in progress");
assertContains(log, status, "name=" + expName);
+ assertContains(log, status, "incremental=" + isIncremental);
+
+ if (isIncremental)
+ assertContains(log, status, "incrementIndex=1");
}
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotMetricTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotMetricTest.java
new file mode 100644
index 00000000000..42408e452cb
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotMetricTest.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotStartRecord;
+import org.apache.ignite.internal.pagemem.wal.record.RolloverType;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.incremental.AbstractIncrementalSnapshotTest;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
+import org.apache.ignite.internal.processors.metric.MetricRegistry;
+import org.apache.ignite.internal.processors.metric.impl.ObjectGauge;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.plugin.AbstractTestPluginProvider;
+import org.apache.ignite.plugin.PluginContext;
+import org.apache.ignite.spi.metric.IntMetric;
+import org.apache.ignite.spi.metric.LongMetric;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.jetbrains.annotations.Nullable;
+import org.junit.Test;
+
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.snp;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.INCREMENTAL_SNAPSHOT_METRICS;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotRestoreProcess.SNAPSHOT_RESTORE_METRICS;
+
+/** */
+public class IncrementalSnapshotMetricTest extends AbstractIncrementalSnapshotTest {
+ /** */
+ private static CountDownLatch beforeFinRecLatch;
+
+ /** */
+ private static CountDownLatch logFinRecLatch;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String instanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(instanceName);
+
+ if (getTestIgniteInstanceIndex(instanceName) == 0)
+ cfg.setPluginProviders(new BlockingWALPluginProvider());
+
+ return cfg;
+ }
+
+ /** @throws Exception If fails. */
+ @Test
+ public void testCreateIncrementalSnapshotMetrics() throws Exception {
+ beforeFinRecLatch = new CountDownLatch(1);
+ logFinRecLatch = new CountDownLatch(1);
+
+ MetricRegistry mreg0 = grid(0).context().metric().registry(INCREMENTAL_SNAPSHOT_METRICS);
+
+ LongMetric startTime = mreg0.findMetric("startTime");
+ LongMetric endTime = mreg0.findMetric("endTime");
+ IntMetric incIdx = mreg0.findMetric("incrementIndex");
+ ObjectGauge<String> snpName = mreg0.findMetric("snapshotName");
+ ObjectGauge<String> errMsg = mreg0.findMetric("error");
+
+ assertEquals(0, startTime.value());
+ assertEquals(0, endTime.value());
+ assertEquals(0, incIdx.value());
+ assertTrue(snpName.value().isEmpty());
+ assertTrue(errMsg.value().isEmpty());
+
+ IgniteCache<Integer, Integer> cache = grid(0).cache(CACHE);
+
+ for (int i = 0; i < 1024; i++)
+ cache.put(i, ThreadLocalRandom.current().nextInt());
+
+ IgniteFuture<Void> incSnpFut = grid(0).snapshot().createIncrementalSnapshot(SNP);
+
+ assertTrue(GridTestUtils.waitForCondition(
+ () -> snp(grid(0)).currentSnapshotTask(IncrementalSnapshotFutureTask.class) != null,
+ 5_000, 10));
+
+ U.await(logFinRecLatch);
+
+ assertTrue(startTime.value() > 0);
+ assertEquals(0, endTime.value());
+ assertEquals(1, incIdx.value());
+ assertEquals(SNP, snpName.value());
+ assertTrue(errMsg.value().isEmpty());
+
+ beforeFinRecLatch.countDown();
+
+ incSnpFut.get(getTestTimeout());
+
+ assertTrue(startTime.value() > 0);
+ assertTrue(endTime.value() > 0);
+ assertEquals(1, incIdx.value());
+ assertEquals(SNP, snpName.value());
+ assertTrue(errMsg.value().isEmpty());
+
+ stopGrid(1);
+
+ GridTestUtils.assertThrows(log,
+ () -> grid(0).snapshot().createIncrementalSnapshot(SNP).get(getTestTimeout()),
+ IgniteException.class,
+ null);
+
+ assertFalse(errMsg.value().isEmpty());
+ }
+
+ /** @throws Exception If fails. */
+ @Test
+ public void testRestoreIncrementalSnapshotMetrics() throws Exception {
+ IgniteCache<Integer, Integer> cache = grid(0).cache(CACHE);
+
+ for (int i = 0; i < 1024; i++) {
+ if (i == 512)
+ rollWalSegment(grid(0));
+
+ cache.put(i, ThreadLocalRandom.current().nextInt());
+ }
+
+ grid(0).snapshot().createIncrementalSnapshot(SNP).get(getTestTimeout());
+
+ restartWithCleanPersistence(nodes(), Collections.singletonList(CACHE));
+
+ MetricRegistry mreg0 = grid(0).context().metric().registry(SNAPSHOT_RESTORE_METRICS);
+
+ IntMetric incIdx = mreg0.findMetric("incrementIndex");
+ IntMetric totalWalSeg = mreg0.findMetric("totalWalSegments");
+ IntMetric procWalSeg = mreg0.findMetric("processedWalSegments");
+ LongMetric procEntries = mreg0.findMetric("processedWalEntries");
+
+ assertEquals(0, incIdx.value());
+ assertEquals(-1, totalWalSeg.value());
+ assertEquals(-1, procWalSeg.value());
+ assertEquals(-1, procEntries.value());
+
+ grid(0).snapshot().restoreSnapshot(SNP, null, 1).get(getTestTimeout());
+
+ assertEquals(1, incIdx.value());
+ assertEquals(2, totalWalSeg.value());
+ assertEquals(2, procWalSeg.value());
+ assertEquals(1024L, procEntries.value()); // 1 primary and 2 backups.
+ }
+
+ /** */
+ private static class BlockingWALPluginProvider extends AbstractTestPluginProvider {
+ /** {@inheritDoc} */
+ @Override public String name() {
+ return "BlockingWALProvider";
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T> @Nullable T createComponent(PluginContext ctx, Class<T> cls) {
+ if (IgniteWriteAheadLogManager.class.equals(cls))
+ return (T)new BlockingWALManager(((IgniteEx)ctx.grid()).context());
+
+ return null;
+ }
+ }
+
+ /** Blocks writing to WAL {@link IncrementalSnapshotStartRecord}. */
+ protected static class BlockingWALManager extends FileWriteAheadLogManager {
+ /** */
+ public BlockingWALManager(GridKernalContext ctx) {
+ super(ctx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public WALPointer log(WALRecord record, RolloverType rolloverType) throws IgniteCheckedException, StorageException {
+ if (record.type() == WALRecord.RecordType.INCREMENTAL_SNAPSHOT_FINISH_RECORD && logFinRecLatch != null) {
+ logFinRecLatch.countDown();
+
+ U.awaitQuiet(beforeFinRecLatch);
+ }
+
+ return super.log(record, rolloverType);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int nodes() {
+ return 3;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int backups() {
+ return 2;
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotRebalanceTest.java
new file mode 100644
index 00000000000..a99cac9d1f6
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotRebalanceTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.incremental.AbstractIncrementalSnapshotTest;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.Test;
+
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.snp;
+
+/** */
+public class IncrementalSnapshotRebalanceTest extends AbstractIncrementalSnapshotTest {
+ /** */
+ private static final String CACHE2 = "CACHE2";
+
+ /** */
+ @Override protected IgniteConfiguration getConfiguration(String instanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(instanceName);
+
+ cfg.setCacheConfiguration(cacheConfiguration(CACHE), cacheConfiguration(CACHE2));
+
+ return cfg;
+ }
+
+ /** */
+ @Test
+ public void createIncSnpAfterClusterRestartIsAllowed() throws Exception {
+ stopAllGrids();
+
+ Ignite ign = startGrids(3);
+
+ awaitPartitionMapExchange();
+
+ ign.cluster().state(ClusterState.ACTIVE);
+
+ snp(grid(0)).createIncrementalSnapshot(SNP).get(getTestTimeout());
+ }
+
+ /** */
+ @Test
+ public void skipNonBaselineNodesDoesNotForbidIncSnapshot() throws Exception {
+ startGrid(nodes() + 2);
+
+ for (int i = 0; i < 1_000; i++)
+ grid(0).cache(CACHE).put(i, i);
+
+ stopGrid(nodes() + 1);
+ stopGrid(nodes() + 2);
+
+ awaitPartitionMapExchange();
+
+ snp(grid(0)).createIncrementalSnapshot(SNP).get(getTestTimeout());
+ }
+
+ /** */
+ @Test
+ public void createIncSnpAfterRebalanceIsForbidden() throws Exception {
+ for (int i = 0; i < 1_000; i++)
+ grid(0).cache(CACHE).put(i, i);
+
+ stopGrid(1);
+
+ awaitPartitionMapExchange();
+
+ GridTestUtils.assertThrows(log, () -> snp(grid(0)).createIncrementalSnapshot(SNP).get(getTestTimeout()),
+ IgniteException.class, "Create incremental snapshot request has been rejected. Node from full snapshot offline");
+
+ for (int i = 1_000; i < 2_000; i++)
+ grid(0).cache(CACHE).put(i, i);
+
+ startGrid(1);
+
+ awaitPartitionMapExchange();
+
+ GridTestUtils.assertThrows(log, () -> snp(grid(0)).createIncrementalSnapshot(SNP).get(getTestTimeout()),
+ IgniteException.class, "WAL was disabled");
+
+ stopAllGrids();
+
+ startGrids(nodes());
+
+ awaitPartitionMapExchange();
+
+ GridTestUtils.assertThrows(log, () -> snp(grid(0)).createIncrementalSnapshot(SNP).get(getTestTimeout()),
+ IgniteException.class, "WAL was disabled");
+ }
+
+ /** */
+ @Test
+ public void forbidIncSnpCreationAfterRestore() throws Exception {
+ for (int i = 0; i < 1_000; i++)
+ grid(0).cache(CACHE).put(i, i);
+
+ snp(grid(0)).createIncrementalSnapshot(SNP).get(getTestTimeout());
+
+ restartWithCleanPersistence(nodes(), F.asList(CACHE, CACHE2));
+
+ snp(grid(0)).restoreSnapshot(SNP, null, 1).get(getTestTimeout());
+
+ GridTestUtils.assertThrows(log, () -> snp(grid(0)).createIncrementalSnapshot(SNP).get(getTestTimeout()),
+ IgniteException.class, "WAL was disabled");
+ }
+
+ /** */
+ @Test
+ public void forbidIncSnpCreationAllowedAfterFullSnapshot() throws Exception {
+ for (int i = 0; i < 1_000; i++)
+ grid(0).cache(CACHE).put(i, i);
+
+ snp(grid(0)).createIncrementalSnapshot(SNP).get(getTestTimeout());
+
+ restartWithCleanPersistence(nodes(), F.asList(CACHE, CACHE2));
+
+ snp(grid(0)).restoreSnapshot(SNP, null, 1).get(getTestTimeout());
+
+ GridTestUtils.assertThrows(log, () -> snp(grid(0)).createIncrementalSnapshot(SNP).get(getTestTimeout()),
+ IgniteException.class, "WAL was disabled");
+
+ snp(grid(0)).createSnapshot(SNP + "0").get(getTestTimeout());
+ snp(grid(0)).createIncrementalSnapshot(SNP + "0").get(getTestTimeout());
+ }
+
+ /** */
+ @Test
+ public void incSnpIsNotAllowedIfOnlyOneCacheWalIsDisabled() throws Exception {
+ for (int i = 0; i < 1_000; i++)
+ grid(0).cache(CACHE).put(i, i);
+
+ snp(grid(0)).createIncrementalSnapshot(SNP).get(getTestTimeout());
+
+ stopGrid(1);
+
+ for (int i = 0; i < 1_000; i++)
+ grid(0).cache(CACHE2).put(i, i);
+
+ startGrid(1);
+
+ awaitPartitionMapExchange();
+
+ GridTestUtils.assertThrows(log, () -> snp(grid(0)).createIncrementalSnapshot(SNP).get(getTestTimeout()),
+ IgniteException.class, "WAL was disabled");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int nodes() {
+ return 3;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int backups() {
+ return 2;
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotTest.java
new file mode 100644
index 00000000000..3dbf3f54e89
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotTest.java
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.UnaryOperator;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridLocalConfigManager;
+import org.apache.ignite.internal.processors.cache.StoredCacheData;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.junit.Test;
+
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.snapshotMetaFileName;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+import static org.junit.Assume.assumeFalse;
+
+/**
+ * Basic tests for incremental snapshots.
+ */
+public class IncrementalSnapshotTest extends AbstractSnapshotSelfTest {
+ /** */
+ public static final int GRID_CNT = 3;
+
+ /** */
+ public static final String OTHER_CACHE = "other-cache";
+
+ /** */
+ public static final String GROUPED_CACHE = "my-grouped-cache2";
+
+ /** @see DataStorageConfiguration#isWalCompactionEnabled() */
+ public boolean walCompactionEnabled = true;
+
+ /** */
+ private AtomicInteger cntr = new AtomicInteger();
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.getDataStorageConfiguration()
+ .setWalCompactionEnabled(walCompactionEnabled)
+ .setWalSegmentSize((int)U.MB);
+
+ return cfg;
+ }
+
+ /** */
+ @Test
+ public void testCreation() throws Exception {
+ assumeFalse("https://issues.apache.org/jira/browse/IGNITE-17819", encryption);
+
+ IgniteEx srv = startGridsWithCache(
+ GRID_CNT,
+ CACHE_KEYS_RANGE,
+ key -> new Account(key, key),
+ new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+ );
+
+ File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true);
+
+ assertTrue("Target directory is not empty: " + snpDir, F.isEmpty(snpDir.list()));
+
+ IgniteEx cli = startClientGrid(
+ GRID_CNT,
+ (UnaryOperator<IgniteConfiguration>)
+ cfg -> cfg.setCacheConfiguration(new CacheConfiguration<>(DEFAULT_CACHE_NAME))
+ );
+
+ File exSnpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true);
+
+ for (boolean client : new boolean[] {false, true}) {
+ IgniteSnapshotManager snpCreate = snp(client ? cli : srv);
+
+ for (File snpPath : new File[] {null, exSnpDir}) {
+ if (client && snpPath != null) // Set snapshot path not supported for snapshot from client nodes.
+ continue;
+
+ String snpName = SNAPSHOT_NAME + "_" + client + "_" + (snpPath == null ? "" : snpPath.getName());
+
+ if (snpPath == null)
+ snpCreate.createSnapshot(snpName).get(TIMEOUT);
+ else
+ snpCreate.createSnapshot(snpName, snpPath.getAbsolutePath(), false).get(TIMEOUT);
+
+ for (int incIdx = 1; incIdx < 3; incIdx++) {
+ addData(cli);
+
+ if (snpPath == null)
+ snpCreate.createIncrementalSnapshot(snpName).get(TIMEOUT);
+ else
+ snpCreate.createSnapshot(snpName, snpPath.getAbsolutePath(), true).get(TIMEOUT);
+
+ for (int gridIdx = 0; gridIdx < GRID_CNT; gridIdx++) {
+ assertTrue("Incremental snapshot must exists on node " + gridIdx, checkIncremental(
+ grid(gridIdx),
+ snpName,
+ snpPath == null ? null : snpPath.getAbsolutePath(),
+ incIdx
+ ));
+ }
+ }
+ }
+ }
+ }
+
+ /** */
+ private void addData(IgniteEx node) {
+ IgniteCache<Integer, byte[]> cache = node.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+ for (int i = 0; i < (int)(U.MB * 3 * GRID_CNT / U.KB); i++) {
+ byte[] bytes = new byte[(int)U.KB];
+
+ ThreadLocalRandom.current().nextBytes(bytes);
+
+ cache.put(cntr.incrementAndGet(), bytes);
+ }
+ }
+
+ /** */
+ @Test
+ public void testFailForUnknownBaseSnapshot() throws Exception {
+ assumeFalse("https://issues.apache.org/jira/browse/IGNITE-17819", encryption);
+
+ IgniteEx ign = startGridsWithCache(1, CACHE_KEYS_RANGE, key -> new Account(key, key),
+ new CacheConfiguration<>(DEFAULT_CACHE_NAME));
+
+ assertThrowsWithCause(
+ () -> snp(ign).createIncrementalSnapshot("unknown").get(TIMEOUT),
+ IgniteException.class
+ );
+
+ snp(ign).createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ assertThrowsWithCause(
+ () -> snp(ign).createIncrementalSnapshot("unknown").get(TIMEOUT),
+ IgniteException.class
+ );
+ }
+
+ /** */
+ @Test
+ public void testFailIfPreviousIncrementNotAvailable() throws Exception {
+ assumeFalse("https://issues.apache.org/jira/browse/IGNITE-17819", encryption);
+
+ IgniteEx srv = startGridsWithCache(
+ GRID_CNT,
+ CACHE_KEYS_RANGE,
+ key -> new Account(key, key),
+ new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+ );
+
+ IgniteEx cli = startClientGrid(
+ GRID_CNT,
+ (UnaryOperator<IgniteConfiguration>)
+ cfg -> cfg.setCacheConfiguration(new CacheConfiguration<>(DEFAULT_CACHE_NAME))
+ );
+
+ cli.snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ cli.snapshot().createIncrementalSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+ cli.snapshot().createIncrementalSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ File toRmv = new File(
+ snp(ignite(GRID_CNT - 1)).incrementalSnapshotLocalDir(SNAPSHOT_NAME, null, 2),
+ snapshotMetaFileName(ignite(GRID_CNT - 1).localNode().consistentId().toString()));
+
+ assertTrue(toRmv.exists());
+
+ U.delete(toRmv);
+
+ assertThrowsWithCause(
+ () -> cli.snapshot().createIncrementalSnapshot(SNAPSHOT_NAME).get(TIMEOUT),
+ IgniteException.class
+ );
+ }
+
+ /** */
+ @Test
+ public void testFailIfSegmentNotFound() throws Exception {
+ assumeFalse("https://issues.apache.org/jira/browse/IGNITE-17819", encryption);
+
+ IgniteEx srv = startGridsWithCache(
+ 1,
+ CACHE_KEYS_RANGE,
+ key -> new Account(key, key),
+ new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+ );
+
+ srv.snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ addData(srv);
+
+ FileWriteAheadLogManager wal = (FileWriteAheadLogManager)srv.context().cache().context().wal();
+
+ assertTrue(waitForCondition(() -> wal.lastCompactedSegment() >= 0, getTestTimeout()));
+
+ long segIdx = wal.lastCompactedSegment();
+
+ U.delete(wal.compactedSegment(segIdx));
+
+ assertThrowsWithCause(
+ () -> srv.snapshot().createIncrementalSnapshot(SNAPSHOT_NAME).get(TIMEOUT),
+ IgniteException.class
+ );
+ }
+
+ /** */
+ @Test
+ public void testIncrementalSnapshotFailsOnTopologyChange() throws Exception {
+ assumeFalse("https://issues.apache.org/jira/browse/IGNITE-17819", encryption);
+
+ IgniteEx srv = startGridsWithCache(
+ GRID_CNT,
+ CACHE_KEYS_RANGE,
+ key -> new Account(key, key),
+ new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+ );
+
+ IgniteSnapshotManager snpCreate = snp(srv);
+
+ snpCreate.createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ String consId = grid(1).context().discovery().localNode().consistentId().toString();
+
+ // Stop some node.
+ stopGrid(1);
+
+ assertThrows(
+ null,
+ () -> snpCreate.createIncrementalSnapshot(SNAPSHOT_NAME).get(TIMEOUT),
+ IgniteException.class,
+ "Create incremental snapshot request has been rejected. " +
+ "Node from full snapshot offline [consistentId=" + consId + ']'
+ );
+ }
+
+ /** */
+ @Test
+ public void testIncrementalSnapshotFailsOnCacheDestroy() throws Exception {
+ assumeFalse("https://issues.apache.org/jira/browse/IGNITE-17819", encryption);
+
+ checkFailWhenCacheDestroyed(OTHER_CACHE, "Create incremental snapshot request has been rejected. " +
+ "Cache group destroyed [groupId=" + CU.cacheId(OTHER_CACHE) + ']');
+ }
+
+ /** */
+ @Test
+ public void testIncrementalSnapshotFailsOnGroupedCacheDestroy() throws Exception {
+ assumeFalse("https://issues.apache.org/jira/browse/IGNITE-17819", encryption);
+
+ checkFailWhenCacheDestroyed(GROUPED_CACHE, "Create incremental snapshot request has been rejected. " +
+ "Cache destroyed [cacheId=" + CU.cacheId(GROUPED_CACHE) + ", cacheName=" + GROUPED_CACHE + ']');
+ }
+
+ /** */
+ @Test
+ public void testIncrementalSnapshotFailsOnCacheChange() throws Exception {
+ assumeFalse("https://issues.apache.org/jira/browse/IGNITE-17819", encryption);
+
+ CacheConfiguration<Integer, Account> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
+
+ IgniteEx srv = startGridsWithCache(1, CACHE_KEYS_RANGE, key -> new Account(key, key), ccfg);
+
+ snp(srv).createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ GridLocalConfigManager locCfgMgr = srv.context().cache().configManager();
+
+ File ccfgFile = locCfgMgr.cacheConfigurationFile(ccfg);
+
+ StoredCacheData cacheData = locCfgMgr.readCacheData(ccfgFile);
+
+ assertNotNull(cacheData);
+
+ cacheData.queryEntities(Collections.singletonList(new QueryEntity(String.class, Account.class)));
+
+ locCfgMgr.writeCacheData(cacheData, ccfgFile);
+
+ assertThrows(
+ null,
+ () -> snp(srv).createIncrementalSnapshot(SNAPSHOT_NAME).get(TIMEOUT),
+ IgniteException.class,
+ "Cache changed [cacheId=" + CU.cacheId(DEFAULT_CACHE_NAME) + ", cacheName=" + DEFAULT_CACHE_NAME + ']'
+ );
+ }
+
+ /** */
+ @Test
+ public void testIncrementalSnapshotFailOnDirtyDir() throws Exception {
+ assumeFalse("https://issues.apache.org/jira/browse/IGNITE-17819", encryption);
+
+ IgniteEx srv = startGridsWithCache(
+ GRID_CNT,
+ CACHE_KEYS_RANGE,
+ key -> new Account(key, key),
+ new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+ );
+
+ snp(srv).createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ assertTrue(snp(srv).incrementalSnapshotsLocalRootDir(SNAPSHOT_NAME, null).mkdirs());
+ assertTrue(snp(srv).incrementalSnapshotLocalDir(SNAPSHOT_NAME, null, 1).createNewFile());
+
+ assertThrows(
+ null,
+ () -> snp(srv).createIncrementalSnapshot(SNAPSHOT_NAME).get(TIMEOUT),
+ IgniteException.class,
+ "Failed to create snapshot WAL directory"
+ );
+
+ for (int i = 0; i < GRID_CNT; i++)
+ assertFalse(snp(grid(i)).incrementalSnapshotLocalDir(SNAPSHOT_NAME, null, 1).exists());
+ }
+
+ /** */
+ @Test
+ public void testFailIfWalCompactionDisabled() throws Exception {
+ assumeFalse("https://issues.apache.org/jira/browse/IGNITE-17819", encryption);
+
+ walCompactionEnabled = false;
+
+ try {
+ IgniteEx srv = startGridsWithCache(
+ 1,
+ CACHE_KEYS_RANGE,
+ key -> new Account(key, key),
+ new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+ );
+
+ srv.snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ assertThrows(
+ null,
+ () -> srv.snapshot().createIncrementalSnapshot(SNAPSHOT_NAME).get(TIMEOUT),
+ IgniteException.class,
+ "Create incremental snapshot request has been rejected. WAL compaction must be enabled."
+ );
+ }
+ finally {
+ walCompactionEnabled = true;
+ }
+
+ }
+
+ /** */
+ private void checkFailWhenCacheDestroyed(String cache2rvm, String errMsg) throws Exception {
+ IgniteEx srv = startGridsWithCache(
+ 1,
+ CACHE_KEYS_RANGE,
+ key -> new Account(key, key),
+ new CacheConfiguration<>(DEFAULT_CACHE_NAME),
+ new CacheConfiguration<>(OTHER_CACHE),
+ new CacheConfiguration<Integer, Object>("my-grouped-cache1").setGroupName("mygroup"),
+ new CacheConfiguration<Integer, Object>(GROUPED_CACHE).setGroupName("mygroup")
+ );
+
+ IgniteSnapshotManager snpCreate = snp(srv);
+
+ snpCreate.createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ snpCreate.createIncrementalSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ srv.destroyCache(cache2rvm);
+
+ assertThrows(
+ null,
+ () -> snpCreate.createIncrementalSnapshot(SNAPSHOT_NAME).get(TIMEOUT),
+ IgniteException.class,
+ errMsg
+ );
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotBlockingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotBlockingTest.java
new file mode 100644
index 00000000000..e295a414814
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotBlockingTest.java
@@ -0,0 +1,372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot.incremental;
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotStartRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T3;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.plugin.AbstractTestPluginProvider;
+import org.apache.ignite.plugin.PluginContext;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.snp;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.incremental.AbstractIncrementalSnapshotBlockingTest.BlkNodeType.NEAR;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.incremental.AbstractIncrementalSnapshotBlockingTest.BlkNodeType.PRIMARY;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.incremental.AbstractIncrementalSnapshotBlockingTest.BlkSnpType.AFTER_START;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.incremental.AbstractIncrementalSnapshotBlockingTest.BlkSnpType.BEFORE_START;
+import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
+
+/** Base class for testing incremental snapshot blocking some events. */
+public abstract class AbstractIncrementalSnapshotBlockingTest extends AbstractIncrementalSnapshotTest {
+ /** Number of current testing case. */
+ protected int caseNum;
+
+ /** */
+ protected static BlkNodeType snpBlkNodeType;
+
+ /** */
+ protected static BlkSnpType snpBlkType;
+
+ /** */
+ protected static BlkNodeType txBlkNodeType;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String instanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(instanceName);
+
+ cfg.setPluginProviders(
+ new BlockingWALPluginProvider(),
+ new BlockingSnapshotPluginProvider());
+
+ return cfg;
+ }
+
+ /**
+ * Run test cases.
+ *
+ * @param cases Collection of cases. Every case is a collection of key description - tuple (primaryNodeId, backupNodeId).
+ */
+ protected void runCases(List<TransactionTestCase> cases) throws Exception {
+ // Ignite coordinator, ordinary server node, client node.
+ // If test with stopping client node then use it as near node.
+ List<Integer> nears = F.asList(0, nodes() - 1, nodes());
+
+ for (int near: nears) {
+ // Client nodes doesn't prepare anything after version update.
+ if (near == nodes() && snpBlkNodeType == NEAR && snpBlkType == AFTER_START)
+ continue;
+
+ for (int c = 0; c < cases.size(); c++) {
+ runCase(cases.get(c), near, TransactionConcurrency.PESSIMISTIC);
+ runCase(cases.get(c), near, TransactionConcurrency.OPTIMISTIC);
+ }
+ }
+ }
+
+ /** Run single test cases. */
+ protected abstract void runCase(TransactionTestCase testCase, int nearNodeIdx, TransactionConcurrency txConcurrency) throws Exception;
+
+ /**
+ * Run test case.
+ *
+ * @param tx Function that performs transaction.
+ * @param txBlkNodeIdx ID of node to block transaction.
+ * @param snpBlkNodeIdx ID of node to block incremental snapshot.
+ */
+ protected void run(Runnable tx, int txBlkNodeIdx, int snpBlkNodeIdx) throws Exception {
+ caseNum++;
+
+ awaitSnapshotResourcesCleaned();
+
+ // 1. Block transaction.
+ blockTx(grid(txBlkNodeIdx));
+
+ // 2. Start transaction.
+ IgniteInternalFuture<?> txFut = multithreadedAsync(tx, 1);
+
+ // 3. Await transaction has blocked.
+ awaitTxBlocked(grid(txBlkNodeIdx));
+
+ // 4. Start incremental snapshot procedure concurrently with running transaction.
+ if (snpBlkNodeIdx != -1)
+ ((BlockingSnapshotManager)snp(grid(snpBlkNodeIdx))).block(snpBlkType);
+
+ IgniteFuture<Void> snpFut = snp(grid(0)).createIncrementalSnapshot(SNP);
+
+ // 5. Await incremental snapshot has blocked.
+ if (snpBlkNodeIdx != -1)
+ ((BlockingSnapshotManager)snp(grid(snpBlkNodeIdx))).awaitSnpBlockedOrFinished(snpFut);
+
+ // 6. Resume the blocking transaction.
+ unblockTx(grid(txBlkNodeIdx));
+
+ // 7. Await transaction completed.
+ txFut.get(getTestTimeout(), TimeUnit.MILLISECONDS);
+
+ // 8. Resume the blocking incremental snapshot.
+ if (snpBlkNodeIdx != -1)
+ ((BlockingSnapshotManager)snp(grid(snpBlkNodeIdx))).unblock(snpBlkType);
+
+ // 9. Await while incremental snapshot completed.
+ snpFut.get(getTestTimeout());
+
+ clear();
+ }
+
+ /**
+ * Function that runs transaction with specified keys.
+ *
+ * @param near ID of node to coordinate a transaction.
+ * @param testCase Test case.
+ */
+ protected void tx(int near, TransactionTestCase testCase, TransactionConcurrency concurrency) {
+ try (Transaction tx = grid(near).transactions().txStart(concurrency, SERIALIZABLE)) {
+ for (int key: testCase.keys(grid(0), CACHE))
+ grid(near).cache(CACHE).put(key, 0);
+
+ tx.commit();
+ }
+ }
+
+ /** */
+ protected abstract void blockTx(IgniteEx blkNode);
+
+ /** */
+ protected abstract void awaitTxBlocked(IgniteEx blkNode) throws Exception;
+
+ /** */
+ protected abstract void unblockTx(IgniteEx blkNode);
+
+ /** Finds index of node to be blocked during a case. */
+ protected int blkNodeIndex(int nearNodeIdx, BlkNodeType blkNodeType, TransactionTestCase c) {
+ if (blkNodeType == NEAR)
+ return nearNodeIdx;
+ else if (blkNodeType == PRIMARY)
+ return c.keys[0][0];
+ else
+ return c.keys[0][1];
+ }
+
+ /** Checks WALs correctness for incremental snapshots from all nodes. */
+ protected void checkWalsConsistency() throws Exception {
+ checkWalsConsistency(caseNum, caseNum);
+ }
+
+ /** */
+ private void clear() {
+ for (int n = 0; n < nodes(); n++)
+ BlockingWALManager.walMgr(grid(n)).clear();
+ }
+
+ /** */
+ private static class BlockingWALPluginProvider extends AbstractTestPluginProvider {
+ /** {@inheritDoc} */
+ @Override public String name() {
+ return "BlockingWALProvider";
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T> @Nullable T createComponent(PluginContext ctx, Class<T> cls) {
+ if (IgniteWriteAheadLogManager.class.equals(cls))
+ return (T)new BlockingWALManager(((IgniteEx)ctx.grid()).context());
+
+ return null;
+ }
+ }
+
+ /** Blocks writing to WAL {@link IncrementalSnapshotStartRecord}. */
+ protected static class BlockingWALManager extends FileWriteAheadLogManager {
+ /** First CDL is actual block of record, second CDL shows whether first is blocked. */
+ private final Map<WALRecord.RecordType, T3<Predicate<WALRecord>, CountDownLatch, CountDownLatch>> map = new ConcurrentHashMap<>();
+
+ /** */
+ public BlockingWALManager(GridKernalContext ctx) {
+ super(ctx);
+ }
+
+ /** */
+ static BlockingWALManager walMgr(IgniteEx ign) {
+ return (BlockingWALManager)ign.context().cache().context().wal();
+ }
+
+ /** */
+ public void block(WALRecord.RecordType recType, Predicate<WALRecord> pred) {
+ map.put(recType, new T3<>(pred, new CountDownLatch(1), new CountDownLatch(1)));
+ }
+
+ /** */
+ public void unblock(WALRecord.RecordType recType) {
+ map.get(recType).get2().countDown();
+ }
+
+ /** */
+ public void awaitBlocked(WALRecord.RecordType recType) {
+ U.awaitQuiet(map.get(recType).get3());
+ }
+
+ /** */
+ public void clear() {
+ map.clear();
+ }
+
+ /** {@inheritDoc} */
+ @Override public WALPointer log(WALRecord record) throws IgniteCheckedException, StorageException {
+ T3<Predicate<WALRecord>, CountDownLatch, CountDownLatch> info = map.get(record.type());
+
+ if (info != null && info.get2().getCount() > 0 && info.get1().test(record)) {
+ info.get3().countDown();
+
+ U.awaitQuiet(info.get2());
+ }
+
+ return super.log(record);
+ }
+ }
+
+ /** */
+ private static class BlockingSnapshotPluginProvider extends AbstractTestPluginProvider {
+ /** {@inheritDoc} */
+ @Override public String name() {
+ return "BlockingSnapshotPluginProvider";
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T> @Nullable T createComponent(PluginContext ctx, Class<T> cls) {
+ if (IgniteSnapshotManager.class.equals(cls))
+ return (T)new BlockingSnapshotManager(((IgniteEx)ctx.grid()).context());
+
+ return null;
+ }
+ }
+
+ /** Blocks local incremental snapshot before or after start. */
+ protected static class BlockingSnapshotManager extends IgniteSnapshotManager {
+ /** Blocks this record after local incremental snapshot started. */
+ private static final WALRecord.RecordType blkStartRecType = WALRecord.RecordType.INCREMENTAL_SNAPSHOT_START_RECORD;
+
+ /** Latch that blocks before local incremental snapshot started. */
+ private volatile CountDownLatch beforeStartLatch;
+
+ /** */
+ private volatile CountDownLatch blockedLatch;
+
+ /** */
+ public BlockingSnapshotManager(GridKernalContext ctx) {
+ super(ctx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void handleIncrementalSnapshotId(UUID id, long topVer) {
+ // Block only discovery worker thread.
+ if (Thread.currentThread().getName().contains("disco")) {
+ if (beforeStartLatch != null) {
+ blockedLatch.countDown();
+ blockedLatch = null;
+
+ U.awaitQuiet(beforeStartLatch);
+
+ beforeStartLatch = null;
+ }
+ }
+
+ super.handleIncrementalSnapshotId(id, topVer);
+ }
+
+ /** */
+ public void block(BlkSnpType type) {
+ blockedLatch = new CountDownLatch(1);
+
+ if (type == BEFORE_START)
+ beforeStartLatch = new CountDownLatch(1);
+ else if (type == AFTER_START)
+ ((BlockingWALManager)cctx.wal()).block(blkStartRecType, (rec) -> rec.type() == blkStartRecType);
+ }
+
+ /** */
+ public void awaitSnpBlockedOrFinished(@Nullable IgniteFuture<?> snpFut) {
+ CountDownLatch latch = blockedLatch;
+
+ if (latch == null)
+ return;
+
+ snpFut.listen((f) -> latch.countDown());
+
+ if (beforeStartLatch != null)
+ U.awaitQuiet(latch);
+ else {
+ latch.countDown();
+
+ ((BlockingWALManager)cctx.wal()).awaitBlocked(blkStartRecType);
+ }
+ }
+
+ /** */
+ public void unblock(BlkSnpType type) {
+ if (type == BEFORE_START && beforeStartLatch != null)
+ beforeStartLatch.countDown();
+ else if (type == AFTER_START)
+ ((BlockingWALManager)cctx.wal()).unblock(blkStartRecType);
+ }
+ }
+
+ /** */
+ protected enum BlkNodeType {
+ /** */
+ NEAR,
+
+ /** */
+ PRIMARY,
+
+ /** */
+ BACKUP
+ }
+
+ /** */
+ protected enum BlkSnpType {
+ /** */
+ NONE,
+
+ /** */
+ BEFORE_START,
+
+ /** */
+ AFTER_START,
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotMessagesBlockingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotMessagesBlockingTest.java
new file mode 100644
index 00000000000..09f2c8f0ea1
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotMessagesBlockingTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot.incremental;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRequest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareRequest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareResponse;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishResponse;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareResponse;
+import org.apache.ignite.transactions.TransactionConcurrency;
+
+/** */
+public abstract class AbstractIncrementalSnapshotMessagesBlockingTest extends AbstractIncrementalSnapshotBlockingTest {
+ /** */
+ private static Class<?> txMsgBlkCls;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String instanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(instanceName);
+
+ cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+ return cfg;
+ }
+
+ /** Initialize latches for test cases with blocking tx messages. */
+ protected final void initMsgCase(Class<?> msgCls, BlkNodeType txBlkNode, BlkSnpType snpBlkType, BlkNodeType snpBlkNode) {
+ txBlkNodeType = txBlkNode;
+
+ txMsgBlkCls = msgCls;
+ AbstractIncrementalSnapshotBlockingTest.snpBlkType = snpBlkType;
+ snpBlkNodeType = snpBlkNode;
+ }
+
+ /** */
+ @Override protected void runCase(
+ TransactionTestCase testCase,
+ int nearNodeIdx,
+ TransactionConcurrency txConcurrency
+ ) throws Exception {
+ int txBlkNodeIdx = blkNodeIndex(nearNodeIdx, txBlkNodeType, testCase);
+
+ int snpBlkNodeIdx = -1;
+
+ if (snpBlkType != BlkSnpType.NONE)
+ snpBlkNodeIdx = blkNodeIndex(nearNodeIdx, snpBlkNodeType, testCase);
+
+ if (skipMsgTestCase(txBlkNodeIdx, nearNodeIdx, testCase))
+ return;
+
+ log.info("START CASE " + caseNum +
+ ". Data=" + testCase +
+ ", nearNodeIdx=" + nearNodeIdx +
+ ", txBlkNodeIdx=" + txBlkNodeIdx +
+ ", txBlkNodeType=" + txBlkNodeType +
+ ", snpBlkNodeIdx=" + snpBlkNodeIdx +
+ ", msg=" + txMsgBlkCls.getSimpleName());
+
+ run(() -> tx(nearNodeIdx, testCase, txConcurrency), txBlkNodeIdx, snpBlkNodeIdx);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void blockTx(IgniteEx blkNode) {
+ TestRecordingCommunicationSpi.spi(blkNode).blockMessages((n, msg) ->
+ msg.getClass().equals(txMsgBlkCls)
+ );
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void unblockTx(IgniteEx blkNode) {
+ TestRecordingCommunicationSpi.spi(blkNode).stopBlock();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void awaitTxBlocked(IgniteEx blkNode) throws Exception {
+ TestRecordingCommunicationSpi.spi(blkNode).waitForBlocked(1);
+ }
+
+ /**
+ * Test cases are auto-generated and some of them are invalid, when node {@code #txBlkNodeIdx} that should block
+ * a transaction on sending {@link #txMsgBlkCls} actually doesn't participate in sending such message. Basing on
+ * transctional protocol it checks whether specified test case is invalid and return {@code true} if it is.
+ *
+ * @return true if test case should be skipped.
+ */
+ private boolean skipMsgTestCase(int txBlkNodeIdx, int nearNodeIdx, TransactionTestCase testCase) {
+ // GridNearTxPrepareRequest is sent only from near node if at least one primary copy isn't collocated.
+ if (txMsgBlkCls.equals(GridNearTxPrepareRequest.class)) {
+ return txBlkNodeType != BlkNodeType.NEAR || testCase.allPrimaryOnNear(nearNodeIdx);
+ }
+
+ // GridNearTxPrepareResponse is sent only from primary node if it isn't collocated on near node.
+ if (txMsgBlkCls.equals(GridNearTxPrepareResponse.class)) {
+ return txBlkNodeType != BlkNodeType.PRIMARY || txBlkNodeIdx == nearNodeIdx;
+ }
+
+ // GridNearTxFinishRequest is sent only from near node for two-phase-commit and if at least one primary isn't collocated.
+ if (txMsgBlkCls.equals(GridNearTxFinishRequest.class)) {
+ return txBlkNodeType != BlkNodeType.NEAR || testCase.onePhase() || testCase.allPrimaryOnNear(nearNodeIdx);
+ }
+
+ // GridNearTxFinishResponse is sent only from primary node for two-phase-commit and if it isn't collocated.
+ if (txMsgBlkCls.equals(GridNearTxFinishResponse.class)) {
+ return txBlkNodeType != BlkNodeType.PRIMARY || testCase.onePhase() || txBlkNodeIdx == nearNodeIdx;
+ }
+
+ // GridDhtTxPrepareRequest is sent from primary node, or from near node if primaries are collocated.
+ if (txMsgBlkCls.equals(GridDhtTxPrepareRequest.class)) {
+ if (txBlkNodeType == BlkNodeType.NEAR)
+ return !testCase.allPrimaryOnNear(nearNodeIdx);
+
+ return txBlkNodeType != BlkNodeType.PRIMARY;
+ }
+
+ // GridDhtTxPrepareResponse is sent from backup nodes only.
+ if (txMsgBlkCls.equals(GridDhtTxPrepareResponse.class))
+ return txBlkNodeType != BlkNodeType.BACKUP;
+
+ // GridDhtTxFinishRequest is sent from primary node for two-phase-commit, or from near node if primaries are collocated.
+ if (txMsgBlkCls.equals(GridDhtTxFinishRequest.class)) {
+ if (testCase.onePhase())
+ return true;
+
+ if (txBlkNodeType == BlkNodeType.NEAR)
+ return !testCase.allPrimaryOnNear(nearNodeIdx);
+
+ return txBlkNodeType != BlkNodeType.PRIMARY;
+ }
+
+ return false;
+ }
+
+ /** */
+ protected static List<Class<?>> messages(boolean backup) {
+ List<Class<?>> msgCls = new ArrayList<>();
+
+ msgCls.add(GridNearTxPrepareRequest.class);
+ msgCls.add(GridNearTxPrepareResponse.class);
+ msgCls.add(GridNearTxFinishRequest.class);
+ msgCls.add(GridNearTxFinishResponse.class);
+
+ if (backup) {
+ msgCls.add(GridDhtTxPrepareRequest.class);
+ msgCls.add(GridDhtTxPrepareResponse.class);
+ msgCls.add(GridDhtTxFinishRequest.class);
+ }
+
+ return msgCls;
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotTest.java
new file mode 100644
index 00000000000..a1f22ea3711
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotTest.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot.incremental;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotFinishRecord;
+import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotStartRecord;
+import org.apache.ignite.internal.pagemem.wal.record.RolloverType;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.ClusterSnapshotRecord;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_RECORD_V2;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.INCREMENTAL_SNAPSHOT_FINISH_RECORD;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.INCREMENTAL_SNAPSHOT_START_RECORD;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.snp;
+
+/** Base class for testing incremental snapshot algorithm. */
+public abstract class AbstractIncrementalSnapshotTest extends GridCommonAbstractTest {
+ /** */
+ protected static final String CACHE = "CACHE";
+
+ /** */
+ protected static final String SNP = "base";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String instanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(instanceName);
+
+ cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+ .setWalCompactionEnabled(true)
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setName("incremental-snapshot-persist")
+ .setPersistenceEnabled(true)));
+
+ cfg.setCacheConfiguration(cacheConfiguration(CACHE));
+
+ cfg.setConsistentId(instanceName);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ cleanPersistenceDir();
+
+ startGrids(nodes());
+
+ grid(0).cluster().state(ClusterState.ACTIVE);
+
+ startClientGrid(nodes());
+
+ snp(grid(0)).createSnapshot(SNP).get();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** */
+ protected CacheConfiguration<Integer, Integer> cacheConfiguration(String cacheName) {
+ return new CacheConfiguration<Integer, Integer>()
+ .setName(cacheName)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setBackups(backups());
+ }
+
+ /**
+ * @return Number of server nodes.
+ */
+ protected abstract int nodes();
+
+ /**
+ * @return Number of backups for cache.
+ */
+ protected abstract int backups();
+
+ /** */
+ protected void awaitSnapshotResourcesCleaned() {
+ try {
+ assertTrue(GridTestUtils.waitForCondition(() -> {
+ for (Ignite g: G.allGrids()) {
+ if (snp((IgniteEx)g).currentCreateRequest() != null)
+ return false;
+ }
+
+ return true;
+ }, getTestTimeout(), 0));
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /**
+ * Checks WALs for correct incremental snapshots.
+ *
+ * @param txCnt Count of run transactions.
+ * @param snpCnt Count of incremental snapshots were run within a test.
+ */
+ protected void checkWalsConsistency(int txCnt, int snpCnt) throws Exception {
+ List<IncrementalSnapshotWalReader> readers = new ArrayList<>();
+
+ for (int i = 0; i < nodes(); i++) {
+ try (WALIterator walIter = walIter(i)) {
+ IncrementalSnapshotWalReader reader = new IncrementalSnapshotWalReader(walIter);
+
+ readers.add(reader);
+
+ reader.read();
+
+ int expSnpCnt = reader.snps.get(reader.snps.size() - 1).id != null ? snpCnt : snpCnt + 1;
+
+ assertEquals(expSnpCnt, reader.snps.size());
+ }
+ }
+
+ // Transaction ID -> (incSnpId, nodeIdx).
+ Map<GridCacheVersion, T2<UUID, Integer>> txMap = new HashMap<>();
+
+ // +1 - includes incomplete state also.
+ for (int snpId = 0; snpId < snpCnt + 1; snpId++) {
+ for (int nodeIdx = 0; nodeIdx < nodes(); nodeIdx++) {
+ // Skip if the latest snapshot is completed.
+ if (readers.get(nodeIdx).snps.size() == snpId)
+ continue;
+
+ IncrementalSnapshot snp = readers.get(nodeIdx).snps.get(snpId);
+
+ for (GridCacheVersion xid: snp.txs) {
+ T2<UUID, Integer> prev = txMap.put(xid, new T2<>(snp.id, nodeIdx));
+
+ if (prev != null) {
+ assertTrue("Transaction missed: [xid=" + xid + ", node" + prev.get2() + "=" + prev.get1() +
+ ", node" + nodeIdx + "=" + snp.id,
+ Objects.equals(prev == null ? null : prev.get1(), snp.id));
+ }
+ }
+ }
+ }
+
+ assertEquals(txCnt, txMap.size());
+ }
+
+ /** Get iterator over WAL. */
+ protected WALIterator walIter(int nodeIdx) throws Exception {
+ Path workDir = Paths.get(U.defaultWorkDirectory());
+
+ IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
+
+ String subfolderName = U.maskForFileName(getTestIgniteInstanceName(nodeIdx));
+
+ File wal = workDir.resolve(DataStorageConfiguration.DFLT_WAL_PATH).resolve(subfolderName).toFile();
+ File archive = workDir.resolve(DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH).resolve(subfolderName).toFile();
+
+ IgniteWalIteratorFactory.IteratorParametersBuilder params = new IgniteWalIteratorFactory.IteratorParametersBuilder()
+ .filesOrDirs(wal, archive);
+
+ return factory.iterator(params);
+ }
+
+ /** Prepare for snapshot restoring - restart grids, with clean persistence. */
+ protected void restartWithCleanPersistence(int nodes, List<String> caches) throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir(true);
+
+ startGrids(nodes);
+
+ grid(0).cluster().state(ClusterState.ACTIVE);
+
+ // Caches are configured with IgniteConiguration, need to destroy them before restoring snapshot.
+ grid(0).destroyCaches(caches);
+
+ awaitPartitionMapExchange();
+ }
+
+ /** Rolls WAL segment for specified grid. */
+ protected void rollWalSegment(IgniteEx g) {
+ g.context().cache().context().database().checkpointReadLock();
+
+ try {
+ g.context().cache().context().wal().log(new ClusterSnapshotRecord("dummy"), RolloverType.CURRENT_SEGMENT);
+ }
+ catch (IgniteCheckedException e) {
+ throw new RuntimeException(e);
+ }
+ finally {
+ g.context().cache().context().database().checkpointReadUnlock();
+ }
+ }
+
+ /**
+ * Read WAL and sort transactions by incremental snapshots.
+ */
+ private static class IncrementalSnapshotWalReader {
+ /** Iterator over WAL archive files. */
+ private final WALIterator walIter;
+
+ /** Collection of incremental snapshots. */
+ final List<IncrementalSnapshot> snps = new ArrayList<>();
+
+ /** For test purposes. */
+ IncrementalSnapshotWalReader(WALIterator walIter) {
+ this.walIter = walIter;
+ }
+
+ /** Read WAL and fills {@link #snps} with read incremental snapshot. */
+ void read() {
+ snps.add(new IncrementalSnapshot());
+
+ while (walIter.hasNext()) {
+ IgniteBiTuple<WALPointer, WALRecord> next = walIter.next();
+
+ WALRecord rec = next.getValue();
+
+ IncrementalSnapshot snp = currentSnapshot();
+
+ if (rec.type() == DATA_RECORD_V2)
+ snp.addTransaction(((DataRecord)rec).writeEntries().get(0).nearXidVersion());
+ else if (rec.type() == INCREMENTAL_SNAPSHOT_START_RECORD) {
+ assert snp.id == null : "Lost FINISH record: " + rec;
+
+ snp.id = ((IncrementalSnapshotStartRecord)rec).id();
... 4441 lines suppressed ...