You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by zs...@apache.org on 2026/10/22 13:13:17 UTC
(ignite) branch master updated: IGNITE-29050 Introduce snapshot deletion command (#13577)
This is an automated email from the ASF dual-hosted git repository.
zstan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new b1e0e790e9f IGNITE-29050 Introduce snapshot deletion command (#13577)
b1e0e790e9f is described below
commit b1e0e790e9f4d6467591281c0be7250aebf2375d
Author: Vladimir Steshin <vl...@gmail.com>
AuthorDate: Thu Oct 22 16:13:10 2026 +0300
IGNITE-29050 Introduce snapshot deletion command (#13577)
---
docs/_docs/snapshots/snapshots.adoc | 44 +
.../processors/query/calcite/hint/HintUtils.java | 2 +-
.../query/calcite/prepare/PlannerHelper.java | 2 +-
.../commandline/CommandHandlerParsingTest.java | 3 +
.../testsuites/IgniteControlUtilityTestSuite6.java | 32 +
.../util/GridCommandHandlerDeleteSnapshotTest.java | 240 ++++++
.../ignite/internal/CoreMessagesProvider.java | 4 +
.../management/snapshot/SnapshotCommand.java | 3 +-
.../management/snapshot/SnapshotDeleteCommand.java | 123 +++
.../snapshot/SnapshotDeleteCommandArg.java | 62 ++
.../management/snapshot/SnapshotDeleteTask.java | 92 ++
.../persistence/filename/SnapshotFileTree.java | 4 +-
.../snapshot/IgniteSnapshotManager.java | 225 ++++-
.../snapshot/IncrementalSnapshotProcessor.java | 3 +-
.../persistence/snapshot/SnapshotCheckProcess.java | 23 +-
.../snapshot/SnapshotDeleteProcess.java | 415 +++++++++
.../snapshot/SnapshotDeleteProcessResult.java | 86 ++
.../snapshot/SnapshotDeleteRequest.java | 65 ++
.../snapshot/SnapshotDeleteResponse.java | 68 ++
.../snapshot/SnapshotMetadataVerificationTask.java | 5 +-
.../snapshot/SnapshotRestoreProcess.java | 5 +
.../persistence/DistributedMetaStorageImpl.java | 2 +-
.../feature/CoreFeatureRegistry.java | 3 +
.../util/distributed/DistributedProcess.java | 5 +
.../snapshot/AbstractSnapshotSelfTest.java | 149 +++-
.../snapshot/EncryptedSnapshotTest.java | 2 +-
.../snapshot/IgniteClusterSnapshotCheckTest.java | 51 +-
...iteClusterSnapshotDeleteRollingUpgradeTest.java | 256 ++++++
.../snapshot/IgniteClusterSnapshotDeleteTest.java | 931 +++++++++++++++++++++
.../IgniteClusterSnapshotRestoreSelfTest.java | 42 +
.../snapshot/IgniteClusterSnapshotSelfTest.java | 40 +-
.../IgniteSnapshotRestoreFromRemoteTest.java | 5 +
.../AbstractIncrementalSnapshotTest.java | 2 +-
.../IncrementalSnapshotNodeFailureTest.java | 2 +-
.../testsuites/IgniteSnapshotTestSuite9.java | 49 ++
...ridCommandHandlerClusterByClassTest_help.output | 7 +
...andHandlerClusterByClassWithSSLTest_help.output | 7 +
.../tests/ContinuousDataLoadApplication.java | 2 +-
...apshot.SnapshotDeleteResponse$DeleteStatus.json | 8 +
...persistence.snapshot.SnapshotDeleteRequest.json | 8 +
...ersistence.snapshot.SnapshotDeleteResponse.json | 7 +
41 files changed, 3019 insertions(+), 65 deletions(-)
diff --git a/docs/_docs/snapshots/snapshots.adoc b/docs/_docs/snapshots/snapshots.adoc
index fe61b3c4923..3f5255a798f 100644
--- a/docs/_docs/snapshots/snapshots.adoc
+++ b/docs/_docs/snapshots/snapshots.adoc
@@ -287,6 +287,50 @@ control.(sh|bat) --snapshot restore snapshot_09062021 --groups cache-group1,cach
control.(sh|bat) --snapshot restore snapshot_09062021 --increment 1
----
+== Deleting Snapshot
+
+You can delete a snapshot using the `control.sh|bat` script.
+
+The deletion is performed on all *online* server nodes of the cluster.
+[NOTE]
+====
+The snapshot integrity, topology and correctness aren't checked. Snapshot data on offline server nodes aren't deleted.
+====
+
+[tabs]
+--
+tab:Unix[]
+[source,shell]
+----
+# Delete the snapshot "snapshot_09062021".
+control.sh --snapshot delete snapshot_09062021
+
+# Delete the snapshot "snapshot_09062021" located in the "/tmp/ignite/snapshots" folder.
+control.sh --snapshot delete snapshot_09062021 --src /tmp/ignite/snapshots
+----
+
+tab:Windows[]
+[source,shell]
+----
+# Delete the snapshot "snapshot_09062021".
+control.bat --snapshot delete snapshot_09062021
+
+# Delete the snapshot "snapshot_09062021" located in the "C:\tmp\ignite\snapshots" folder.
+control.bat --snapshot delete snapshot_09062021 --src C:\tmp\ignite\snapshots
+----
+--
+
+=== Delete operation limitations
+
+The delete operation is subject to the following limitations:
+
+* The deletion is rejected if any snapshot operation (create, restore, check, delete) is active for the snapshot.
+* The operation requires the snapshot administration permissions via `IgniteSecurity` (if configured).
+* The operation cannot be undone and the deleted snapshot cannot be restored. The command prompts for a confirmation.
+* Before deletion, no validation is performed on the snapshot other than locating and reading its metadata.
+If the metadata isn't found or cannot be read, snapshot isn't deleted.
+* The deletion operation is independent of the status of the snapshot operation and cancellation requests.
+
== Getting Snapshot Operation Status
The status of the current snapshot operation in the cluster can be obtained using the `control.sh|bat` script or JMX interface:
diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/hint/HintUtils.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/hint/HintUtils.java
index 40c267ded09..5597b4c776a 100644
--- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/hint/HintUtils.java
+++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/hint/HintUtils.java
@@ -99,7 +99,7 @@ public final class HintUtils {
/**
* @return Distinct hints within {@code hints} filtered with {@code hintDefs}, {@link HintOptionsChecker} and
- * removed inherit pathes.
+ * removed inherit paths.
* @see HintOptionsChecker
* @see RelHint#inheritPath
*/
diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerHelper.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerHelper.java
index d88275fb786..39fb078f6b2 100644
--- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerHelper.java
+++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerHelper.java
@@ -134,7 +134,7 @@ public class PlannerHelper {
rel = planner.trimUnusedFields(root.withRel(rel)).rel;
// The following pushed down project can erase top-level hints. We store them to reassign hints for join nodes.
- // Clear the inherit pathes to consider the hints as not propogated ones.
+ // Clear the inherit paths to consider the hints as not propagated ones.
List<RelHint> topHints = HintUtils.allRelHints(rel).stream().map(h -> h.inheritPath.isEmpty()
? h
: h.copy(Collections.emptyList())).collect(Collectors.toList());
diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index bf180045bfc..2e5b918f7d1 100644
--- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -82,6 +82,7 @@ import org.apache.ignite.internal.management.metric.MetricCommand;
import org.apache.ignite.internal.management.performancestatistics.PerformanceStatisticsCommand;
import org.apache.ignite.internal.management.property.PropertyCommand;
import org.apache.ignite.internal.management.snapshot.SnapshotCommand;
+import org.apache.ignite.internal.management.snapshot.SnapshotDeleteCommand;
import org.apache.ignite.internal.management.snapshot.SnapshotRestoreCommand;
import org.apache.ignite.internal.management.tx.TxCommand;
import org.apache.ignite.internal.management.tx.TxCommandArg;
@@ -548,6 +549,8 @@ public class CommandHandlerParsingTest {
cmdText = F.concat(cmdText, "cacheGroup1");
else if (cmd.getClass() == SnapshotRestoreCommand.class)
cmdText = F.concat(cmdText, "snp1");
+ else if (cmd.getClass() == SnapshotDeleteCommand.class)
+ cmdText = F.concat(cmdText, "snp1");
else if (cmd.getClass() == MetaUpdateCommand.class)
return;
else if (cmd.getClass() == MetaRemoveCommand.class)
diff --git a/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite6.java b/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite6.java
new file mode 100644
index 00000000000..3719e8d887d
--- /dev/null
+++ b/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite6.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import org.apache.ignite.util.GridCommandHandlerDeleteSnapshotTest;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/**
+ * Test suite for control utility.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ GridCommandHandlerDeleteSnapshotTest.class
+})
+public class IgniteControlUtilityTestSuite6 {
+}
diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java
new file mode 100644
index 00000000000..6e2a303d7a5
--- /dev/null
+++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.io.File;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collection;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.management.snapshot.SnapshotDeleteCommand;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.Test;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import static java.nio.file.Files.newDirectoryStream;
+import static org.apache.ignite.cluster.ClusterState.ACTIVE;
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.snp;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+import static org.junit.Assume.assumeTrue;
+
+/** Test for the command '--snapshot delete'. */
+public class GridCommandHandlerDeleteSnapshotTest extends GridCommandHandlerAbstractTest {
+ /** Flag to add an extra server node after the cluster start and the snapshot creation. */
+ @Parameter(1)
+ public boolean extraServerNode;
+
+ /** */
+ @Parameter(2)
+ public boolean incremental;
+
+ /** */
+ @Parameter(3)
+ public boolean changeBaseline;
+
+ /** */
+ @Parameter(4)
+ public boolean customPath;
+
+ /** */
+ @Parameter(5)
+ public boolean separatedWorkDir;
+
+ /** */
+ @Parameters(name = "client={0}, useExtraNode={1}, inc={2}, chBaseln={3}, cstSnpPath={4}, separatedWorkDirs={5}")
+ public static Collection<?> parameters() {
+ return GridTestUtils.cartesianProduct(
+ commandHandlers(),
+ F.asList(false, true), // Use or not extra server node after the cluster start and the snapshot creation.
+ F.asList(false, true), // Add incremental snapshot.
+ F.asList(false, true), // Change baseline.
+ F.asList(false, true), // Use custom snapshot path.
+ F.asList(false, true) // Separated nodes work directories.
+ );
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ /** Handy if test running is interrupted and {@link #afterTest()} isn't invoked. */
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void cleanPersistenceDir() throws Exception {
+ super.cleanPersistenceDir();
+
+ // Also cleans separated snapshot working directories and custom snapshot paths.
+ try (DirectoryStream<Path> files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) {
+ for (Path path : files)
+ U.delete(path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ if (separatedWorkDir)
+ cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath());
+
+ return cfg;
+ }
+
+ /** */
+ @Test
+ public void testSnapshotDelete() throws Exception {
+ // A custom snapshot path actually puts snapshots in a shared directory. This skews the results when dedicated
+ // work directories are set.
+ assumeTrue(!customPath || !separatedWorkDir);
+
+ int entriesCnt = 100;
+ int initNodes = 3;
+
+ walCompactionEnabled(incremental);
+
+ IgniteEx ig = (IgniteEx)startGridsMultiThreaded(initNodes);
+
+ if (changeBaseline) {
+ ig.cluster().baselineAutoAdjustEnabled(false);
+
+ ig.cluster().setBaselineTopology(ig.cluster().topologyVersion());
+ }
+
+ ig.cluster().state(ACTIVE);
+
+ createCacheAndPreload(ig, entriesCnt);
+
+ File cstSnpsRoot = customPath
+ ? new File(grid(0).context().pdsFolderResolver().fileTree().snapshotsRoot(), "ex_snapshots")
+ : null;
+ File snpDir = new File(customPath ? cstSnpsRoot : ig.context().pdsFolderResolver().fileTree().snapshotsRoot(), "testSnapshot");
+
+ snp(ig).createSnapshot("testSnapshot", customPath ? cstSnpsRoot.getAbsolutePath() : null, false, false)
+ .get(getTestTimeout());
+
+ if (incremental) {
+ for (int i = 0; i < initNodes; ++i) {
+ int dataIdx = entriesCnt + entriesCnt / 4 * i;
+
+ try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(DEFAULT_CACHE_NAME)) {
+ for (int d = dataIdx; d < dataIdx + entriesCnt / 4; ++d)
+ streamer.addData(d, d);
+ }
+
+ snp(ig).createSnapshot("testSnapshot", customPath ? cstSnpsRoot.getAbsolutePath() : null, true, false)
+ .get(getTestTimeout());
+ }
+ }
+
+ String baselineGone = null;
+
+ // Optionally restarts with the same servers number, but changed baseline. The snapshot is kept on the same
+ // previous nodes independently of the baseline.
+ if (changeBaseline) {
+ baselineGone = grid(initNodes - 1).name();
+
+ ig.destroyCache(DEFAULT_CACHE_NAME);
+ awaitPartitionMapExchange();
+
+ stopAllGrids();
+
+ ig = (IgniteEx)startGridsMultiThreaded(initNodes - 1);
+
+ ig.cluster().setBaselineTopology(ig.cluster().topologyVersion());
+
+ startGrid(initNodes - 1);
+
+ assertEquals(initNodes - 1, ig.cluster().currentBaselineTopology().size());
+ assertEquals(initNodes, ig.cluster().nodes().size());
+ }
+
+ // Optionally adds extra server.
+ if (extraServerNode)
+ startGrid(initNodes);
+
+ injectTestSystemOut();
+
+ // Tests missing snapshot deletion.
+ if (customPath) {
+ assertEquals(EXIT_CODE_OK, execute(newCommandHandler(createTestLogger()), "--snapshot", "delete", "--src",
+ cstSnpsRoot.getAbsolutePath(), "wrongSnapshot"));
+
+ assertEquals(EXIT_CODE_OK, execute(newCommandHandler(createTestLogger()), "--snapshot", "delete", "--src",
+ cstSnpsRoot.getAbsolutePath() + "_wrongPath", "testSnapshot"));
+ }
+ else
+ assertEquals(EXIT_CODE_OK, execute(newCommandHandler(createTestLogger()), "--snapshot", "delete", "wrongSnapshot"));
+
+ String out = testOut.toString();
+
+ assertFalse(out.contains(SnapshotDeleteCommand.REMOVED_PREF));
+ assertFalse(out.contains(SnapshotDeleteCommand.NODE_NOT_FOUND_PREF));
+ assertTrue(out.contains(SnapshotDeleteCommand.NOT_FOUND));
+
+ testOut.reset();
+ assertTrue(testOut.toString().isEmpty());
+
+ if (customPath) {
+ assertEquals(EXIT_CODE_OK, execute(newCommandHandler(createTestLogger()), "--snapshot", "delete", "--src",
+ cstSnpsRoot.getAbsolutePath(), "testSnapshot"));
+ }
+ else
+ assertEquals(EXIT_CODE_OK, execute(newCommandHandler(createTestLogger()), "--snapshot", "delete", "testSnapshot"));
+
+ out = testOut.toString();
+
+ if (changeBaseline)
+ assertTrue(out.contains(SnapshotDeleteCommand.MISSING_BASELINES + baselineGone));
+
+ if (separatedWorkDir) {
+ // When the nodes use separated work directory, we expect a strict result.
+ assertTrue(out.contains(SnapshotDeleteCommand.REMOVED_PREF));
+ assertFalse(out.contains(SnapshotDeleteCommand.UNSURE_DELETION_PREF));
+
+ if (extraServerNode)
+ assertTrue(out.contains(SnapshotDeleteCommand.NODE_NOT_FOUND_PREF));
+ }
+ else {
+ // When nodes use a shared work directory, there is a race for the delete operation. One node can get faster
+ // than others and remove snapshot or some its parts before the rest finds it.
+ assertTrue(out.contains(SnapshotDeleteCommand.REMOVED_PREF) || out.contains(SnapshotDeleteCommand.UNSURE_DELETION_PREF));
+ }
+
+ assertFalse(out.contains(SnapshotDeleteCommand.NOT_FOUND));
+
+ assertTrue(waitForCondition(() -> !snpDir.exists(), getTestTimeout()));
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java
index 923ef50c937..4d90506b098 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java
@@ -162,6 +162,8 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.Snapshot
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckPartitionHashesResponse;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcessRequest;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckResponse;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteRequest;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesFailureMessage;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesRequestMessage;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotHandlerResult;
@@ -436,6 +438,8 @@ public class CoreMessagesProvider extends AbstractMessageFactoryProvider {
register(SnapshotFilesFailureMessage.class);
register(IncrementalSnapshotVerifyResult.class);
register(IncrementalSnapshotAwareMessage.class);
+ register(SnapshotDeleteRequest.class);
+ register(SnapshotDeleteResponse.class);
// [6300 - 6400]: Services messages. Most of them originally come from Discovery.
msgIdx = 6300;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java
index deb5416b8e2..8979958bd8f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java
@@ -28,7 +28,8 @@ public class SnapshotCommand extends CommandRegistryImpl {
new SnapshotCancelCommand(),
new SnapshotCheckCommand(),
new SnapshotRestoreCommand(),
- new SnapshotStatusCommand()
+ new SnapshotStatusCommand(),
+ new SnapshotDeleteCommand()
);
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommand.java
new file mode 100644
index 00000000000..4db4491964f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommand.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.management.snapshot;
+
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcess;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcessResult;
+import org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Snapshot deletion command.
+ *
+ * @see CoreFeatureRegistry#SNAPSHOT_DELETE_FEATURE
+ * @see SnapshotDeleteProcess
+ */
+public class SnapshotDeleteCommand extends AbstractSnapshotCommand<SnapshotDeleteCommandArg, SnapshotDeleteProcessResult> {
+ /** */
+ public static final String DESC = "Deletes the snapshot and all its incremental snapshots from all online server nodes";
+
+ /** */
+ public static final String UNSURE_DELETION_PREF = "WARNING: the following nodes found snapshot data but might not " +
+ "remove it completely: ";
+
+ /** */
+ public static final String REMOVED_PREF = "Snapshot removal is completed on: ";
+
+ /** */
+ public static final String NODE_NOT_FOUND_PREF = "NOTE: the following nodes can't find any snapshot data, " +
+ "operation skipped: ";
+
+ /** */
+ public static final String NOT_FOUND = "Snapshot not found on available server nodes.";
+
+ /** */
+ public static final String MISSING_BASELINES = "WARNING: the snapshot's baseline nodes with the following consistent " +
+ "IDs are missing in current cluster: ";
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public String description() {
+ return DESC;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Class<SnapshotDeleteCommandArg> argClass() {
+ return SnapshotDeleteCommandArg.class;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Class<SnapshotDeleteTask> taskClass() {
+ return SnapshotDeleteTask.class;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void printResult(SnapshotDeleteCommandArg arg, SnapshotDeleteProcessResult res, Consumer<String> printer) {
+ boolean found = false;
+
+ if (!res.uncompletedNodes().isEmpty()) {
+ found = true;
+
+ printer.accept(UNSURE_DELETION_PREF + nodeIdPairsStrLst(res.uncompletedNodes()));
+
+ printer.accept("");
+ }
+
+ if (!res.completedNodes().isEmpty()) {
+ found = true;
+
+ printer.accept(REMOVED_PREF + nodeIdPairsStrLst(res.completedNodes()));
+ printer.accept("");
+ }
+
+ if (found) {
+ if (!res.emptyNodes().isEmpty())
+ printer.accept(NODE_NOT_FOUND_PREF + nodeIdPairsStrLst(res.emptyNodes()));
+
+ if (!res.absentBaselines().isEmpty())
+ printer.accept(MISSING_BASELINES + String.join(", ", res.absentBaselines()));
+ }
+ else
+ printer.accept(NOT_FOUND);
+ }
+
+ /** */
+ private static String nodeIdPairsStrLst(Map<UUID, String> uuids) {
+ return uuids.entrySet().stream()
+ .map(e -> e.getValue() + " [uuid=" + e.getKey() + ']')
+ .collect(Collectors.joining(", "));
+ }
+
+ /** {@inheritDoc} */
+ @Override public String confirmationPrompt(SnapshotDeleteCommandArg arg) {
+ return "This operation will completely remove snapshot: '" + arg.snapshotName() + "' and all its incremental snapshots." +
+ U.nl() + U.nl() +
+ "The operation may require the snapshot administration permissions." +
+ U.nl() + U.nl() +
+ "Deletion of any data without (valid) snapshot metadata is prohibited." +
+ U.nl() + U.nl() +
+ "The operation cannot be reverted." +
+ U.nl() + U.nl() +
+ "NOTE: Snapshot data on offline server nodes will remain untouched.";
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommandArg.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommandArg.java
new file mode 100644
index 00000000000..afe1afef7d4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommandArg.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.management.snapshot;
+
+import org.apache.ignite.internal.Order;
+import org.apache.ignite.internal.dto.IgniteDataTransferObject;
+import org.apache.ignite.internal.management.api.Argument;
+import org.apache.ignite.internal.management.api.Positional;
+import org.jetbrains.annotations.Nullable;
+
+/** */
+public class SnapshotDeleteCommandArg extends IgniteDataTransferObject {
+ /** */
+ private static final long serialVersionUID = 0;
+
+ /** */
+ @Order(0)
+ @Positional
+ @Argument(description = "Snapshot name")
+ @Nullable String snapshotName;
+
+ /** */
+ @Order(1)
+ @Argument(example = "path/to/snapshots", optional = true, description = "Path to snapshot location directory. " +
+ "If not specified, the default snapshot configuration directory will be used")
+ @Nullable String src;
+
+ /** */
+ public @Nullable String snapshotName() {
+ return snapshotName;
+ }
+
+ /** */
+ public void snapshotName(@Nullable String snapshotName) {
+ this.snapshotName = snapshotName;
+ }
+
+ /** */
+ public @Nullable String src() {
+ return src;
+ }
+
+ /** */
+ public void src(@Nullable String src) {
+ this.src = src;
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteTask.java
new file mode 100644
index 00000000000..b9238d397f9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteTask.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.management.snapshot;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcess;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcessResult;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.resources.IgniteInstanceResource;
+
+import static org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry.SNAPSHOT_DELETE_FEATURE;
+
+/**
+ * @see IgniteSnapshotManager#deleteSnapshot(String, String)
+ * @see SnapshotDeleteProcess
+ */
+@GridInternal
+public class SnapshotDeleteTask extends VisorOneNodeTask<SnapshotDeleteCommandArg, SnapshotDeleteProcessResult> {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ @IgniteInstanceResource
+ private IgniteEx ignite;
+
+ /** {@inheritDoc} */
+ @Override protected VisorJob<SnapshotDeleteCommandArg, SnapshotDeleteProcessResult> job(SnapshotDeleteCommandArg arg) {
+ return new SnapshotDeleteJob(arg, debug);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Map<? extends ComputeJob, ClusterNode> map0(
+ List<ClusterNode> subgrid,
+ VisorTaskArgument<SnapshotDeleteCommandArg> arg
+ ) {
+ if (!ignite.context().rollingUpgrade().features().isActive(SNAPSHOT_DELETE_FEATURE))
+ throw new IgniteException(SnapshotDeleteProcess.OP_REJECT_FEATURE_MSG);
+
+ return super.map0(subgrid, arg);
+ }
+
+ /** */
+ private static class SnapshotDeleteJob extends SnapshotJob<SnapshotDeleteCommandArg, SnapshotDeleteProcessResult> {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * @param arg Snapshot delete task argument.
+ * @param debug Flag indicating whether debug information should be printed into node log.
+ */
+ protected SnapshotDeleteJob(SnapshotDeleteCommandArg arg, boolean debug) {
+ super(arg, debug);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected SnapshotDeleteProcessResult run(SnapshotDeleteCommandArg arg) {
+ IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr();
+
+ try {
+ return snpMgr.deleteSnapshot(arg.snapshotName(), arg.src()).get();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException("Failed to delete snapshot.", e);
+ }
+ }
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java
index f53abc94ba4..387ee695860 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java
@@ -332,6 +332,7 @@ public class SnapshotFileTree extends NodeFileTree {
public static File root(SharedFileTree ft, String name, @Nullable String path) {
assert name != null : "Snapshot name cannot be empty or null.";
+ // TODO: use the default snapshots directory for relative {@code path} https://issues.apache.org/jira/browse/IGNITE-29126
return path == null ? new File(ft.snapshotsRoot(), name) : new File(path, name);
}
@@ -373,6 +374,8 @@ public class SnapshotFileTree extends NodeFileTree {
}
/**
+ * TODO: support the extra storages for incremental snapshots https://issues.apache.org/jira/browse/IGNITE-29128
+ * <p>
* Modifies {@link #extraStorages} for this tree to reflect snapshot options.
* In case {@link IgniteConfiguration#getSnapshotPath()} points to absolute directory or {@link #path} for snapshot provided
* then all snapshot files must be stored inside one folder.
@@ -457,7 +460,6 @@ public class SnapshotFileTree extends NodeFileTree {
return res;
}
-
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(SnapshotFileTree.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
index 4d32bc1d38c..d3b19b65f0f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
@@ -60,6 +60,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
@@ -238,6 +239,7 @@ import static org.apache.ignite.spi.systemview.view.SnapshotView.SNAPSHOT_SYS_VI
* <li>Create snapshot of the whole persistent cluster cache groups by triggering PME to achieve consistency.</li>
* <li>Create cache dump - snapshot of cluster cache groups including in-memory.</li>
* <li>Create incremental snapshot using lightweight, non-blocking Consistent Cut algorithm.</li>
+ * <li>Delete a snapshot on the cluster.</li>
* </ul>
*/
public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
@@ -344,6 +346,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/** Snapshot validation distributed process. */
private final SnapshotCheckProcess checkSnpProc;
+ /** Distributed process to delete cluster snapshot. */
+ private final SnapshotDeleteProcess deleteSnpProc;
+
/** Check previously performed snapshot operation and delete uncompleted files if we need. */
private final DistributedProcess<SnapshotOperationEndRequest, SnapshotOperationResponse> endSnpProc;
@@ -446,6 +451,8 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
checkSnpProc = new SnapshotCheckProcess(ctx);
+ deleteSnpProc = new SnapshotDeleteProcess(ctx);
+
// Manage remote snapshots.
snpRmtMgr = new SequentialRemoteSnapshotManager();
}
@@ -664,6 +671,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
restoreCacheGrpProc.interrupt(stopErr);
checkSnpProc.interrupt(stopErr);
+ deleteSnpProc.interrupt(stopErr);
// Try stop all snapshot processing if not yet.
for (AbstractSnapshotFutureTask<?> sctx : locSnpTasks.values())
@@ -701,57 +709,159 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/**
* @param snpDir Snapshot dir.
*/
- public void deleteSnapshot(File snpDir) {
+ public void deleteLocalSnapshot(File snpDir) {
if (!snpDir.exists())
return;
if (!snpDir.isDirectory())
return;
- deleteSnapshot(new SnapshotFileTree(
+ SnapshotFileTree sft = new SnapshotFileTree(
cctx.kernalContext(),
snpDir.getName(),
snpDir.getParent(),
ft.folderName(),
- pdsSettings.consistentId().toString()));
+ pdsSettings.consistentId().toString()
+ );
+
+ deleteLocalSnapshot(sft, true, false);
}
- /** */
- public void deleteSnapshot(SnapshotFileTree sft) {
+ /**
+ * TODO : revise the incremental snapshots parts in the scoped case https://issues.apache.org/jira/browse/IGNITE-29095
+ *
+ * Tries to delete local snapshot data.
+ *
+ * @param sft Snapshot file tree.
+ * @param scoped If {@code true}, deletes only node's data and deletes shared snapshot directories only if they are empty.
+ * Otherwise, completely deletes entire snapshot data.
+ * @param ignoreErrs If {@code true}, logs possible deletion exceptions and marks the result as undeleted.
+ * @return A pair of {@code boolean} values. The first indicates whether snapshot was completely deleted. The second
+ * indicates whether the snapshot was found.
+ */
+ public T2<Boolean, Boolean> deleteLocalSnapshot(SnapshotFileTree sft, boolean scoped, boolean ignoreErrs) {
+ T2<Boolean, Boolean> res = new T2<>(false, false);
+
+ List<File> allStorages = sft.allStorages().toList();
+
+ if (sft.root().exists())
+ res.set2(true);
+ else {
+ for (File storage : allStorages) {
+ if (storage.exists()) {
+ res.set2(true);
+
+ break;
+ }
+ }
+ }
+
+ // Not found at all - nothing to delete.
+ if (!res.get2())
+ return res;
+
+ // Assume we'll succeed.
+ res.set1(true);
+
try {
- U.delete(sft.binaryMeta());
- sft.allStorages().forEach(U::delete);
- U.delete(sft.meta());
+ for (File s : allStorages) {
+ // Extra storage's or root's snapshot node data.
+ if (!deleteDirectory(s, false)) {
+ res.set1(false);
- deleteDirectory(sft.binaryMetaRoot());
- deleteDirectory(sft.marshaller());
+ continue;
+ }
- // Delete parent dir which is {snapshot_root}/db if empty.
- sft.marshaller().getParentFile().delete();
- // Delete root dir which is {snapshot_root} if empty.
- sft.root().delete();
+ // Proceed only for the extra storages.
+ if (s.equals(sft.nodeStorage()))
+ continue;
+
+ // Extra storage's "db" directory.
+ s = s.getParentFile();
+
+ if (!deleteDirectory(s, scoped)) {
+ res.set1(false);
+
+ continue;
+ }
+
+ // Extra storage's snapshot root.
+ s = s.getParentFile();
+
+ if (!deleteDirectory(s, scoped))
+ res.set1(false);
+ }
+
+ // Snapshots not-data directories.
+ for (File p : F.asList(sft.binaryMeta(), sft.binaryMetaRoot(), sft.marshaller())) {
+ if (!deleteDirectory(p, false))
+ res.set1(false);
+ }
+
+ // Root "db" directory.
+ if (!deleteDirectory(sft.marshaller().getParentFile(), scoped))
+ res.set1(false);
+
+ // Remove the metadata at the end. For the scoped removal, delete anyway because it sets 'false' as the result
+ // if can't remove a non-empty shared directory. This is ok.
+ if ((scoped || res.get1()) && !sft.meta().delete() && sft.meta().exists())
+ res.set1(false);
+
+ // Finally, delete the snapshot root directory.
+ if (!deleteDirectory(sft.root(), scoped))
+ res.set1(false);
}
- catch (IOException e) {
- throw new IgniteException(e);
+ catch (Exception e) {
+ String errMsg = "Failed to delete local snapshot [snpName=" + sft.name() + ']';
+
+ if (ignoreErrs) {
+ log.warning(errMsg, e);
+
+ res.set1(false);
+ }
+ else
+ throw new IgniteException(errMsg, e);
}
+
+ return res;
+ }
+
+ /** @return {@code True} if {@code dir} was deleted or doesn't exist as result. */
+ private boolean deleteDirectory(File dir, boolean doNotTraverse) throws IOException {
+ // The 'exists' checks are for a concurrent deletion when nodes share their working and snapshot directories.
+ // Nodes may steal removal jobs and the files aren't synchronized. There are gaps between `exists()` and `delete()`.
+ // We try to delete first. If the snapshot data wasn't deleted because it doesn't exist anymore, it's not an error.
+ return !((doNotTraverse && !dir.delete() || !doNotTraverse && !deleteDirectory(dir)) && dir.exists());
}
/** Concurrently traverse the directory and delete all files. */
- private void deleteDirectory(File dir) throws IOException {
- Files.walkFileTree(dir.toPath(), new SimpleFileVisitor<Path>() {
+ private boolean deleteDirectory(File dir) throws IOException {
+ var res = new AtomicBoolean(true);
+
+ Files.walkFileTree(dir.toPath(), new SimpleFileVisitor<>() {
@Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
- U.delete(file);
+ File f0 = file.toFile();
+
+ // Check for a concurrent deletion.
+ if (!U.delete(f0) && f0.exists())
+ res.set(false);
return FileVisitResult.CONTINUE;
}
@Override public FileVisitResult visitFileFailed(Path file, IOException exc) {
// Skip files which can be concurrently removed from FileTree.
+ res.set(false);
+
return FileVisitResult.CONTINUE;
}
@Override public FileVisitResult postVisitDirectory(Path dir, IOException e) {
- dir.toFile().delete();
+ File f0 = dir.toFile();
+
+ // Check for a concurrent deletion.
+ if (!f0.delete() && f0.exists())
+ res.set(false);
if (log.isInfoEnabled() && e != null)
log.info("Snapshot directory cleaned with an exception [dir=" + dir + ", e=" + e.getMessage() + ']');
@@ -759,6 +869,8 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
return FileVisitResult.CONTINUE;
}
});
+
+ return res.get();
}
/**
@@ -813,6 +925,11 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
"re-encryption process is not finished yet."));
}
+ if (cctx.snapshotMgr().isSnapshotDeleting(req.snapshotName(), req.snapshotPath())) {
+ return new GridFinishedFuture<>(new IgniteCheckedException("Snapshot operation has been rejected. Delete " +
+ "operation of snapshot '%s' is in progress.".formatted(req.snapshotName())));
+ }
+
List<Integer> grpIds = new ArrayList<>(F.viewReadOnly(req.groups(), CU::cacheId));
Collection<Integer> comprGrpIds = F.view(grpIds, i -> {
CacheGroupDescriptor desc = cctx.cache().cacheGroupDescriptor(i);
@@ -1282,7 +1399,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
if (snpStartReq.incremental())
U.delete(snpOp.snapshotFileTree().incrementalSnapshotFileTree(snpStartReq.incrementIndex()).root());
else
- deleteSnapshot(snpOp.snapshotFileTree());
+ deleteLocalSnapshot(snpOp.snapshotFileTree(), true, false);
}
else if (!F.isEmpty(endReq.warnings())) {
// Pass the warnings further to the next stage for the case when snapshot started from not coordinator.
@@ -1446,6 +1563,27 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
return checkSnpProc.isSnapshotChecking(snpName);
}
+ /**
+ * @return {@code True} if a snapshot {@code snpName} delete operation is in progress.
+ */
+ public boolean isSnapshotDeleting(String snpName, @Nullable String snpPath) {
+ return deleteSnpProc.isDeleting(snpName, snpPath);
+ }
+
+ /**
+ * Deletes the cluster-wide snapshot with the given name.
+ * <p>
+ * The operation is rejected if a concurrent snapshot operation (create, restore, check, etc...) is in progress
+ * for the snapshot.
+ *
+ * @param name Snapshot name.
+ * @param snpPath Snapshot directory path. If {@code null}, the default configured snapshot directory will be used.
+ * @return Future which will be completed when the snapshot is deleted on all the online server nodes.
+ */
+ public IgniteInternalFuture<SnapshotDeleteProcessResult> deleteSnapshot(String name, @Nullable String snpPath) {
+ return deleteSnpProc.start(name, snpPath);
+ }
+
/**
* Sets the streamer warning flag to current snapshot process if it is active.
*/
@@ -1478,8 +1616,11 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
* @param snpName Snapshot name.
* @return {@code True} if the snapshot restore operation from the specified snapshot is in progress locally.
*/
- public boolean isRestoring(String snpName) {
- return snpName.equals(restoreCacheGrpProc.restoringSnapshotName());
+ public boolean isRestoring(@Nullable String snpName) {
+ if (snpName == null)
+ return false;
+
+ return snpName.equalsIgnoreCase(restoreCacheGrpProc.restoringSnapshotName());
}
/**
@@ -1830,6 +1971,21 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
* local node will be placed on the first place.
*/
public List<SnapshotMetadata> readSnapshotMetadatas(SnapshotFileTree sft) {
+ return readSnapshotMetadatas(sft, true);
+ }
+
+ /**
+ * Note, there can be snapshots from other nodes.
+ * This method will read all metadata.
+ * Some instances can return {@link SnapshotMetadata#folderName()} and {@link SnapshotMetadata#consistentId()} that differs from local.
+ *
+ * @param sft Snapshot file tree.
+ * @param failIfCantRead If {@code true}, throws an exception if cannot read a metadata file.
+ * @return List of snapshot metadata for the given snapshot name on local node.
+ * If snapshot has been taken from local node the snapshot metadata for given
+ * local node will be placed on the first place.
+ */
+ public List<SnapshotMetadata> readSnapshotMetadatas(SnapshotFileTree sft, boolean failIfCantRead) {
if (!(sft.root().exists() && sft.root().isDirectory()))
return Collections.emptyList();
@@ -1841,8 +1997,8 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
Map<String, SnapshotMetadata> metasMap = new HashMap<>();
SnapshotMetadata prev = null;
- try {
- for (File smf : smfs) {
+ for (File smf : smfs) {
+ try {
SnapshotMetadata curr = readSnapshotMetadata(smf);
if (prev != null && !prev.sameSnapshot(curr)) {
@@ -1854,9 +2010,14 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
prev = curr;
}
- }
- catch (IgniteCheckedException | IOException e) {
- throw new IgniteException(e);
+ catch (Exception e) {
+ String err = "Fail to read snapshot metadata [meta=" + smf + ']';
+
+ if (failIfCantRead)
+ throw new IgniteException(err, e);
+ else
+ log.error(err, e);
+ }
}
SnapshotMetadata currNodeSmf = metasMap.remove(cctx.localNode().consistentId().toString());
@@ -2032,7 +2193,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
if (!incremental && snpExists) {
throw new IgniteException("Create snapshot request has been rejected. " +
- "Snapshot with given name already exists on local node.");
+ "Snapshot with given name already exists on local node or the path is not empty.");
}
if (incremental) {
@@ -2263,7 +2424,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
if (SnapshotFileTree.incrementSnapshotDir(snpDir))
U.delete(snpDir);
else
- deleteSnapshot(snpDir);
+ deleteLocalSnapshot(snpDir);
}
if (log.isInfoEnabled()) {
@@ -3957,7 +4118,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
log.info("The Local snapshot sender closed. All resources released [dbNodeSnpDir=" + sft.nodeStorage() + ']');
}
else {
- deleteSnapshot(sft);
+ deleteLocalSnapshot(sft, true, false);
if (log.isDebugEnabled())
log.debug("Local snapshot sender closed due to an error occurred: " + th.getMessage());
@@ -4317,7 +4478,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
}
/** {@inheritDoc} */
- @Override public Boolean call() throws Exception {
+ @Override public Boolean call() {
if (reqId != null)
return ignite.context().cache().context().snapshotMgr().cancelLocalSnapshotOperations(reqId);
else {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotProcessor.java
index 9ba4277dd92..c1bdb9d69a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotProcessor.java
@@ -138,7 +138,8 @@ abstract class IncrementalSnapshotProcessor {
WALRecord rec = walRec.getValue();
- if (rec.type() == CLUSTER_SNAPSHOT && ((ClusterSnapshotRecord)rec).clusterSnapshotName().equals(sft.name())) {
+ // A filesystem might not support the character case of directory or file name.
+ if (rec.type() == CLUSTER_SNAPSHOT && ((ClusterSnapshotRecord)rec).clusterSnapshotName().equalsIgnoreCase(sft.name())) {
startIdx = walRec.getKey().index();
break;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java
index dd7ec0ef291..cc5a85092d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java
@@ -24,6 +24,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
@@ -32,6 +33,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteIllegalStateException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.GridKernalContext;
@@ -131,7 +133,7 @@ public class SnapshotCheckProcess {
return new GridFinishedFuture<>();
try {
- contexts.remove(ctx.req.snapshotName());
+ contexts.remove(ctx.req.snapshotName().toLowerCase(Locale.ROOT));
GridFutureAdapter<SnapshotPartitionsVerifyResult> clusterOpFut = clusterOpFuts.get(reqId);
@@ -473,7 +475,7 @@ public class SnapshotCheckProcess {
private @Nullable SnapshotCheckContext context(@Nullable String snpName, UUID reqId) {
return snpName == null
? contexts.values().stream().filter(ctx0 -> ctx0.req.requestId().equals(reqId)).findFirst().orElse(null)
- : contexts.get(snpName);
+ : contexts.get(snpName.toLowerCase(Locale.ROOT));
}
/** Phase 1 beginning: prepare, collect and check local metas. */
@@ -487,12 +489,17 @@ public class SnapshotCheckProcess {
if (nodeStopping)
return new GridFinishedFuture<>(new NodeStoppingException("The node is stopping: " + kctx.localNodeId()));
- ctx = contexts.computeIfAbsent(req.snapshotName(), snpName -> new SnapshotCheckContext(req));
+ ctx = contexts.computeIfAbsent(req.snapshotName().toLowerCase(Locale.ROOT), snpName -> new SnapshotCheckContext(req));
}
if (!ctx.req.requestId().equals(req.requestId())) {
- return new GridFinishedFuture<>(new IllegalStateException("Validation of snapshot '" + req.snapshotName()
- + "' has already started [ctx=" + ctx + ']'));
+ return new GridFinishedFuture<>(new IgniteIllegalStateException("Validation of snapshot '" + req.snapshotName()
+ + "' has already started [req=" + req + ']'));
+ }
+
+ if (kctx.cache().context().snapshotMgr().isSnapshotDeleting(req.snapshotName(), req.snapshotPath())) {
+ return new GridFinishedFuture<>(new IgniteIllegalStateException("Delete operation of snapshot '" + req.snapshotName() +
+ "' is in progress [req=" + req + ']'));
}
// Excludes non-baseline initiator.
@@ -587,7 +594,7 @@ public class SnapshotCheckProcess {
if (ctx != null) {
unregisterMetrics(ctx.req.snapshotName());
- contexts.remove(ctx.req.snapshotName());
+ contexts.remove(ctx.req.snapshotName().toLowerCase(Locale.ROOT));
}
if (clusterOpFut != null)
@@ -693,7 +700,7 @@ public class SnapshotCheckProcess {
/** @return {@code True} if snapshot with specified name is checking. */
boolean isSnapshotChecking(String snpName) {
- return contexts.get(snpName) != null;
+ return contexts.get(snpName.toLowerCase(Locale.ROOT)) != null;
}
/** @return {@code True} if node with the provided id is in the cluster and is a baseline node. {@code False} otherwise. */
@@ -771,7 +778,7 @@ public class SnapshotCheckProcess {
*/
@Nullable private volatile List<SnapshotMetadata> metas;
- /** Map of snapshot pathes per consistent id for {@link #metas}. */
+ /** Map of snapshot paths per consistent id for {@link #metas}. */
@GridToStringInclude
@Nullable private Map<String, SnapshotFileTree> locFileTree;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java
new file mode 100644
index 00000000000..baf90e82174
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java
@@ -0,0 +1,415 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.IgniteIllegalStateException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.NodeStoppingException;
+import org.apache.ignite.internal.processors.cache.persistence.filename.SnapshotFileTree;
+import org.apache.ignite.internal.util.distributed.DistributedProcess;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteReducer;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry.SNAPSHOT_DELETE_FEATURE;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.DELETE_SNAPSHOT;
+import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_SNAPSHOT;
+
+/**
+ * Distributed process to delete a cluster snapshot. The operation is rejected if any concurrent snapshot operation is
+ * active.
+ */
+public class SnapshotDeleteProcess {
+ /** Reject operation messages. */
+ private static final String OP_REJECT_MSG = "Snapshot deletion was rejected. ";
+
+ /** */
+ public static final String OP_REJECT_FEATURE_MSG = OP_REJECT_MSG + "The snapshot deletion feature isn't activated yet.";
+
+ /** */
+ private static final String CONCURRENT_OP_PREF = "Snapshot with the same name is being ";
+
+ /** */
+ public static final String BEING_CREATED_PREF = CONCURRENT_OP_PREF + "created ";
+
+ /** */
+ public static final String BEING_RESTORED_PREF = CONCURRENT_OP_PREF + "restored ";
+
+ /** */
+ public static final String BEING_CHECKED_PREF = CONCURRENT_OP_PREF + "checked ";
+
+ /** Kernal context. */
+ private final GridKernalContext kctx;
+
+ /** Logger. */
+ private final IgniteLogger log;
+
+ /** */
+ private volatile boolean interrupted;
+
+ /** Cluster-wide operation futures per request id on certain node. */
+ private final Map<UUID, GridFutureAdapter<SnapshotDeleteProcessResult>> clusterOpFuts = new ConcurrentHashMap<>();
+
+ /** Current operations represented by the full canonical path. */
+ private final Set<File> requests = ConcurrentHashMap.newKeySet();
+
+ /** The distributed process. */
+ private final DistributedProcess<SnapshotDeleteRequest, SnapshotDeleteResponse> distrProc;
+
+ /**
+ * @param ctx Kernal context.
+ */
+ public SnapshotDeleteProcess(GridKernalContext ctx) {
+ kctx = ctx;
+
+ log = ctx.log(getClass());
+
+ distrProc = new DistributedProcess<>(ctx, DELETE_SNAPSHOT, this::deletePhase, this::reducePhase);
+ }
+
+ /**
+ * Starts the cluster snapshot delete process.
+ *
+ * @param snpName Snapshot name.
+ * @param snpPath Snapshot directory path (optional).
+ * @return Future that will be completed when the snapshot is deleted.
+ */
+ public IgniteInternalFuture<SnapshotDeleteProcessResult> start(String snpName, @Nullable String snpPath) {
+ GridFutureAdapter<SnapshotDeleteProcessResult> clusterOpFut = new GridFutureAdapter<>();
+
+ if (!kctx.rollingUpgrade().features().isActive(SNAPSHOT_DELETE_FEATURE)) {
+ clusterOpFut.onDone(new IgniteIllegalStateException(OP_REJECT_FEATURE_MSG));
+
+ return clusterOpFut;
+ }
+
+ UUID reqId = UUID.randomUUID();
+
+ clusterOpFut.listen(fut -> clusterOpFuts.remove(reqId));
+
+ try {
+ if (interrupted || kctx.isStopping())
+ throw new NodeStoppingException("Failed to start snapshot delete process: node is stopping.");
+
+ clusterOpFuts.put(reqId, clusterOpFut);
+
+ SnapshotDeleteRequest req = new SnapshotDeleteRequest(reqId, snpName, snpPath);
+
+ distrProc.start(reqId, req);
+ }
+ catch (Throwable t) {
+ log.error("Failed to start distributed delete snapshot process [snpName=" + snpName + ", snpPath=" + snpPath + ']', t);
+
+ clusterOpFut.onDone(t);
+ }
+
+ return clusterOpFut;
+ }
+
+ /** */
+ private IgniteInternalFuture<SnapshotDeleteResponse> deletePhase(UUID ignored, SnapshotDeleteRequest req) {
+ if (interrupted || kctx.isStopping()) {
+ return new GridFinishedFuture<>(new NodeStoppingException(OP_REJECT_MSG +
+ " Node is stopping [req=" + req + ']'));
+ }
+
+ if (kctx.cluster().get().localNode().isClient())
+ return new GridFinishedFuture<>(new SnapshotDeleteResponse());
+
+ kctx.security().authorize(ADMIN_SNAPSHOT);
+
+ IgniteSnapshotManager snpMgr = kctx.cache().context().snapshotMgr();
+
+ SnapshotOperationRequest curCreateRq = snpMgr.currentCreateRequest();
+
+ if (curCreateRq != null && curCreateRq.snpName.equalsIgnoreCase(req.snpName)) {
+ return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + BEING_CREATED_PREF +
+ "[req=" + req + ']'));
+ }
+
+ if (snpMgr.isRestoring(req.snpName)) {
+ return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + BEING_RESTORED_PREF +
+ "[req=" + req + ']'));
+ }
+
+ if (snpMgr.isSnapshotChecking(req.snpName)) {
+ return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + BEING_CHECKED_PREF +
+ "[req=" + req + ']'));
+ }
+
+ File resolvedFullPath = null;
+
+ // Future to delete snapshot contents according to snapshot metadatas.
+ GridFutureAdapter<SnapshotDeleteResponse> resultFut = new GridFutureAdapter<>();
+
+ try {
+ resolvedFullPath = resolveFullPath(req.snpName, req.snpPath);
+
+ if (!requests.add(resolvedFullPath)) {
+ return new GridFinishedFuture<>(new IgniteIllegalStateException("Deletion of the snapshot has already " +
+ "started [req=" + req + ']'));
+ }
+
+ File rootPath = resolvedFullPath.getParentFile();
+
+ SnapshotFileTree snpFiles = new SnapshotFileTree(kctx, req.snpName, rootPath.getAbsolutePath());
+
+ // We need to find and read snapshot metas to ensure the content is a snapshot. Also, the metas contain
+ // initial cluster topology and actual snapshot folder names.
+ List<SnapshotMetadata> locMetas = kctx.cache().context().snapshotMgr().readSnapshotMetadatas(snpFiles, false);
+
+ if (locMetas.isEmpty()) {
+ requests.remove(resolvedFullPath);
+
+ log.warning("Snapshot deletion won't process, no snapshot metadata found [req=" + req + ']');
+
+ return new GridFinishedFuture<>(new SnapshotDeleteResponse(SnapshotDeleteResponse.DeleteStatus.NOT_FOUND, null));
+ }
+
+ File resolvedFullPath0 = resolvedFullPath;
+
+ resultFut.listen(fut -> requests.remove(resolvedFullPath0));
+
+ // Per-metas results.
+ AtomicInteger pendingMetaFuts = new AtomicInteger(locMetas.size());
+ MetaFuturesReducer resReducer = new MetaFuturesReducer();
+ AtomicReference<Throwable> resErr = new AtomicReference<>();
+
+ for (SnapshotMetadata meta : locMetas) {
+ GridFutureAdapter<SnapshotDeleteResponse> perMetaFut = new GridFutureAdapter<>();
+
+ // Allows to wait until all the per-meta futures are done.
+ perMetaFut.listen(metaFut -> {
+ if (metaFut.error() != null)
+ resErr.compareAndSet(null, metaFut.error());
+ else
+ resReducer.collect(metaFut.result());
+
+ if (pendingMetaFuts.decrementAndGet() == 0) {
+ if (resErr.get() == null)
+ resultFut.onDone(resReducer.reduce());
+ else
+ resultFut.onDone(resErr.get());
+ }
+ });
+
+ kctx.pools().getSnapshotExecutorService().submit(() -> {
+ try {
+ // Read file tree of the snapshot.
+ SnapshotFileTree byMetaSft = new SnapshotFileTree(
+ kctx.config(),
+ kctx.pdsFolderResolver().fileTree(),
+ req.snpName,
+ req.snpPath == null ? null : rootPath.getAbsolutePath(),
+ meta.folderName(),
+ meta.consId
+ );
+
+ T2<Boolean, Boolean> deleted = snpMgr.deleteLocalSnapshot(byMetaSft, false, true);
+
+ SnapshotDeleteResponse.DeleteStatus status;
+
+ // If found.
+ if (deleted.get2()) {
+ if (deleted.get1() && log.isInfoEnabled())
+ log.info("Snapshot successfully deleted [req=" + req + ']');
+ else if (!deleted.get1())
+ log.warning("Snapshot deleted not completely [req=" + req + ']');
+
+ status = deleted.get1()
+ ? SnapshotDeleteResponse.DeleteStatus.DELETED
+ : SnapshotDeleteResponse.DeleteStatus.PARTLY;
+ }
+ else {
+ if (log.isInfoEnabled())
+ log.info("Snapshot not found to delete [req=" + req + ']');
+
+ status = SnapshotDeleteResponse.DeleteStatus.NOT_FOUND;
+ }
+
+ perMetaFut.onDone(new SnapshotDeleteResponse(status, meta.bltNodes));
+ }
+ catch (Throwable e) {
+ perMetaFut.onDone(e);
+ }
+ });
+ }
+
+ if (log.isInfoEnabled())
+ log.info("Deletion of snapshot initialized [req=" + req + ']');
+ }
+ catch (Throwable t) {
+ if (resolvedFullPath != null)
+ requests.remove(resolvedFullPath);
+
+ log.warning("An error occurred during snapshot deletion [req=" + req + ']', t);
+
+ resultFut.onDone(t);
+ }
+
+ return resultFut;
+ }
+
+ /** */
+ private File resolveFullPath(String name, @Nullable String path) throws IOException {
+ return new SnapshotFileTree(kctx, name, path).root().getCanonicalFile();
+ }
+
+ /** */
+ private void reducePhase(UUID reqId, Map<UUID, SnapshotDeleteResponse> results, Map<UUID, Throwable> errors) {
+ GridFutureAdapter<SnapshotDeleteProcessResult> clusterOpFut = clusterOpFuts.get(reqId);
+
+ if (clusterOpFut == null)
+ return;
+
+ try {
+ Map.Entry<UUID, Throwable> errP = F.isEmpty(errors) ? null : F.first(errors.entrySet());
+
+ if (errP != null) {
+ log.warning("Snapshot deletion finished with an error [reqId=" + reqId + ", nodeId="
+ + errP.getKey() + ", err='" + errP.getValue().getMessage() + "']", errP.getValue());
+
+ clusterOpFut.onDone(errP.getValue());
+
+ return;
+ }
+
+ Map<UUID, String> completedNodes = U.newHashMap(results.size());
+ Map<UUID, String> uncompletedNodes = U.newHashMap(results.size());
+ Map<UUID, String> emptyNodes = U.newHashMap(results.size());
+ Set<String> snpNodes = new HashSet<>();
+
+ results.forEach((nodeId, nodeRes) -> {
+ if (!F.isEmpty(nodeRes.nodeIds))
+ snpNodes.addAll(nodeRes.nodeIds);
+
+ if (nodeRes.status != null) {
+ switch (nodeRes.status) {
+ case NOT_FOUND:
+ emptyNodes.put(nodeId, optionalConsistentId(nodeId));
+ break;
+ case DELETED:
+ completedNodes.put(nodeId, optionalConsistentId(nodeId));
+ break;
+ case PARTLY:
+ uncompletedNodes.put(nodeId, optionalConsistentId(nodeId));
+ break;
+ default:
+ throw new IgniteIllegalStateException("Unknown snapshot deletion node result, [nodeRes=" +
+ nodeRes + ", nodeId=" + nodeId + ']');
+ }
+ }
+ });
+
+ Collection<ClusterNode> curBaseline = kctx.discovery().discoCache().aliveBaselineNodes();
+
+ if (!F.isEmpty(curBaseline))
+ curBaseline.stream().map(bn -> bn.consistentId().toString()).toList().forEach(snpNodes::remove);
+
+ clusterOpFut.onDone(new SnapshotDeleteProcessResult(completedNodes, uncompletedNodes, emptyNodes, snpNodes));
+ }
+ catch (Throwable t) {
+ clusterOpFut.onDone(t);
+ }
+ }
+
+ /** */
+ private String optionalConsistentId(UUID nodeId) {
+ ClusterNode node = kctx.discovery().node(nodeId);
+
+ if (node == null)
+ node = kctx.discovery().historicalNode(nodeId);
+
+ return node == null ? "" : node.consistentId().toString();
+ }
+
+ /** */
+ public boolean isDeleting(String snpName, @Nullable String snpPath) {
+ File fullPath;
+
+ try {
+ fullPath = resolveFullPath(snpName, snpPath);
+ }
+ catch (IOException e) {
+ return false;
+ }
+
+ return requests.contains(fullPath);
+ }
+
+ /**
+ * @param err The interrupt reason.
+ */
+ void interrupt(Throwable err) {
+ interrupted = true;
+
+ clusterOpFuts.forEach((reqId, clusterOpFut) -> clusterOpFut.onDone(err));
+ }
+
+ /** */
+ private static class MetaFuturesReducer implements IgniteReducer<SnapshotDeleteResponse, SnapshotDeleteResponse> {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private @Nullable SnapshotDeleteResponse.DeleteStatus status;
+
+ /** */
+ private final Collection<String> nodeIds = new HashSet<>();
+
+ /** {@inheritDoc} */
+ @Override public boolean collect(SnapshotDeleteResponse res) {
+ assert res != null;
+
+ synchronized (this) {
+ if (!F.isEmpty(res.nodeIds))
+ nodeIds.addAll(res.nodeIds);
+
+ if (status == null)
+ status = res.status;
+ else if (status != res.status)
+ status = SnapshotDeleteResponse.DeleteStatus.PARTLY;
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public SnapshotDeleteResponse reduce() {
+ return new SnapshotDeleteResponse(status == null ? SnapshotDeleteResponse.DeleteStatus.NOT_FOUND : status, nodeIds);
+ }
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcessResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcessResult.java
new file mode 100644
index 00000000000..5bf62b5373c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcessResult.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.Order;
+import org.apache.ignite.internal.dto.IgniteDataTransferObject;
+import org.apache.ignite.plugin.extensions.communication.MessageFactory;
+
+/** Result of {@link SnapshotDeleteProcess}. */
+public final class SnapshotDeleteProcessResult extends IgniteDataTransferObject {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 0L;
+
+ /** Nodes which found snapshot data and completely removed it. */
+ @Order(0)
+ Map<UUID, String> completedNodes;
+
+ /** Nodes which found snapshot data but didn't remove it completely. */
+ @Order(1)
+ Map<UUID, String> uncompletedNodes;
+
+ /** Server nodes which didn't find any snapshot data. */
+ @Order(2)
+ Map<UUID, String> emptyNodes;
+
+ /** Snapshot's baseline nodes missing in current cluster. */
+ @Order(3)
+ Collection<String> absentBaselines;
+
+ /** Default constructor for {@link MessageFactory}. */
+ public SnapshotDeleteProcessResult() {
+ // No-op.
+ }
+
+ /** */
+ public SnapshotDeleteProcessResult(
+ Map<UUID, String> completedNodes,
+ Map<UUID, String> uncompletedNodes,
+ Map<UUID, String> emptyNodes,
+ Collection<String> absentBaselines
+ ) {
+ this.completedNodes = completedNodes;
+ this.uncompletedNodes = uncompletedNodes;
+ this.emptyNodes = emptyNodes;
+ this.absentBaselines = absentBaselines;
+ }
+
+ /** */
+ public Map<UUID, String> completedNodes() {
+ return Collections.unmodifiableMap(completedNodes);
+ }
+
+ /** */
+ public Map<UUID, String> uncompletedNodes() {
+ return Collections.unmodifiableMap(uncompletedNodes);
+ }
+
+ /** */
+ public Map<UUID, String> emptyNodes() {
+ return Collections.unmodifiableMap(emptyNodes);
+ }
+
+ /** */
+ public Collection<String> absentBaselines() {
+ return Collections.unmodifiableCollection(absentBaselines);
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java
new file mode 100644
index 00000000000..51cafd30259
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.util.UUID;
+import org.apache.ignite.internal.Order;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.AbstractMessage;
+import org.apache.ignite.plugin.extensions.communication.MessageFactory;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Cluster snapshot delete distributed process request.
+ *
+ * @see SnapshotDeleteProcess
+ */
+public class SnapshotDeleteRequest extends AbstractMessage {
+ /** Request ID. */
+ @Order(0)
+ UUID reqId;
+
+ /** Snapshot name. */
+ @Order(1)
+ String snpName;
+
+ /** Snapshot directory path. */
+ @Order(2)
+ @Nullable String snpPath;
+
+ /** Default constructor for {@link MessageFactory}. */
+ public SnapshotDeleteRequest() {
+ // No-op.
+ }
+
+ /**
+ * @param reqId Request ID.
+ * @param snpName Snapshot name.
+ * @param snpPath Snapshot directory path.
+ */
+ SnapshotDeleteRequest(UUID reqId, String snpName, @Nullable String snpPath) {
+ this.reqId = reqId;
+ this.snpName = snpName;
+ this.snpPath = snpPath;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(SnapshotDeleteRequest.class, this);
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteResponse.java
new file mode 100644
index 00000000000..607ec310238
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteResponse.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.util.Collection;
+import org.apache.ignite.internal.Order;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.AbstractMessage;
+import org.apache.ignite.plugin.extensions.communication.MessageFactory;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Single-node result of the snapshot deletion distributed process.
+ *
+ * @see SnapshotDeleteProcess
+ */
+public class SnapshotDeleteResponse extends AbstractMessage {
+ /** {@code null} for client node. */
+ @Order(0)
+ @Nullable SnapshotDeleteResponse.DeleteStatus status;
+
+ /** Snapshot's node ids. */
+ @Order(1)
+ @Nullable Collection<String> nodeIds;
+
+ /** Default constructor for {@link MessageFactory}. */
+ public SnapshotDeleteResponse() {
+ // No-op.
+ }
+
+ /** {@code null} for client node. */
+ SnapshotDeleteResponse(DeleteStatus status, Collection<String> nodeIds) {
+ this.status = status;
+ this.nodeIds = nodeIds;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(SnapshotDeleteResponse.class, this);
+ }
+
+ /** */
+ enum DeleteStatus {
+ /** Snapshot found and completely deleted. */
+ DELETED,
+
+ /** Snapshot found but some files or directories might not be deleted (locked). */
+ PARTLY,
+
+ /** Snapshot not found. */
+ NOT_FOUND;
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java
index 643838ac3c4..4a8fea2b43b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java
@@ -35,7 +35,10 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescripto
import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
import org.apache.ignite.internal.util.typedef.F;
-/** Snapshot task to verify snapshot metadata on the baseline nodes for given snapshot name. */
+/**
+ * Snapshot task to verify snapshot metadata on the baseline nodes for given snapshot name.
+ * TODO : Revise in https://issues.apache.org/jira/browse/IGNITE-29062
+ */
public class SnapshotMetadataVerificationTask implements Supplier<List<SnapshotMetadata>> {
/** */
private final IgniteEx ignite;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
index 43db339d103..2643a5575f6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
@@ -659,6 +659,11 @@ public class SnapshotRestoreProcess {
if (snpMgr.isSnapshotCreating())
throw new IgniteCheckedException(OP_REJECT_MSG + "A cluster snapshot operation is in progress.");
+ if (snpMgr.isSnapshotDeleting(req.snapshotName(), req.snapshotPath())) {
+ throw new IgniteCheckedException(OP_REJECT_MSG + "Delete operation of '" + req.snapshotName() +
+ "' is in progress.");
+ }
+
if (ctx.encryption().isMasterKeyChangeInProgress()) {
return new GridFinishedFuture<>(new IgniteCheckedException(OP_REJECT_MSG + "Master key changing " +
"process is not finished yet."));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metastorage/persistence/DistributedMetaStorageImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metastorage/persistence/DistributedMetaStorageImpl.java
index 575300772a9..bf7f6f34c25 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/metastorage/persistence/DistributedMetaStorageImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metastorage/persistence/DistributedMetaStorageImpl.java
@@ -103,7 +103,7 @@ import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metr
* </p>
* <p>Whole updates history until some point in the past is stored along with the data, so when an outdated node
* connects to the cluster it will receive all the missing data and apply it locally. Listeners will also be invoked
- * after such updates. If there's not enough history stored or joining node is clear then it'll receive shapshot of
+ * after such updates. If there's not enough history stored or joining node is clear then it'll receive snapshot of
* distributed metastorage (usually called {@code fullData} in code) so there won't be inconsistencies.
* </p>
*
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java
index 4e67d42d174..6952606d1d5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java
@@ -105,4 +105,7 @@ public class CoreFeatureRegistry {
* This mechanism was replaced by the raw-field suffix.
*/
public static final IgniteFeature PREV_CLUSTER_VER_DEPRECATED_FEATURE = new IgniteCoreFeature(2);
+
+ /** Feature that introduces the snapshot deletion command. */
+ public static final IgniteFeature SNAPSHOT_DELETE_FEATURE = new IgniteCoreFeature(3);
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
index dcbd07fac68..d1e6a206653 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
@@ -574,5 +574,10 @@ public class DistributedProcess<I extends Message, R extends Message> {
* Cluster version finalization abort process.
*/
RU_ABORT_VERSION_FINALIZATION,
+
+ /**
+ * Delete snapshot procedure.
+ */
+ DELETE_SNAPSHOT
}
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java
index 8857a4ddd65..d1d6dc9fde0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java
@@ -36,7 +36,9 @@ import java.util.Optional;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
@@ -90,6 +92,7 @@ import org.apache.ignite.internal.util.future.IgniteFutureImpl;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -98,6 +101,8 @@ import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteFutureCancelledException;
import org.apache.ignite.lang.IgniteFutureTimeoutException;
import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.plugin.AbstractTestPluginProvider;
+import org.apache.ignite.plugin.PluginContext;
import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
@@ -148,6 +153,9 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
protected static final int PAGE_SIZE =
IgniteSystemProperties.getInteger(IGNITE_DEFAULT_DATA_STORAGE_PAGE_SIZE, DFLT_PAGE_SIZE);
+ /** */
+ protected static boolean caseInsensitiveFs;
+
/** List of collected snapshot test events. */
protected final List<Integer> locEvts = new CopyOnWriteArrayList<>();
@@ -173,6 +181,9 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
return valBuilder;
}
+ /** */
+ protected @Nullable AbstractTestPluginProvider pluginProvider;
+
/** Enable encryption of all caches in {@code IgniteConfiguration} before start. */
@Parameterized.Parameter
public boolean encryption;
@@ -182,7 +193,7 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
public boolean onlyPrimary;
/** Parameters. */
- @Parameterized.Parameters(name = "encryption={0}, onlyPrimay={1}")
+ @Parameterized.Parameters(name = "encryption={0}, onlyPrimary={1}")
public static Collection<Object[]> params() {
List<Object[]> res = new ArrayList<>();
@@ -200,6 +211,18 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
: F.asList(false, true);
}
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ File workDir = new File(U.defaultWorkDirectory());
+
+ assertTrue(workDir.exists());
+
+ caseInsensitiveFs = new File(workDir.getAbsolutePath().toLowerCase()).exists() &&
+ new File(workDir.getAbsolutePath().toUpperCase()).exists();
+ }
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -217,6 +240,9 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
if (cfg.isClientMode())
return cfg;
+ if (pluginProvider != null)
+ cfg.setPluginProviders(pluginProvider);
+
return cfg.setConsistentId(igniteInstanceName)
.setDataStorageConfiguration(new DataStorageConfiguration()
.setDefaultDataRegionConfiguration(new DataRegionConfiguration()
@@ -280,7 +306,33 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
stopAllGrids();
}
- cleanPersistenceDir();
+ cleanPersistenceDir(false);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void cleanPersistenceDir() throws Exception {
+ if (cleanWorkDirectoryFully())
+ return;
+
+ super.cleanPersistenceDir();
+ }
+
+ /**
+ * Cleans everything in the persistent directory, including the snapshot working directory,
+ * custom snapshot paths, and so on.
+ * <p>
+ * If full cleanup is not required, this method might be overridden to return {@code false},
+ * so that the default {@link #cleanPersistenceDir()} will be used.
+ *
+ * @return {@code True} if the cleanup is done, {@code False} if the default cleanup is required.
+ */
+ protected boolean cleanWorkDirectoryFully() throws Exception {
+ try (DirectoryStream<Path> files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) {
+ for (Path path : files)
+ U.delete(path);
+ }
+
+ return true;
}
/**
@@ -701,8 +753,8 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
* @param ignite Ignite instance.
* @return Snapshot manager related to given ignite instance.
*/
- public static IgniteSnapshotManager snp(IgniteEx ignite) {
- return ignite.context().cache().context().snapshotMgr();
+ public static IgniteSnapshotManager snp(Ignite ignite) {
+ return ((IgniteEx)ignite).context().cache().context().snapshotMgr();
}
/**
@@ -745,7 +797,7 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
List<BlockingExecutor> execs = new ArrayList<>();
for (Ignite grid : grids) {
- IgniteSnapshotManager mgr = snp((IgniteEx)grid);
+ IgniteSnapshotManager mgr = snp(grid);
Function<SnapshotFileTree, SnapshotSender> old = mgr.localSnapshotSenderFactory();
BlockingExecutor block = new BlockingExecutor(mgr.snapshotExecutorService());
@@ -814,6 +866,86 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
assertEquals("Snapshot directory must be empty due to snapshot cancelled", 0, snpDir.list().length);
}
+ /** Tests concurrent snapshot operation when snapshot deletion with the same name is active. */
+ protected void doConcurrentSnapshotOperationWhenDeletionIsActive(
+ ExRunnable prepareCluster,
+ ExRunnable concurrentOp,
+ @Nullable Function<Exception, Boolean> errValidator,
+ boolean rerunAtTheEnd
+ ) throws Exception {
+ CountDownLatch delProcInitLatch = new CountDownLatch(1);
+ CountDownLatch delProcProceedLatch = new CountDownLatch(1);
+
+ pluginProvider = new AbstractTestPluginProvider() {
+ @Override public String name() {
+ return "TestSnpMgrProvider";
+ }
+
+ @Override public <T> T createComponent(PluginContext ctx, Class<T> cls) {
+ if (IgniteSnapshotManager.class.isAssignableFrom(cls)) {
+ return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) {
+ @Override public T2<Boolean, Boolean> deleteLocalSnapshot(
+ SnapshotFileTree sft,
+ boolean scoped,
+ boolean ignoreErrs
+ ) {
+ delProcInitLatch.countDown();
+
+ try {
+ assertTrue(delProcProceedLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS));
+ }
+ catch (InterruptedException e) {
+ throw new RuntimeException("Interrupted.", e);
+ }
+
+ return super.deleteLocalSnapshot(sft, scoped, ignoreErrs);
+ }
+ };
+ }
+
+ return super.createComponent(ctx, cls);
+ }
+ };
+
+ prepareCluster.run();
+
+ IgniteInternalFuture<SnapshotDeleteProcessResult> delFut = snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null);
+
+ assertTrue(delProcInitLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS));
+
+ try {
+ concurrentOp.run();
+
+ if (errValidator != null)
+ throw new IllegalStateException("Exception is not thrown.");
+ }
+ catch (Exception e) {
+ if (errValidator == null || !errValidator.apply(e))
+ throw new IllegalStateException("Unexpected exception: " + e.getMessage(), e);
+ }
+
+ delProcProceedLatch.countDown();
+
+ delFut.get(getTestTimeout());
+
+ for (Ignite node : G.allGrids())
+ assertFalse(new SnapshotFileTree(((IgniteEx)node).context(), SNAPSHOT_NAME, null).root().exists());
+
+ if (!rerunAtTheEnd)
+ return;
+
+ assertThrowsAnyCause(
+ null,
+ () -> {
+ concurrentOp.run();
+
+ return null;
+ },
+ IllegalArgumentException.class,
+ "Snapshot does not exists "
+ );
+ }
+
/**
* @param sft Snapshot file tree.
* @param parts Collection of pairs group and appropriate cache partition to be snapshot.
@@ -994,6 +1126,13 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
}
}
+ /** */
+ @FunctionalInterface
+ protected interface ExRunnable {
+ /** */
+ void run() throws Exception;
+ }
+
/** */
protected static class Value {
/** */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java
index f6ae22c2db4..913a4dc4259 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java
@@ -232,7 +232,7 @@ public class EncryptedSnapshotTest extends AbstractSnapshotSelfTest {
assertTrue(snpDir.renameTo(tmpSnpDir));
- cleanPersistenceDir();
+ cleanPersistenceDir(false);
assertTrue(tmpSnpDir.renameTo(snpDir));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java
index 69b23adbe46..695c3b42fa3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java
@@ -46,6 +46,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteIllegalStateException;
import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cluster.BaselineNode;
@@ -124,6 +125,7 @@ import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause
import static org.apache.ignite.testframework.GridTestUtils.cartesianProduct;
import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
import static org.junit.Assume.assumeFalse;
+import static org.junit.Assume.assumeTrue;
/**
* Cluster-wide snapshot check procedure tests.
@@ -1207,6 +1209,39 @@ public class IgniteClusterSnapshotCheckTest extends AbstractSnapshotSelfTest {
);
}
+ /** */
+ @Test
+ public void testConcurrentSnapshotDeleteAndCheckOperations() throws Exception {
+ String snapshotName = caseInsensitiveFs ? SNAPSHOT_NAME.toLowerCase() : SNAPSHOT_NAME;
+
+ doConcurrentSnapshotOperationWhenDeletionIsActive(
+ () -> prepareGridsAndSnapshot(4, 3, 1, false),
+ () -> snp(grid(2)).checkSnapshot(snapshotName, null).get(),
+ e -> e.getMessage().contains("Delete operation of snapshot '%s' is in progress".formatted(snapshotName)),
+ true
+ );
+ }
+
+ /** */
+ @Test
+ public void testConcurrentSnapshotDeleteAndCheckOperationsWithDifferentPath() throws Exception {
+ // The test uses thread blocking.
+ assumeTrue(snpThrdPoolSz > 1);
+
+ String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath();
+
+ doConcurrentSnapshotOperationWhenDeletionIsActive(
+ () -> {
+ prepareGridsAndSnapshot(4, 3, 1, false);
+
+ snp(grid(0)).createSnapshot(SNAPSHOT_NAME, snpPath, false, false).get(getTestTimeout());
+ },
+ () -> snp(grid(2)).checkSnapshot(SNAPSHOT_NAME, snpPath).get(),
+ null,
+ false
+ );
+ }
+
/** Tests that concurrent snapshot full check is declined when the same snapshot is being fully restored (checked). */
@Test
public void testConcurrentTheSameSnpFullCheckWhenFullyRestoringDeclined() throws Exception {
@@ -1415,8 +1450,8 @@ public class IgniteClusterSnapshotCheckTest extends AbstractSnapshotSelfTest {
/**
* Tests concurrent snapshot operations related to the snapshot checking.
*
- * @param originatorOp First snapshot operation on an originator node.
- * @param trierOp Second concurrent snapshot operation on a trier node.
+ * @param firstOp First snapshot operation on an originator node.
+ * @param secondOp Second concurrent snapshot operation on a trier node.
* @param firstDelay First distributed process full message of {@code originatorOp} to delay on the coordinator
* to launch {@code trierOp}.
* @param secondDelay Second distributed process full message of {@code originatorOp} to delay on the coordinator
@@ -1428,8 +1463,8 @@ public class IgniteClusterSnapshotCheckTest extends AbstractSnapshotSelfTest {
* @param cleaner If not {@code null}, is executed at the end.
*/
private void doTestConcurrentSnpCheckOperations(
- Supplier<IgniteFuture<?>> originatorOp,
- Supplier<IgniteFuture<?>> trierOp,
+ Supplier<IgniteFuture<?>> firstOp,
+ Supplier<IgniteFuture<?>> secondOp,
DistributedProcess.DistributedProcessType firstDelay,
@Nullable DistributedProcess.DistributedProcessType secondDelay,
boolean expectFailure,
@@ -1446,17 +1481,17 @@ public class IgniteClusterSnapshotCheckTest extends AbstractSnapshotSelfTest {
&& ((FullMessage<?>)msg).type() == firstDelay.ordinal()
&& (waitForBothFirstDelays || firstDelayed.compareAndSet(false, true)));
- IgniteFuture<?> fut = originatorOp.get();
+ IgniteFuture<?> fut = firstOp.get();
discoSpi(grid(0)).waitBlocked(getTestTimeout());
- IgniteFuture<?> fut2 = trierOp.get();
+ IgniteFuture<?> fut2 = secondOp.get();
if (expectFailure) {
assertThrowsAnyCause(
log,
fut2::get,
- IllegalStateException.class,
+ IgniteIllegalStateException.class,
"Validation of snapshot '" + SNAPSHOT_NAME + "' has already started"
);
@@ -1479,7 +1514,7 @@ public class IgniteClusterSnapshotCheckTest extends AbstractSnapshotSelfTest {
assertThrowsAnyCause(
log,
fut2::get,
- IllegalStateException.class,
+ IgniteIllegalStateException.class,
"Validation of snapshot '" + SNAPSHOT_NAME + "' has already started"
);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteRollingUpgradeTest.java
new file mode 100644
index 00000000000..cdd5f2d4740
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteRollingUpgradeTest.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.io.File;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteIllegalStateException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.management.snapshot.SnapshotDeleteCommandArg;
+import org.apache.ignite.internal.management.snapshot.SnapshotDeleteTask;
+import org.apache.ignite.internal.processors.rollingupgrade.AbstractRollingUpgradeTest;
+import org.apache.ignite.internal.util.distributed.SingleNodeMessage;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.Test;
+
+import static java.nio.file.Files.newDirectoryStream;
+import static org.apache.ignite.internal.TestRecordingCommunicationSpi.spi;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RU_PREPARE_VERSION_FINALIZATION;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
+/** */
+public class IgniteClusterSnapshotDeleteRollingUpgradeTest extends AbstractRollingUpgradeTest {
+ /** */
+ private static final int ALL_GRIDS = 4;
+
+ /** */
+ private static final int CLIENTS = 1;
+
+ /** */
+ private static final String SNP_NAME = "testSnapshot";
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ cleanPersistenceDir();
+
+ // Clean all: also separated snapshot working directories.
+ try (DirectoryStream<Path> files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) {
+ for (Path path : files)
+ U.delete(path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName, String ver) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName, ver);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+ )
+ );
+
+ cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath());
+
+ return cfg;
+ }
+
+ /** */
+ @Test
+ public void testParallelRollingUpgradeInProgress() throws Exception {
+ for (int i = 0; i < ALL_GRIDS; i++)
+ startGrid(i, "2.18.0", i >= ALL_GRIDS - CLIENTS);
+
+ grid(0).cluster().active(true);
+
+ int testNodeIx = ALL_GRIDS - CLIENTS - 1;
+
+ createCacheAndSnapshot(testNodeIx);
+
+ ru(grid(testNodeIx)).enableVersionUpgrade();
+
+ for (int i = 0; i < ALL_GRIDS; i++) {
+ assertTrue(ru(grid(i)).isVersionUpgradeEnabled());
+
+ upgradeNodeVersion(i, "2.18.0", MASTER_VER_REF);
+ }
+
+ spi(grid(testNodeIx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage<?> snm &&
+ snm.type() == RU_PREPARE_VERSION_FINALIZATION.ordinal());
+
+ IgniteInternalFuture<?> finalizeFut = GridTestUtils.runAsync(() -> ru(testNodeIx).finalizeClusterVersion());
+
+ assertTrue(spi(grid(testNodeIx)).waitForBlocked(1, getTestTimeout()));
+
+ ensureSnapshotDeletionFailed(false);
+
+ spi(grid(testNodeIx)).stopBlock();
+
+ assertFalse(spi(grid(testNodeIx)).hasBlockedMessages());
+
+ finalizeFut.get(getTestTimeout());
+
+ for (int i = 0; i < ALL_GRIDS; i++) {
+ int i0 = i;
+
+ assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout()));
+ }
+
+ assertFalse(snp(1).deleteSnapshot(SNP_NAME, null).get(getTestTimeout()).completedNodes().isEmpty());
+ }
+
+ /** */
+ @Test
+ public void testSnapshotDeleteFeature() throws Exception {
+ doTestSnapshotDeleteFeature(false);
+ }
+
+ /** */
+ @Test
+ public void testSnapshotDeleteFeatureWithTask() throws Exception {
+ doTestSnapshotDeleteFeature(true);
+ }
+
+ /** */
+ private void doTestSnapshotDeleteFeature(boolean useTask) throws Exception {
+ for (int i = 0; i < ALL_GRIDS; i++)
+ startGrid(i, "2.18.0", i >= ALL_GRIDS - CLIENTS);
+
+ grid(0).cluster().active(true);
+
+ createCacheAndSnapshot(1);
+
+ ensureSnapshotDeletionFailed(useTask);
+
+ ru(grid(0)).enableVersionUpgrade();
+
+ for (int i = 0; i < ALL_GRIDS; i++) {
+ assertTrue(ru(grid(i)).isVersionUpgradeEnabled());
+
+ upgradeNodeVersion(i, "2.18.0", MASTER_VER_REF);
+
+ ensureSnapshotDeletionFailed(useTask);
+ }
+
+ ru(grid(1)).finalizeClusterVersion();
+
+ for (int i = 0; i < ALL_GRIDS; i++) {
+ int i0 = i;
+
+ assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout()));
+ }
+
+ SnapshotDeleteProcessResult delRes;
+
+ if (useTask) {
+ SnapshotDeleteCommandArg args = new SnapshotDeleteCommandArg();
+
+ args.snapshotName(SNP_NAME);
+
+ IgniteEx ig = grid(1);
+
+ delRes = ig.compute().execute(new SnapshotDeleteTask(), new VisorTaskArgument<>(ig.localNode().id(), args, false)).result();
+
+ assertNotNull(delRes);
+ }
+ else
+ delRes = snp(1).deleteSnapshot(SNP_NAME, null).get(getTestTimeout());
+
+ assertEquals(ALL_GRIDS - CLIENTS, delRes.completedNodes().size());
+ }
+
+ /** */
+ private void createCacheAndSnapshot(int gridIdx) {
+ int partsCnt = 4;
+ int keysCnt = partsCnt * 5;
+
+ grid(gridIdx).createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+ .setCacheMode(CacheMode.REPLICATED)
+ .setAffinity(new RendezvousAffinityFunction().setPartitions(partsCnt))
+ .setAtomicityMode(CacheAtomicityMode.ATOMIC));
+
+ try (IgniteDataStreamer<Integer, Integer> ds = grid(gridIdx).dataStreamer(DEFAULT_CACHE_NAME)) {
+ for (int i = 0; i < keysCnt; i++)
+ ds.addData(i, i);
+ }
+
+ createSnapshot(gridIdx);
+ }
+
+ /** */
+ private void createSnapshot(int gridIdx) {
+ snp(gridIdx).createSnapshot(SNP_NAME).get(getTestTimeout());
+ }
+
+ /** */
+ private void ensureSnapshotDeletionFailed(boolean useTask) {
+ String err = "The snapshot deletion feature isn't activated yet";
+
+ for (int i = 0; i < ALL_GRIDS; i++) {
+ int i0 = i;
+
+ if (useTask) {
+ SnapshotDeleteCommandArg args = new SnapshotDeleteCommandArg();
+
+ args.snapshotName(SNP_NAME);
+
+ IgniteEx ig = grid(i0);
+
+ assertThrowsAnyCause(
+ null,
+ () -> ig.compute().execute(new SnapshotDeleteTask(), new VisorTaskArgument<>(ig.localNode().id(), args, false)),
+ IgniteException.class,
+ err
+ );
+ }
+ else {
+ assertThrowsAnyCause(
+ null,
+ () -> snp(i0).deleteSnapshot(SNP_NAME, null).get(),
+ IgniteIllegalStateException.class,
+ err
+ );
+ }
+ }
+ }
+
+ /** */
+ private IgniteSnapshotManager snp(int gridIdx) {
+ return grid(gridIdx).context().cache().context().snapshotMgr();
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteTest.java
new file mode 100644
index 00000000000..ab568f3c2d9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteTest.java
@@ -0,0 +1,931 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteIllegalStateException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.filename.SnapshotFileTree;
+import org.apache.ignite.internal.util.distributed.DistributedProcess;
+import org.apache.ignite.internal.util.distributed.SingleNodeMessage;
+import org.apache.ignite.internal.util.future.IgniteFutureImpl;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.plugin.AbstractTestPluginProvider;
+import org.apache.ignite.plugin.PluginContext;
+import org.jetbrains.annotations.Nullable;
+import org.junit.Test;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_METAS;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.DELETE_SNAPSHOT;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.END_SNAPSHOT;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_START;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_INCREMENTAL_SNAPSHOT_START;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.START_SNAPSHOT;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
+import static org.junit.Assume.assumeFalse;
+import static org.junit.Assume.assumeTrue;
+
+/** */
+public class IgniteClusterSnapshotDeleteTest extends AbstractSnapshotSelfTest {
+ /** */
+ private static final int CACHE_KEYS_RANGE = 10;
+
+ /** */
+ private static final int INC_CACHE_KEYS_RANGE = 15;
+
+ /** Extra storage path. */
+ private static final String EXT_STORAGE_PATH = "extStorage";
+
+ /** */
+ private static boolean posixPermissions;
+
+ /** */
+ private boolean separatedWorkDir;
+
+ /** */
+ private boolean extraStorages;
+
+ /** */
+ @Parameter(2)
+ public boolean incremental;
+
+ /** */
+ private @Nullable String cstIdSuffix;
+
+ /** */
+ private @Nullable String[] extStoragePaths;
+
+ /** Parameters. */
+ @Parameterized.Parameters(name = "encryption={0}, onlyPrimary={1}, incremental={2}")
+ public static Collection<?> runParams() {
+ /** Use {@link #incremental} only. */
+ return F.asList(
+ new Object[] {false, false, false},
+ new Object[] {false, false, true}
+ );
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ var cfg = super.getConfiguration(igniteInstanceName);
+
+ String workDir = separatedWorkDir
+ ? new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath()
+ : U.defaultWorkDirectory();
+
+ cfg.setWorkDirectory(workDir);
+
+ if (cstIdSuffix != null)
+ cfg.setConsistentId(cfg.getConsistentId().toString() + '_' + cstIdSuffix);
+
+ if (extraStorages) {
+ cfg.getDataStorageConfiguration().setExtraStoragePaths(
+ workDir + File.separator,
+ workDir + File.separator + EXT_STORAGE_PATH
+ );
+
+ extStoragePaths = cfg.getDataStorageConfiguration().getExtraStoragePaths();
+
+ cfg.getDataStorageConfiguration().setExtraSnapshotPaths("", EXT_STORAGE_PATH);
+ }
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void afterTestSnapshot() throws Exception {
+ super.afterTestSnapshot();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void beforeTestSnapshot() throws Exception {
+ super.beforeTestSnapshot();
+
+ /** Handy if test running is interrupted and {@link #afterTestSnapshot()} isn't invoked. */
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ File workDir = new File(U.defaultWorkDirectory());
+
+ assertTrue(workDir.exists());
+
+ Path workPath = workDir.toPath();
+
+ try {
+ Files.getPosixFilePermissions(workPath);
+
+ posixPermissions = true;
+ }
+ catch (UnsupportedOperationException ignored) {
+ // No-op.
+ }
+ }
+
+ /** */
+ @Test
+ public void testDeniedPermissions() throws Exception {
+ assumeTrue(posixPermissions);
+
+ // Doesn't matter here.
+ assumeFalse(incremental);
+
+ separatedWorkDir = true;
+
+ AtomicReference<Set<PosixFilePermission>> prevPerms = new AtomicReference<>();
+ AtomicReference<Path> pathRef = new AtomicReference<>();
+
+ pluginProvider = new AbstractTestPluginProvider() {
+ @Override public String name() {
+ return "TestSnpMgrProvider";
+ }
+
+ @Override public <T> T createComponent(PluginContext ctx, Class<T> cls) {
+ if (IgniteSnapshotManager.class.isAssignableFrom(cls)) {
+ return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) {
+ @Override public T2<Boolean, Boolean> deleteLocalSnapshot(
+ SnapshotFileTree sft,
+ boolean scoped,
+ boolean ignoreErrs
+ ) {
+ if (ctx.localNode().id().equals(grid(1).localNode().id())) {
+ Path path = sft.root().toPath();
+
+ pathRef.set(path);
+
+ try {
+ prevPerms.set(Files.getPosixFilePermissions(path));
+
+ // Denies writing (deletion).
+ Files.setPosixFilePermissions(path, PosixFilePermissions.fromString("r-xr-x---"));
+ }
+ catch (Exception e) {
+ throw new IgniteException("Unable to set the posix permissions.", e);
+ }
+ }
+
+ return super.deleteLocalSnapshot(sft, scoped, ignoreErrs);
+ }
+ };
+ }
+
+ return super.createComponent(ctx, cls);
+ }
+ };
+
+ startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg);
+
+ snp(grid(0)).createSnapshot(SNAPSHOT_NAME).get(getTestTimeout());
+
+ try {
+ SnapshotDeleteProcessResult res = snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout());
+
+ assertEquals(1, res.uncompletedNodes().size());
+ assertEquals(2, res.completedNodes().size());
+ }
+ finally {
+ if (pathRef.get() != null && prevPerms.get() != null)
+ Files.setPosixFilePermissions(pathRef.get(), prevPerms.get());
+ }
+ }
+
+ /** */
+ @Test
+ public void testExtraStoragesDeleted() throws Exception {
+ extraStorages = true;
+
+ dfltCacheCfg = null;
+
+ startGridsMultiThreaded(3);
+
+ assertFalse(F.isEmpty(extStoragePaths));
+
+ assertTrue(grid(0).cache(DEFAULT_CACHE_NAME) == null);
+
+ dfltCacheCfg = defaultCacheConfiguration();
+
+ // Works only with a shared work directory.
+ dfltCacheCfg.setStoragePaths(extStoragePaths);
+
+ grid(0).createCache(dfltCacheCfg);
+ awaitPartitionMapExchange();
+
+ // Fills the cache.
+ startGridsWithCache(0, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg);
+
+ snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout());
+
+ if (incremental)
+ addIncrementalSnapshot(null);
+
+ for (Ignite ig : G.allGrids()) {
+ SnapshotFileTree snpTree = new SnapshotFileTree(((IgniteEx)ig).context(), SNAPSHOT_NAME, null);
+
+ assertEquals(2, snpTree.extraStorages().size());
+
+ for (File extSnpStorage : snpTree.allStorages().toList()) {
+ assertTrue(extSnpStorage.exists());
+ assertTrue(extSnpStorage.isDirectory());
+ }
+ }
+
+ snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout());
+
+ for (Ignite ig : G.allGrids()) {
+ SnapshotFileTree snpTree = new SnapshotFileTree(((IgniteEx)ig).context(), SNAPSHOT_NAME, null);
+
+ for (File extSnpStorage : snpTree.allStorages().toList()) {
+ // Snapshot root, even in an external storage.
+ extSnpStorage = extSnpStorage.getParentFile().getParentFile();
+
+ assertEquals(SNAPSHOT_NAME, extSnpStorage.getName());
+
+ assertFalse(extSnpStorage.exists());
+ }
+ }
+ }
+
+ /** Tests snapshot deletion when one node finds snapshot but fails to delete its data. */
+ @Test
+ public void testUncompletedNodes() throws Exception {
+ separatedWorkDir = true;
+
+ // Simulates a deletion error on some node.
+ pluginProvider = new AbstractTestPluginProvider() {
+ @Override public String name() {
+ return "TestSnpMgrProvider";
+ }
+
+ // Simulates the deletion failure.
+ @Override public <T> T createComponent(PluginContext ctx, Class<T> cls) {
+ if (IgniteSnapshotManager.class.isAssignableFrom(cls)) {
+ return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) {
+ @Override public T2<Boolean, Boolean> deleteLocalSnapshot(
+ SnapshotFileTree sft,
+ boolean scoped,
+ boolean ignoreErrs
+ ) {
+ if (ctx.localNode().id().equals(grid(1).localNode().id()))
+ return new T2<>(false, true);
+
+ return super.deleteLocalSnapshot(sft, scoped, ignoreErrs);
+ }
+ };
+ }
+
+ return super.createComponent(ctx, cls);
+ }
+ };
+
+ startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg);
+
+ snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout());
+
+ if (incremental)
+ addIncrementalSnapshot(null);
+
+ SnapshotDeleteProcessResult delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout());
+
+ assertTrue(delSnpRes.emptyNodes().isEmpty());
+ assertEquals(1, delSnpRes.uncompletedNodes().size());
+ assertTrue(delSnpRes.uncompletedNodes().containsKey(grid(1).localNode().id()));
+ }
+
+ /** Tests the snapshot deletion when cluster restarts with changed nodes consistent ids. */
+ @Test
+ public void testDeleteOtherConsistentId() throws Exception {
+ startGridsWithSnapshot(3, CACHE_KEYS_RANGE, false, true);
+
+ stopAllGrids();
+
+ cstIdSuffix = "_ext";
+
+ startGridsMultiThreaded(3);
+
+ SnapshotDeleteProcessResult delRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout());
+
+ for (var ig : G.allGrids()) {
+ assertTrue(Files.list(((IgniteEx)ig).context().pdsFolderResolver().fileTree().snapshotsRoot().toPath())
+ .findFirst().isEmpty());
+ }
+
+ assertEquals(3, delRes.absentBaselines().size());
+ }
+
+ /** Tests that snapshot is not deleted if snaphot metadata is not found. Shared working directory. */
+ @Test
+ public void testDeleteSnapshotNoMetaSharedDirectory() throws Exception {
+ doTestDeleteNotSnapshot(false, false);
+ }
+
+ /** Tests that snapshot is not deleted if snaphot metadata is not found. Dedicated working directories. */
+ @Test
+ public void testDeleteSnapshotNoMetaDedicatedDirectories() throws Exception {
+ doTestDeleteNotSnapshot(true, false);
+ }
+
+ /** Tests that snapshot is not deleted if snaphot metadata cannot be read (corrupted). Shared working directory. */
+ @Test
+ public void testDeleteSnapshotCorruptedMetaSharedDirectory() throws Exception {
+ doTestDeleteNotSnapshot(false, true);
+ }
+
+ /** Tests that snapshot is not deleted if snaphot metadata cannot be read (corrupted). Dedicated working directories. */
+ @Test
+ public void testDeleteSnapshotCorruptedMetaDedicatedDirectories() throws Exception {
+ doTestDeleteNotSnapshot(true, true);
+ }
+
+ /**
+ * Tests that snapshot is not deleted if snaphot metadata isn't found or cannot be read (corrupted).
+ *
+ * @param separatedWorkDir If {@code true}, the dedicated dirictories is used for the nodes. If {@code false},
+ * the nodes use a shared working directory.
+ * @param corruptSnpMeta If {@code true}, corrupts the snapshot metadata file. If {@code false}, removes the snapshot
+ * metadata file
+ */
+ protected void doTestDeleteNotSnapshot(boolean separatedWorkDir, boolean corruptSnpMeta) throws Exception {
+ // Doesn't matter here;
+ assumeFalse(onlyPrimary);
+
+ this.separatedWorkDir = separatedWorkDir;
+
+ startGridsWithCache(3, CACHE_KEYS_RANGE, valueBuilder(), dfltCacheCfg);
+
+ snp(grid(1)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout());
+
+ var snpSft = new SnapshotFileTree(grid(1).context(), SNAPSHOT_NAME, null);
+
+ // Ensure that all the snapshot node folders exist.
+ assertTrue(snpSft.binaryMeta().exists());
+ assertTrue(new SnapshotFileTree(grid(0).context(), SNAPSHOT_NAME, null, folderName(0), consistentId(0))
+ .binaryMeta().exists());
+ assertTrue(new SnapshotFileTree(grid(2).context(), SNAPSHOT_NAME, null, folderName(2), consistentId(2))
+ .binaryMeta().exists());
+
+ assertTrue(snpSft.meta().exists());
+
+ if (corruptSnpMeta) {
+ try (var rwf = new RandomAccessFile(snpSft.meta(), "rw")) {
+ byte[] slop = new byte[128];
+
+ new Random().nextBytes(slop);
+
+ rwf.write(slop);
+ }
+ }
+ else {
+ assertTrue(U.delete(snpSft.meta()));
+ assertFalse(snpSft.meta().exists());
+ }
+
+ var delSnpRes = snp(grid(2)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout());
+
+ // Check the result.
+ if (separatedWorkDir) {
+ // One node doesn't find meta, decided not a snapshot.
+ assertTrue(delSnpRes.uncompletedNodes().isEmpty());
+ assertEquals(2, delSnpRes.completedNodes().size());
+ assertEquals(1, delSnpRes.emptyNodes().size());
+ assertTrue(delSnpRes.emptyNodes().containsKey(grid(1).localNode().id()));
+ assertTrue(snpSft.binaryMeta().exists());
+ }
+ else
+ assertEquals(3, delSnpRes.uncompletedNodes().size() + delSnpRes.completedNodes().size() + delSnpRes.emptyNodes().size());
+
+ assertFalse(new SnapshotFileTree(grid(0).context(), SNAPSHOT_NAME, null, folderName(0), consistentId(0))
+ .binaryMeta().exists());
+ assertFalse(new SnapshotFileTree(grid(2).context(), SNAPSHOT_NAME, null, folderName(2), consistentId(2))
+ .binaryMeta().exists());
+ }
+
+ /** */
+ private String consistentId(int gridIdx) {
+ return grid(gridIdx).configuration().getConsistentId().toString();
+ }
+
+ /** */
+ private String folderName(int gridIdx) {
+ return grid(gridIdx).context().pdsFolderResolver().fileTree().folderName();
+ }
+
+ /** Tests snapshot deletion when one node has no snapshot data. */
+ @Test
+ public void testEmptyNodes() throws Exception {
+ separatedWorkDir = true;
+
+ startGridsWithCache(2, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg);
+
+ snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout());
+
+ if (incremental)
+ addIncrementalSnapshot(null);
+
+ IgniteEx additionalNode = startGrid(G.allGrids().size());
+
+ SnapshotDeleteProcessResult delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout());
+
+ assertFalse(delSnpRes.emptyNodes().isEmpty());
+ assertTrue(delSnpRes.emptyNodes().containsKey(additionalNode.localNode().id()));
+ assertTrue(delSnpRes.uncompletedNodes().isEmpty());
+ }
+
+ /** Tests snapshot deletion repeat after an offline node restarts. */
+ @Test
+ public void testDeletionRepeatAfterOfflineNodeStarts() throws Exception {
+ separatedWorkDir = true;
+
+ startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg);
+
+ snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout());
+
+ if (incremental)
+ addIncrementalSnapshot(null);
+
+ int stoppedNodeIdx = G.allGrids().size() - 1;
+
+ UUID stoppedNodeId = grid(stoppedNodeIdx).localNode().id();
+
+ stopGrid(stoppedNodeIdx);
+
+ SnapshotDeleteProcessResult delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout());
+
+ assertEquals(2, delSnpRes.completedNodes().size());
+ assertFalse(delSnpRes.completedNodes().containsKey(stoppedNodeId));
+
+ assertTrue(delSnpRes.uncompletedNodes().isEmpty());
+ assertTrue(delSnpRes.emptyNodes().isEmpty());
+
+ startGrid(stoppedNodeIdx);
+
+ stoppedNodeId = grid(stoppedNodeIdx).localNode().id();
+
+ delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout());
+
+ assertEquals(1, delSnpRes.completedNodes().size());
+ assertTrue(delSnpRes.completedNodes().containsKey(stoppedNodeId));
+
+ assertTrue(delSnpRes.uncompletedNodes().isEmpty());
+ assertEquals(2, delSnpRes.emptyNodes().size());
+ }
+
+ /** Test snapshot deletion process when one node leaves. */
+ @Test
+ public void testNodeStopsInTheMiddle() throws Exception {
+ separatedWorkDir = true;
+
+ CountDownLatch beginLatch = new CountDownLatch(1);
+ CountDownLatch proceedLatch = new CountDownLatch(1);
+
+ // Simulates a deletion error on some node.
+ pluginProvider = new AbstractTestPluginProvider() {
+ @Override public String name() {
+ return "TestSnpMgrProvider";
+ }
+
+ @Override public <T> T createComponent(PluginContext ctx, Class<T> cls) {
+ if (IgniteSnapshotManager.class.isAssignableFrom(cls)) {
+ return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) {
+ @Override public T2<Boolean, Boolean> deleteLocalSnapshot(
+ SnapshotFileTree sft,
+ boolean scoped,
+ boolean ignoreErrs
+ ) {
+ if (ctx.localNode().id().equals(grid(1).localNode().id())) {
+ beginLatch.countDown();
+
+ try {
+ assertTrue(proceedLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS));
+ }
+ catch (InterruptedException e) {
+ throw new RuntimeException("Interrupted.", e);
+ }
+ }
+
+ return super.deleteLocalSnapshot(sft, scoped, ignoreErrs);
+ }
+ };
+ }
+
+ return super.createComponent(ctx, cls);
+ }
+ };
+
+ startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg);
+
+ snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout());
+
+ if (incremental)
+ addIncrementalSnapshot(null);
+
+ IgniteInternalFuture<SnapshotDeleteProcessResult> delFut = snp(grid(2)).deleteSnapshot(SNAPSHOT_NAME, null);
+
+ assertTrue(beginLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS));
+
+ UUID stoppedGridId = grid(1).localNode().id();
+
+ stopGrid(1);
+
+ proceedLatch.countDown();
+
+ var delRes = delFut.get(getTestTimeout());
+
+ assertEquals(2, delRes.completedNodes().size());
+ assertFalse(delRes.completedNodes().containsKey(stoppedGridId));
+
+ startGrid(1);
+
+ delRes = snp(grid(2)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout());
+
+ assertEquals(1, delRes.completedNodes().size());
+ assertTrue(delRes.completedNodes().containsKey(grid(1).localNode().id()));
+ }
+
+ /** Tests that a concurrent deletion of a snapshot with the same name but different name characters case is allowed. */
+ @Test
+ public void testConcurrentDeleteOfTheSameSnapshotDifferentNameCharactersCase() throws Exception {
+ assumeFalse(caseInsensitiveFs);
+
+ doTestConcurrentDeleteOfTheSameSnapshotDifferentPath(true);
+ }
+
+ /** Tests that a concurrent deletion of a snapshot with the same name but different path is allowed. */
+ @Test
+ public void testConcurrentDeleteOfTheSameSnapshotDifferentPath() throws Exception {
+ doTestConcurrentDeleteOfTheSameSnapshotDifferentPath(false);
+ }
+
+ /**
+ * Tests that a concurrent deletion of a snapshot with the same name but different path is allowed.
+ *
+ * @param useNameCharactersCase If {@code true}, uses a changed characters case of snapshot name as the other snapshot path.
+ * If {@code false}, uses different snapshot root path.
+ */
+ private void doTestConcurrentDeleteOfTheSameSnapshotDifferentPath(boolean useNameCharactersCase) throws Exception {
+ startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg);
+
+ snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(TIMEOUT);
+
+ if (incremental)
+ addIncrementalSnapshot(SNAPSHOT_NAME, null);
+
+ String secondSnpPath = null;
+ String secondSnpName = SNAPSHOT_NAME.toLowerCase();
+
+ if (!useNameCharactersCase) {
+ secondSnpPath = new File(grid(0).context().pdsFolderResolver().fileTree().snapshotsRoot(), "ex_snapshots").getAbsolutePath();
+ secondSnpName = SNAPSHOT_NAME;
+ }
+
+ snp(grid(0)).createSnapshot(secondSnpName, secondSnpPath, false, onlyPrimary).get(getTestTimeout());
+
+ if (incremental)
+ addIncrementalSnapshot(secondSnpName, secondSnpPath);
+
+ TestRecordingCommunicationSpi commSpi1 = (TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi();
+
+ commSpi1.blockMessages((node, msg) ->
+ msg instanceof SingleNodeMessage<?> msg0 && msg0.type() == DELETE_SNAPSHOT.ordinal());
+
+ IgniteInternalFuture<SnapshotDeleteProcessResult> delFut0 = snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null);
+ IgniteInternalFuture<SnapshotDeleteProcessResult> delFut1 = snp(grid(1)).deleteSnapshot(secondSnpName, secondSnpPath);
+
+ commSpi1.waitForBlocked(2, getTestTimeout());
+
+ commSpi1.stopBlock();
+
+ var delRes0 = delFut0.get(getTestTimeout());
+ var delRes1 = delFut1.get(getTestTimeout());
+
+ assertTrue(!delRes0.completedNodes().isEmpty() || !delRes0.uncompletedNodes().isEmpty());
+ assertTrue(!delRes1.completedNodes().isEmpty() || !delRes1.uncompletedNodes().isEmpty());
+ }
+
+ /** Tests that a concurrent deletion of the same snapshot is declined. */
+ @Test
+ public void testConcurrentDeleteOfTheSameSnapshot() throws Exception {
+ // Doesn't matter here.
+ assumeFalse(onlyPrimary || incremental);
+
+ String snapshotName = caseInsensitiveFs ? SNAPSHOT_NAME.toLowerCase() : SNAPSHOT_NAME;
+
+ doConcurrentSnapshotOperationWhenDeletionIsActive(
+ () -> startGridsWithSnapshot(3, CACHE_KEYS_RANGE, false, true),
+ () -> snp(grid(1)).deleteSnapshot(snapshotName, null).get(getTestTimeout()),
+ e -> e.getMessage().contains("Deletion of the snapshot has already started"),
+ false
+ );
+ }
+
+ /**
+ * Tests that a snapshot deletion is declined when a snapshot check operation is in progress.
+ *
+ * @see SnapshotCheckProcess#isSnapshotChecking(String)
+ */
+ @Test
+ public void testSnapshotDeleteWhenCheckInProgress() throws Exception {
+ String firstOpSnpName = caseInsensitiveFs ? SNAPSHOT_NAME.toLowerCase() : SNAPSHOT_NAME;
+
+ SnapshotPartitionsVerifyResult res = doTestConcurrentSnapshotDelete(
+ () -> new IgniteFutureImpl<>(snp(grid(2)).checkSnapshot(firstOpSnpName, null, incremental ? 1 : 0)),
+ F.asList(CHECK_SNAPSHOT_METAS),
+ true,
+ null,
+ SnapshotDeleteProcess.BEING_CREATED_PREF,
+ false,
+ true
+ );
+
+ assertFalse(res == null);
+ assertTrue(F.isEmpty(res.exceptions()));
+ }
+
+ /** Tests that a snapshot deletion is declined when a snapshot create operation is in progress.*/
+ @Test
+ public void testSnapshotDeleteWhenCreateInProgress() throws Exception {
+ String firstOpSnpName = caseInsensitiveFs ? SNAPSHOT_NAME.toLowerCase() : SNAPSHOT_NAME;
+
+ doTestConcurrentSnapshotDelete(
+ () -> snp(grid(0)).createSnapshot(firstOpSnpName, null, incremental, onlyPrimary),
+ F.asList(START_SNAPSHOT, END_SNAPSHOT),
+ false,
+ () -> {
+ new IgniteFutureImpl<>(snp(grid(0)).deleteSnapshot(firstOpSnpName, null)).get(getTestTimeout());
+
+ if (incremental)
+ snp(grid(0)).createSnapshot(firstOpSnpName).get(getTestTimeout());
+ },
+ "Snapshot with the same name is being created",
+ false,
+ false
+ );
+ }
+
+ /** Tests that a snapshot deletion is declined when a snapshot restore begins. */
+ @Test
+ public void testSnapshotDeleteWhenRestoreBegins() throws Exception {
+ String firstOpSnpName = caseInsensitiveFs ? SNAPSHOT_NAME.toLowerCase() : SNAPSHOT_NAME;
+
+ doTestConcurrentSnapshotDelete(
+ () -> {
+ if (incremental)
+ return snp(grid(2)).restoreSnapshot(firstOpSnpName, null, 1);
+ else
+ return snp(grid(2)).restoreSnapshot(firstOpSnpName, null);
+ },
+ F.asList(CHECK_SNAPSHOT_METAS, CHECK_SNAPSHOT_PARTS),
+ true,
+ () -> {
+ grid(0).destroyCache(DEFAULT_CACHE_NAME);
+
+ awaitPartitionMapExchange();
+ },
+ SnapshotDeleteProcess.BEING_CHECKED_PREF,
+ false,
+ false
+ );
+ }
+
+ /** Tests that a snapshot deletion is declined when a snapshot restore is in progress. */
+ @Test
+ public void testSnapshotDeleteWhenRestoreInProgress() throws Exception {
+ String firstOpSnpName = caseInsensitiveFs ? SNAPSHOT_NAME.toLowerCase() : SNAPSHOT_NAME;
+
+ var restoreMsgs = F.asList(
+ RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE,
+ RESTORE_CACHE_GROUP_SNAPSHOT_START
+ );
+
+ if (incremental) {
+ restoreMsgs = new ArrayList<>(restoreMsgs);
+ restoreMsgs.add(RESTORE_INCREMENTAL_SNAPSHOT_START);
+ }
+
+ doTestConcurrentSnapshotDelete(
+ () -> {
+ if (incremental)
+ return snp(grid(2)).restoreSnapshot(firstOpSnpName, null, 1);
+ else
+ return snp(grid(2)).restoreSnapshot(firstOpSnpName, null);
+ },
+ restoreMsgs,
+ true,
+ () -> {
+ grid(0).destroyCache(DEFAULT_CACHE_NAME);
+
+ awaitPartitionMapExchange();
+ },
+ SnapshotDeleteProcess.BEING_RESTORED_PREF,
+ false,
+ false
+ );
+ }
+
+ /** Tests that a snapshot deletion is declined when a snapshot restore is in progress but fails. */
+ @Test
+ public void testSnapshotDeleteWhenRestoreProgressFails() throws Exception {
+ // An in-the-middle failure won't allow to start restoring the incrementals.
+ assumeFalse(incremental);
+
+ var restoreMsgs = F.asList(RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK);
+
+ if (incremental) {
+ restoreMsgs = new ArrayList<>(restoreMsgs);
+ restoreMsgs.add(RESTORE_INCREMENTAL_SNAPSHOT_START);
+ }
+
+ doTestConcurrentSnapshotDelete(
+ () -> {
+ if (incremental)
+ return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null, 1);
+ else
+ return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null);
+ },
+ restoreMsgs,
+ true,
+ () -> {
+ grid(0).destroyCache(DEFAULT_CACHE_NAME);
+
+ awaitPartitionMapExchange();
+
+ SnapshotFileTree sft = snapshotFileTree(grid(1), SNAPSHOT_NAME);
+
+ String failingFilePath = sft.partitionFile(dfltCacheCfg, primaries[0]).getAbsolutePath()
+ .replace(sft.nodeStorage().getAbsolutePath(), "");
+
+ grid(1).context().cache().context().snapshotMgr().ioFactory((file, modes) -> {
+ FileIO delegate = new RandomAccessFileIOFactory().create(file, modes);
+
+ if (file.getPath().endsWith(failingFilePath))
+ throw new RuntimeException("Test exception");
+
+ return delegate;
+ });
+ },
+ SnapshotDeleteProcess.BEING_RESTORED_PREF,
+ true,
+ false
+ );
+ }
+
+ /**
+ * @param firstOp First cluster-wide snapshot operation.
+ * @param msgsToWatch {@link SingleNodeMessage#type()} relating to {@code firstOp} to block on one node.
+ * @param precreateSnp If {@code true}, creates snapshot after the cluster start.
+ * @param prepareIteration If not {@code null}, is invoked in the beginning of test iteration at each {@code msgsToWatch}.
+ * @param concurrentMsgErr Test of failed concurrent to {@code firstOp} delete snapshot operation to watch.
+ * @param ignoreFirstOpFailure If {@code true}, possible failure of {@code firstOp} is ignored.
+ * @param ignoreDeleteOpFailure If {@code true}, possible failure of the deletion operation is ignored. Allows to
+ * focus on the first operation's result.
+ */
+ protected <T> @Nullable T doTestConcurrentSnapshotDelete(
+ Supplier<IgniteFuture<T>> firstOp,
+ Collection<DistributedProcess.DistributedProcessType> msgsToWatch,
+ boolean precreateSnp,
+ @Nullable Runnable prepareIteration,
+ String concurrentMsgErr,
+ boolean ignoreFirstOpFailure,
+ boolean ignoreDeleteOpFailure
+ ) throws Exception {
+ startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg);
+
+ if (precreateSnp) {
+ snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary)
+ .get(TIMEOUT);
+
+ if (incremental)
+ addIncrementalSnapshot(null);
+ }
+
+ TestRecordingCommunicationSpi commSpi1 = (TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi();
+
+ T res = null;
+
+ for (var nodeResMsgType : msgsToWatch) {
+ if (log.isInfoEnabled())
+ log.info("Iteration with message-to-wait-for type: " + nodeResMsgType);
+
+ if (prepareIteration != null)
+ prepareIteration.run();
+
+ commSpi1.blockMessages((node, msg) ->
+ msg instanceof SingleNodeMessage<?> msg0 && msg0.type() == nodeResMsgType.ordinal());
+
+ IgniteFuture<T> firstFut = firstOp.get();
+
+ commSpi1.waitForBlocked(1, getTestTimeout());
+
+ if (ignoreDeleteOpFailure) {
+ try {
+ snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout());
+ }
+ catch (Exception e) {
+ if (log.isDebugEnabled())
+ log.debug("The deletion operation failed but a failure is expected, err: " + e.getMessage());
+ }
+ }
+ else {
+ assertThrowsAnyCause(
+ null,
+ () -> snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()),
+ IgniteIllegalStateException.class,
+ concurrentMsgErr
+ );
+ }
+
+ commSpi1.stopBlock();
+
+ if (ignoreFirstOpFailure) {
+ try {
+ res = firstFut.get(getTestTimeout());
+ }
+ catch (Exception e) {
+ if (log.isDebugEnabled())
+ log.debug("The first operation failed but a failure is expected, err: " + e.getMessage());
+
+ return null;
+ }
+ }
+ else
+ res = firstFut.get(getTestTimeout());
+ }
+
+ return res;
+ }
+
+ /** */
+ private void addIncrementalSnapshot(String name, @Nullable String path) {
+ try (var ds = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) {
+ for (int i = CACHE_KEYS_RANGE; i < INC_CACHE_KEYS_RANGE; i++)
+ ds.addData(i, i);
+ }
+
+ snp(grid(0)).createSnapshot(name, path, true, onlyPrimary)
+ .get(getTestTimeout());
+ }
+
+ /** */
+ private void addIncrementalSnapshot(@Nullable String path) {
+ addIncrementalSnapshot(SNAPSHOT_NAME, path);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void awaitPartitionMapExchange() {
+ try {
+ super.awaitPartitionMapExchange();
+ }
+ catch (InterruptedException e) {
+ throw new RuntimeException("Interrupted.", e);
+ }
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java
index 78ec729ec93..09c67e7ece7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java
@@ -37,6 +37,7 @@ import com.google.common.base.Throwables;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteSnapshot;
import org.apache.ignite.binary.BinaryObject;
@@ -333,6 +334,47 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR
assertCacheKeys(ignite.cache(DEFAULT_CACHE_NAME), CACHE_KEYS_RANGE);
}
+ /** Tests that snapshot restore is declined when the same snapshot is being deleted. */
+ @Test
+ public void testConcurrentSnapshotDeleteAndRestoreOperations() throws Exception {
+ String snapshotName = caseInsensitiveFs ? SNAPSHOT_NAME.toLowerCase() : SNAPSHOT_NAME;
+
+ doConcurrentSnapshotOperationWhenDeletionIsActive(
+ () -> startGridsWithSnapshot(3, CACHE_KEYS_RANGE),
+ () -> snp(grid(2)).restoreSnapshot(snapshotName, null).get(),
+ e -> e.getMessage().contains("Delete operation of snapshot '%s' is in progress".formatted(snapshotName)),
+ true
+ );
+ }
+
+ /** */
+ @Test
+ public void testConcurrentSnapshotDeleteAndRestoreOperationsWithDifferentPath() throws Exception {
+ String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath();
+
+ doConcurrentSnapshotOperationWhenDeletionIsActive(
+ () -> {
+ startGridsWithSnapshot(3, CACHE_KEYS_RANGE);
+
+ grid(0).createCache(DEFAULT_CACHE_NAME);
+
+ try (IgniteDataStreamer<Integer, Integer> ds = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) {
+ for (int i = 0; i < CACHE_KEYS_RANGE; ++i)
+ ds.addData(i, i);
+ }
+
+ snp(grid(0)).createSnapshot(SNAPSHOT_NAME, snpPath, false, false).get(getTestTimeout());
+
+ grid(0).destroyCache(DEFAULT_CACHE_NAME);
+
+ awaitPartitionMapExchange();
+ },
+ () -> snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, snpPath, null).get(),
+ null,
+ false
+ );
+ }
+
/**
* Ensures that the cache doesn't start if one of the baseline nodes fails.
*
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java
index 570761dac22..e5b34271f26 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java
@@ -603,7 +603,7 @@ public class IgniteClusterSnapshotSelfTest extends AbstractSnapshotSelfTest {
assertThrowsAnyCause(log,
() -> snp(ignite).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(),
IgniteException.class,
- "Snapshot with given name already exists on local node.");
+ "Snapshot with given name already exists on local node");
stopAllGrids();
@@ -615,6 +615,44 @@ public class IgniteClusterSnapshotSelfTest extends AbstractSnapshotSelfTest {
waitForEvents(EVT_CLUSTER_SNAPSHOT_STARTED, EVT_CLUSTER_SNAPSHOT_FAILED);
}
+ /**
+ * Tests that snapshot delete operation detects concurrent deletion of the same sanpshot.
+ */
+ @Test
+ public void testConcurrentSnapshotDeleteOperation() throws Exception {
+ String snapshotName = caseInsensitiveFs ? SNAPSHOT_NAME.toLowerCase() : SNAPSHOT_NAME;
+
+ doConcurrentSnapshotOperationWhenDeletionIsActive(
+ () -> {
+ startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE);
+
+ snp(grid(2)).createSnapshot(snapshotName).get();
+ },
+ () -> snp(grid(2)).deleteSnapshot(snapshotName, null).get(),
+ e -> e.getMessage().contains("Deletion of the snapshot has already started"),
+ false
+ );
+ }
+
+ /**
+ * Tests that a concurrent deletion of a same-named snapshot is allowed if it has a different path.
+ */
+ @Test
+ public void testConcurrentSnapshotDeleteOperationWithDifferentPath() throws Exception {
+ String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath();
+
+ doConcurrentSnapshotOperationWhenDeletionIsActive(
+ () -> {
+ startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE);
+
+ snp(grid(2)).createSnapshot(SNAPSHOT_NAME).get();
+ },
+ () -> snp(grid(2)).createSnapshot(SNAPSHOT_NAME, snpPath, false, false).get(),
+ null,
+ false
+ );
+ }
+
/** @throws Exception If fails. */
@Test
public void testClusterSnapshotCleanedOnLeft() throws Exception {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRestoreFromRemoteTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRestoreFromRemoteTest.java
index a31cd9a7137..d37594f64f7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRestoreFromRemoteTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRestoreFromRemoteTest.java
@@ -175,6 +175,11 @@ public class IgniteSnapshotRestoreFromRemoteTest extends IgniteClusterSnapshotRe
cleanupDedicatedPersistenceDirs(FIRST_CLUSTER_PREFIX);
}
+ /** */
+ @Override protected boolean cleanWorkDirectoryFully() {
+ return false;
+ }
+
/** @throws Exception If failed. */
@Test
public void testRestoreWithPairedConnections() throws Exception {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotTest.java
index fcf65fa9b30..8055e7795e3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotTest.java
@@ -132,7 +132,7 @@ public abstract class AbstractIncrementalSnapshotTest extends GridCommonAbstract
try {
assertTrue(GridTestUtils.waitForCondition(() -> {
for (Ignite g: G.allGrids()) {
- if (snp((IgniteEx)g).currentCreateRequest() != null)
+ if (snp(g).currentCreateRequest() != null)
return false;
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/IncrementalSnapshotNodeFailureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/IncrementalSnapshotNodeFailureTest.java
index 88aad62a498..68ff0e44374 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/IncrementalSnapshotNodeFailureTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/IncrementalSnapshotNodeFailureTest.java
@@ -109,7 +109,7 @@ public class IncrementalSnapshotNodeFailureTest extends AbstractIncrementalSnaps
awaitSnapshotResourcesCleaned();
for (Ignite g: G.allGrids())
- assertNull(snp((IgniteEx)g).incrementalSnapshotId());
+ assertNull(snp(g).incrementalSnapshotId());
stopAllGrids();
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java
new file mode 100644
index 00000000000..b584171fc03
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteRollingUpgradeTest;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteTest;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.DynamicSuite;
+import org.junit.runner.RunWith;
+
+/** Split off from {@link IgniteSnapshotTestSuite8} to reduce the single-suite runtime in CI. */
+@RunWith(DynamicSuite.class)
+public class IgniteSnapshotTestSuite9 {
+ /** @return Test suite. */
+ public static List<Class<?>> suite() {
+ return suite(null);
+ }
+
+ /**
+ * @param ignoredTests Tests to ignore.
+ * @return Test suite.
+ */
+ public static List<Class<?>> suite(Collection<Class> ignoredTests) {
+ List<Class<?>> suite = new ArrayList<>();
+
+ GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotDeleteRollingUpgradeTest.class, ignoredTests);
+ GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotDeleteTest.class, ignoredTests);
+
+ return suite;
+ }
+}
diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output
index dd6ff4e43ca..62c99c348b4 100644
--- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output
+++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output
@@ -269,6 +269,13 @@ This utility can do the following commands:
Get the status of the current snapshot operation:
control.(sh|bat) --snapshot status
+ Deletes the snapshot and all its incremental snapshots from all online server nodes:
+ control.(sh|bat) --snapshot delete snapshot_name [--src path/to/snapshots]
+
+ Parameters:
+ snapshot_name - Snapshot name.
+ --src path/to/snapshots - Path to snapshot location directory. If not specified, the default snapshot configuration directory will be used.
+
Change cluster tag to new value:
control.(sh|bat) --change-tag newTagValue [--yes]
diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output
index 76f0f8c7e0e..d2755b6b269 100644
--- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output
+++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output
@@ -269,6 +269,13 @@ This utility can do the following commands:
Get the status of the current snapshot operation:
control.(sh|bat) --snapshot status
+ Deletes the snapshot and all its incremental snapshots from all online server nodes:
+ control.(sh|bat) --snapshot delete snapshot_name [--src path/to/snapshots]
+
+ Parameters:
+ snapshot_name - Snapshot name.
+ --src path/to/snapshots - Path to snapshot location directory. If not specified, the default snapshot configuration directory will be used.
+
Change cluster tag to new value:
control.(sh|bat) --change-tag newTagValue [--yes]
diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java
index bff6187c715..5df1a59847a 100644
--- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java
+++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java
@@ -78,7 +78,7 @@ public class ContinuousDataLoadApplication extends IgniteAwareApplication {
if (notifyTime + TimeUnit.MILLISECONDS.toNanos(1500) < System.nanoTime())
notifyTime = System.nanoTime();
- // Delayed notify of the initialization to make sure the data load has completelly began and
+ // Delayed notify of the initialization to make sure the data load has completely begun and
// has produced some valuable amount of data.
if (!inited() && warmUpCnt == loaded)
markInitialized();
diff --git a/modules/wire-compatibility-check/wire/enums/org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse$DeleteStatus.json b/modules/wire-compatibility-check/wire/enums/org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse$DeleteStatus.json
new file mode 100644
index 00000000000..a53253b290b
--- /dev/null
+++ b/modules/wire-compatibility-check/wire/enums/org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse$DeleteStatus.json
@@ -0,0 +1,8 @@
+{
+ "class": "org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse$DeleteStatus",
+ "fields": [
+ {"order": 0, "type": "org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse$DeleteStatus", "name": "DELETED"},
+ {"order": 1, "type": "org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse$DeleteStatus", "name": "PARTLY"},
+ {"order": 2, "type": "org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse$DeleteStatus", "name": "NOT_FOUND"}
+ ]
+}
diff --git a/modules/wire-compatibility-check/wire/messages/org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteRequest.json b/modules/wire-compatibility-check/wire/messages/org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteRequest.json
new file mode 100644
index 00000000000..0f9e55ed766
--- /dev/null
+++ b/modules/wire-compatibility-check/wire/messages/org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteRequest.json
@@ -0,0 +1,8 @@
+{
+ "class": "org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteRequest",
+ "fields": [
+ {"order": 0, "type": "java.util.UUID", "name": "reqId"},
+ {"order": 1, "type": "java.lang.String", "name": "snpName"},
+ {"order": 2, "type": "java.lang.String", "name": "snpPath"}
+ ]
+}
diff --git a/modules/wire-compatibility-check/wire/messages/org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse.json b/modules/wire-compatibility-check/wire/messages/org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse.json
new file mode 100644
index 00000000000..6d7e154b2ee
--- /dev/null
+++ b/modules/wire-compatibility-check/wire/messages/org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse.json
@@ -0,0 +1,7 @@
+{
+ "class": "org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse",
+ "fields": [
+ {"order": 0, "type": "org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse$DeleteStatus", "name": "status"},
+ {"order": 1, "type": "java.util.Collection<java.lang.String>", "name": "nodeIds"}
+ ]
+}