You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by mm...@apache.org on 2022/01/21 09:10:57 UTC
[ignite] branch master updated: IGNITE-14794 JMX management and metrics for snapshot restore operation (#9186)
This is an automated email from the ASF dual-hosted git repository.
mmuzaf pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 88a65a9 IGNITE-14794 JMX management and metrics for snapshot restore operation (#9186)
88a65a9 is described below
commit 88a65a97355d2402dd816e68e4552d7d2d1ce02b
Author: Pavel Pereslegin <xx...@gmail.com>
AuthorDate: Fri Jan 21 12:10:17 2022 +0300
IGNITE-14794 JMX management and metrics for snapshot restore operation (#9186)
---
.../snapshot/IgniteSnapshotManager.java | 2 +
.../persistence/snapshot/SnapshotMXBeanImpl.java | 20 ++
.../snapshot/SnapshotRestoreProcess.java | 187 +++++++++------
.../org/apache/ignite/mxbean/SnapshotMXBean.java | 25 ++
.../snapshot/IgniteSnapshotMXBeanTest.java | 104 ++++++++-
.../IgniteClusterSnapshotRestoreMetricsTest.java | 253 +++++++++++++++++++++
...niteClusterSnapshotRestoreWithIndexingTest.java | 2 +-
.../testsuites/IgnitePdsWithIndexingTestSuite.java | 2 -
.../IgniteSnapshotWithIndexingTestSuite.java | 6 +-
9 files changed, 519 insertions(+), 82 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
index 80c5332..7c0d951 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
@@ -442,6 +442,8 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
"The list of names of all snapshots currently saved on the local node with respect to " +
"the configured via IgniteConfiguration snapshot working path.");
+ restoreCacheGrpProc.registerMetrics();
+
cctx.exchange().registerExchangeAwareComponent(this);
ctx.internalSubscriptionProcessor().registerMetastorageListener(this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java
index 9dc1361..60abf07 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java
@@ -17,7 +17,11 @@
package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.mxbean.SnapshotMXBean;
@@ -47,4 +51,20 @@ public class SnapshotMXBeanImpl implements SnapshotMXBean {
@Override public void cancelSnapshot(String snpName) {
mgr.cancelSnapshot(snpName).get();
}
+
+ /** {@inheritDoc} */
+ @Override public void restoreSnapshot(String name, String grpNames) {
+ Set<String> grpNamesSet = F.isEmpty(grpNames) ? null :
+ Arrays.stream(grpNames.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toSet());
+
+ IgniteFuture<Void> fut = mgr.restoreSnapshot(name, grpNamesSet);
+
+ if (fut.isDone())
+ fut.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancelSnapshotRestore(String name) {
+ mgr.cancelSnapshotRestore(name).get();
+ }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
index 8f7092a..866a8e3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
@@ -33,10 +33,12 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiPredicate;
import java.util.function.BooleanSupplier;
@@ -63,6 +65,7 @@ import org.apache.ignite.internal.processors.cache.StoredCacheData;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.ClusterSnapshotFuture;
import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
+import org.apache.ignite.internal.processors.metric.MetricRegistry;
import org.apache.ignite.internal.util.distributed.DistributedProcess;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -98,6 +101,9 @@ public class SnapshotRestoreProcess {
/** Temporary cache directory prefix. */
public static final String TMP_CACHE_DIR_PREFIX = "_tmp_snp_restore_";
+ /** Snapshot restore metrics prefix. */
+ public static final String SNAPSHOT_RESTORE_METRICS = "snapshot-restore";
+
/** Reject operation message. */
private static final String OP_REJECT_MSG = "Cache group restore operation was rejected. ";
@@ -128,9 +134,12 @@ public class SnapshotRestoreProcess {
/** Future to be completed when the cache restore process is complete (this future will be returned to the user). */
private volatile ClusterSnapshotFuture fut;
- /** Snapshot restore operation context. */
+ /** Current snapshot restore operation context (will be {@code null} when the operation is not running). */
private volatile SnapshotRestoreContext opCtx;
+ /** Last snapshot restore operation context (saves the metrics of the last operation). */
+ private volatile SnapshotRestoreContext lastOpCtx = new SnapshotRestoreContext();
+
/**
* @param ctx Kernal context.
*/
@@ -171,6 +180,30 @@ public class SnapshotRestoreProcess {
}
/**
+ * Register local metrics.
+ */
+ protected void registerMetrics() {
+ assert !ctx.clientNode();
+
+ MetricRegistry mreg = ctx.metric().registry(SNAPSHOT_RESTORE_METRICS);
+
+ mreg.register("startTime", () -> lastOpCtx.startTime,
+ "The system time of the start of the cluster snapshot restore operation on this node.");
+ mreg.register("endTime", () -> lastOpCtx.endTime,
+ "The system time when the restore operation of a cluster snapshot on this node ended.");
+ mreg.register("snapshotName", () -> lastOpCtx.snpName, String.class,
+ "The snapshot name of the last running cluster snapshot restore operation on this node.");
+ mreg.register("requestId", () -> Optional.ofNullable(lastOpCtx.reqId).map(UUID::toString).orElse(""),
+ String.class, "The request ID of the last running cluster snapshot restore operation on this node.");
+ mreg.register("error", () -> Optional.ofNullable(lastOpCtx.err.get()).map(Throwable::toString).orElse(""),
+ String.class, "Error message of the last running cluster snapshot restore operation on this node.");
+ mreg.register("totalPartitions", () -> lastOpCtx.totalParts,
+ "The total number of partitions to be restored on this node.");
+ mreg.register("processedPartitions", () -> lastOpCtx.processedParts.get(),
+ "The number of processed partitions on this node.");
+ }
+
+ /**
* Start cache group restore operation.
*
* @param snpName Snapshot name.
@@ -236,7 +269,7 @@ public class SnapshotRestoreProcess {
});
String msg = "Cluster-wide snapshot restore operation started [reqId=" + fut0.rqId + ", snpName=" + snpName +
- (cacheGrpNames == null ? "" : ", grps=" + cacheGrpNames) + ']';
+ (cacheGrpNames == null ? "" : ", caches=" + cacheGrpNames) + ']';
if (log.isInfoEnabled())
log.info(msg);
@@ -387,15 +420,6 @@ public class SnapshotRestoreProcess {
* Finish local cache group restore process.
*
* @param reqId Request ID.
- */
- private void finishProcess(UUID reqId) {
- finishProcess(reqId, null);
- }
-
- /**
- * Finish local cache group restore process.
- *
- * @param reqId Request ID.
* @param err Error, if any.
*/
private void finishProcess(UUID reqId, @Nullable Throwable err) {
@@ -406,9 +430,12 @@ public class SnapshotRestoreProcess {
SnapshotRestoreContext opCtx0 = opCtx;
- if (opCtx0 != null && reqId.equals(opCtx0.reqId))
+ if (opCtx0 != null && reqId.equals(opCtx0.reqId)) {
opCtx = null;
+ opCtx0.endTime = U.currentTimeMillis();
+ }
+
synchronized (this) {
ClusterSnapshotFuture fut0 = fut;
@@ -554,10 +581,12 @@ public class SnapshotRestoreProcess {
", caches=" + req.groups() + ']');
}
- SnapshotRestoreContext opCtx0 = prepareContext(req);
+ List<SnapshotMetadata> locMetas = snpMgr.readSnapshotMetadatas(req.snapshotName());
+
+ SnapshotRestoreContext opCtx0 = prepareContext(req, locMetas);
synchronized (this) {
- opCtx = opCtx0;
+ lastOpCtx = opCtx = opCtx0;
ClusterSnapshotFuture fut0 = fut;
@@ -576,8 +605,7 @@ public class SnapshotRestoreProcess {
if (ctx.isStopping())
throw new NodeStoppingException("The node is stopping: " + ctx.localNodeId());
- return new GridFinishedFuture<>(new SnapshotRestoreOperationResponse(opCtx0.cfgs.values(),
- opCtx0.metasPerNode.get(ctx.localNodeId())));
+ return new GridFinishedFuture<>(new SnapshotRestoreOperationResponse(opCtx0.cfgs.values(), locMetas));
}
catch (IgniteIllegalStateException | IgniteCheckedException | RejectedExecutionException e) {
log.error("Unable to restore cache group(s) from the snapshot " +
@@ -597,18 +625,20 @@ public class SnapshotRestoreProcess {
/**
* @param req Request to prepare cache group restore from the snapshot.
+ * @param metas Local snapshot metadatas.
* @return Snapshot restore operation context.
* @throws IgniteCheckedException If failed.
*/
- private SnapshotRestoreContext prepareContext(SnapshotOperationRequest req) throws IgniteCheckedException {
+ private SnapshotRestoreContext prepareContext(
+ SnapshotOperationRequest req,
+ Collection<SnapshotMetadata> metas
+ ) throws IgniteCheckedException {
if (opCtx != null) {
throw new IgniteCheckedException(OP_REJECT_MSG +
"The previous snapshot restore operation was not completed.");
}
GridCacheSharedContext<?, ?> cctx = ctx.cache().context();
- List<SnapshotMetadata> metas = cctx.snapshotMgr().readSnapshotMetadatas(req.snapshotName());
-
// Collection of baseline nodes that must survive and additional discovery data required for the affinity calculation.
DiscoCache discoCache = ctx.discovery().discoCache();
@@ -618,7 +648,7 @@ public class SnapshotRestoreProcess {
DiscoCache discoCache0 = discoCache.copy(discoCache.version(), null);
if (F.isEmpty(metas))
- return new SnapshotRestoreContext(req, discoCache0, Collections.emptyMap(), cctx.localNodeId(), Collections.emptyList());
+ return new SnapshotRestoreContext(req, discoCache0, Collections.emptyMap());
if (F.first(metas).pageSize() != cctx.database().pageSize()) {
throw new IgniteCheckedException("Incompatible memory page size " +
@@ -674,7 +704,7 @@ public class SnapshotRestoreProcess {
Map<Integer, StoredCacheData> cfgsById =
cfgsByName.values().stream().collect(Collectors.toMap(v -> CU.cacheId(v.config().getName()), v -> v));
- return new SnapshotRestoreContext(req, discoCache0, cfgsById, cctx.localNodeId(), metas);
+ return new SnapshotRestoreContext(req, discoCache0, cfgsById);
}
/**
@@ -719,8 +749,7 @@ public class SnapshotRestoreProcess {
}
}
- opCtx0.metasPerNode.computeIfAbsent(e.getKey(), id -> new ArrayList<>())
- .addAll(e.getValue().metas);
+ opCtx0.metasPerNode.put(e.getKey(), new ArrayList<>(e.getValue().metas));
}
opCtx0.cfgs = globalCfgs;
@@ -806,9 +835,10 @@ public class SnapshotRestoreProcess {
}
if (log.isInfoEnabled()) {
- log.info("Starting snapshot preload operation to restore cache groups" +
- "[snapshot=" + opCtx0.snpName +
- ", caches=" + F.transform(opCtx0.dirs, File::getName) + ']');
+ log.info("Starting snapshot preload operation to restore cache groups " +
+ "[reqId=" + reqId +
+ ", snapshot=" + opCtx0.snpName +
+ ", caches=" + F.transform(opCtx0.dirs, FilePageStoreManager::cacheGroupName) + ']');
}
CompletableFuture<Void> metaFut = ctx.localNodeId().equals(opCtx0.opNodeId) ?
@@ -836,12 +866,12 @@ public class SnapshotRestoreProcess {
grp -> calculateAffinity(ctx, data.config(), opCtx0.discoCache));
}
- // First preload everything from the local node.
- List<SnapshotMetadata> locMetas = opCtx0.metasPerNode.get(ctx.localNodeId());
-
+ Map<Integer, Set<PartitionRestoreFuture>> allParts = new HashMap<>();
Map<Integer, Set<PartitionRestoreFuture>> rmtLoadParts = new HashMap<>();
ClusterNode locNode = ctx.cache().context().localNode();
+ List<SnapshotMetadata> locMetas = opCtx0.metasPerNode.get(locNode.id());
+ // First preload everything from the local node.
for (File dir : opCtx0.dirs) {
String cacheOrGrpName = cacheGroupName(dir);
int grpId = CU.cacheId(cacheOrGrpName);
@@ -865,10 +895,10 @@ public class SnapshotRestoreProcess {
Set<PartitionRestoreFuture> partFuts = availParts
.stream()
.filter(p -> p != INDEX_PARTITION && assignment.get(p).contains(locNode))
- .map(PartitionRestoreFuture::new)
+ .map(p -> new PartitionRestoreFuture(p, opCtx0.processedParts))
.collect(Collectors.toSet());
- opCtx0.locProgress.put(grpId, partFuts);
+ allParts.put(grpId, partFuts);
rmtLoadParts.put(grpId, leftParts = new HashSet<>(partFuts));
@@ -883,7 +913,7 @@ public class SnapshotRestoreProcess {
if (leftParts.isEmpty())
break;
- File snpCacheDir = new File(ctx.cache().context().snapshotMgr().snapshotLocalDir(opCtx.snpName),
+ File snpCacheDir = new File(ctx.cache().context().snapshotMgr().snapshotLocalDir(opCtx0.snpName),
Paths.get(databaseRelativePath(meta.folderName()), dir.getName()).toString());
leftParts.removeIf(partFut -> {
@@ -893,7 +923,7 @@ public class SnapshotRestoreProcess {
if (doCopy) {
copyLocalAsync(ctx.cache().context().snapshotMgr(),
- opCtx,
+ opCtx0,
snpCacheDir,
tmpCacheDir,
partFut);
@@ -907,7 +937,8 @@ public class SnapshotRestoreProcess {
if (log.isInfoEnabled()) {
log.info("The snapshot was taken on the same cluster topology. The index will be copied to " +
- "restoring cache group if necessary [snpName=" + opCtx0.snpName + ", dir=" + dir.getName() + ']');
+ "restoring cache group if necessary [reqId=" + reqId + ", snapshot=" + opCtx0.snpName +
+ ", dir=" + dir.getName() + ']');
}
File idxFile = new File(snpCacheDir, FilePageStoreManager.getPartitionFileName(INDEX_PARTITION));
@@ -915,11 +946,11 @@ public class SnapshotRestoreProcess {
if (idxFile.exists()) {
PartitionRestoreFuture idxFut;
- opCtx0.locProgress.computeIfAbsent(grpId, g -> new HashSet<>())
- .add(idxFut = new PartitionRestoreFuture(INDEX_PARTITION));
+ allParts.computeIfAbsent(grpId, g -> new HashSet<>())
+ .add(idxFut = new PartitionRestoreFuture(INDEX_PARTITION, opCtx0.processedParts));
copyLocalAsync(ctx.cache().context().snapshotMgr(),
- opCtx,
+ opCtx0,
snpCacheDir,
tmpCacheDir,
idxFut);
@@ -940,7 +971,7 @@ public class SnapshotRestoreProcess {
.filter(e -> !e.getKey().equals(ctx.localNodeId()))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)),
(grpId, partId) -> rmtLoadParts.get(grpId) != null &&
- rmtLoadParts.get(grpId).remove(new PartitionRestoreFuture(partId)));
+ rmtLoadParts.get(grpId).remove(new PartitionRestoreFuture(partId, opCtx0.processedParts)));
Map<Integer, File> grpToDir = opCtx0.dirs.stream()
.collect(Collectors.toMap(d -> CU.cacheId(FilePageStoreManager.cacheGroupName(d)),
@@ -948,8 +979,9 @@ public class SnapshotRestoreProcess {
try {
if (log.isInfoEnabled() && !snpAff.isEmpty()) {
- log.info("Trying to request partitions from remote nodes" +
- "[snapshot=" + opCtx0.snpName +
+ log.info("Trying to request partitions from remote nodes " +
+ "[reqId=" + reqId +
+ ", snapshot=" + opCtx0.snpName +
", map=" + snpAff.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey,
e -> partitionsMapToCompactString(e.getValue()))) + ']');
}
@@ -968,7 +1000,7 @@ public class SnapshotRestoreProcess {
int grpId = CU.cacheId(cacheGroupName(snpFile.getParentFile()));
int partId = partId(snpFile.getName());
- PartitionRestoreFuture partFut = F.find(opCtx0.locProgress.get(grpId),
+ PartitionRestoreFuture partFut = F.find(allParts.get(grpId),
null,
new IgnitePredicate<PartitionRestoreFuture>() {
@Override public boolean apply(PartitionRestoreFuture f) {
@@ -1007,12 +1039,14 @@ public class SnapshotRestoreProcess {
completeListExceptionally(rmtAwaitParts, e);
}
- List<PartitionRestoreFuture> allParts = opCtx0.locProgress.values().stream().flatMap(Collection::stream)
+ List<PartitionRestoreFuture> allPartFuts = allParts.values().stream().flatMap(Collection::stream)
.collect(Collectors.toList());
- int size = allParts.size();
+ int size = allPartFuts.size();
+
+ opCtx0.totalParts = size;
- CompletableFuture.allOf(allParts.toArray(new CompletableFuture[size]))
+ CompletableFuture.allOf(allPartFuts.toArray(new CompletableFuture[size]))
.runAfterBothAsync(metaFut, () -> {
try {
if (opCtx0.stopChecker.getAsBoolean())
@@ -1065,8 +1099,6 @@ public class SnapshotRestoreProcess {
opCtx0.errHnd.accept(failure);
if (failure != null) {
- opCtx0.locStopCachesCompleteFut.onDone((Void)null);
-
if (U.isLocalNodeCoordinator(ctx.discovery()))
rollbackRestoreProc.start(reqId, reqId);
@@ -1124,7 +1156,7 @@ public class SnapshotRestoreProcess {
orElse(checkNodeLeft(opCtx0.nodes(), res.keySet()));
if (failure == null) {
- finishProcess(reqId);
+ finishProcess(reqId, null);
return;
}
@@ -1375,41 +1407,46 @@ public class SnapshotRestoreProcess {
/** Stop condition checker. */
private final BooleanSupplier stopChecker = () -> err.get() != null;
- /** Progress of processing cache group partitions on the local node.*/
- private final Map<Integer, Set<PartitionRestoreFuture>> locProgress = new HashMap<>();
-
- /**
- * The stop future responsible for stopping cache groups during the rollback phase. Will be completed when the rollback
- * process executes and all the cache group stop actions completes (the processCacheStopRequestOnExchangeDone finishes
- * successfully and all the data deleted from disk).
- */
- private final GridFutureAdapter<Void> locStopCachesCompleteFut = new GridFutureAdapter<>();
-
/** Cache ID to configuration mapping. */
- private volatile Map<Integer, StoredCacheData> cfgs;
+ private volatile Map<Integer, StoredCacheData> cfgs = Collections.emptyMap();
/** Graceful shutdown future. */
private volatile IgniteFuture<?> stopFut;
+ /** Operation start time. */
+ private final long startTime;
+
+ /** Number of processed (copied) partitions. */
+ private final AtomicInteger processedParts = new AtomicInteger(0);
+
+ /** Total number of partitions to be restored. */
+ private volatile int totalParts = -1;
+
+ /** Operation end time. */
+ private volatile long endTime;
+
+ /** Creates an empty context. */
+ protected SnapshotRestoreContext() {
+ reqId = null;
+ snpName = "";
+ startTime = 0;
+ opNodeId = null;
+ discoCache = null;
+ }
+
/**
* @param req Request to prepare cache group restore from the snapshot.
+ * @param discoCache Baseline discovery cache for node IDs that must be alive to complete the operation.
* @param cfgs Cache ID to configuration mapping.
*/
- protected SnapshotRestoreContext(
- SnapshotOperationRequest req,
- DiscoCache discoCache,
- Map<Integer, StoredCacheData> cfgs,
- UUID locNodeId,
- List<SnapshotMetadata> locMetas
- ) {
+ protected SnapshotRestoreContext(SnapshotOperationRequest req, DiscoCache discoCache, Map<Integer, StoredCacheData> cfgs) {
reqId = req.requestId();
snpName = req.snapshotName();
opNodeId = req.operationalNodeId();
- this.discoCache = discoCache;
+ startTime = U.currentTimeMillis();
+ this.discoCache = discoCache;
this.cfgs = cfgs;
-
- metasPerNode.computeIfAbsent(locNodeId, id -> new ArrayList<>()).addAll(locMetas);
}
/**
@@ -1449,11 +1486,23 @@ public class SnapshotRestoreProcess {
/** Partition id. */
private final int partId;
+ /** Counter of the total number of processed partitions. */
+ private final AtomicInteger cntr;
+
/**
* @param partId Partition id.
+ * @param cntr Counter of the total number of processed partitions.
*/
- private PartitionRestoreFuture(int partId) {
+ private PartitionRestoreFuture(int partId, AtomicInteger cntr) {
this.partId = partId;
+ this.cntr = cntr;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean complete(Path path) {
+ cntr.incrementAndGet();
+
+ return super.complete(path);
}
/** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java
index e6b42583..fcb7a399 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java
@@ -17,6 +17,7 @@
package org.apache.ignite.mxbean;
+import java.util.Collection;
import org.apache.ignite.IgniteSnapshot;
/**
@@ -40,4 +41,28 @@ public interface SnapshotMXBean {
*/
@MXBeanDescription("Cancel started cluster-wide snapshot on the node initiator.")
public void cancelSnapshot(@MXBeanParameter(name = "snpName", description = "Snapshot name.") String snpName);
+
+ /**
+ * Restore cluster-wide snapshot.
+ *
+ * @param name Snapshot name.
+ * @param cacheGroupNames Optional comma-separated list of cache group names.
+ * @see IgniteSnapshot#restoreSnapshot(String, Collection)
+ */
+ @MXBeanDescription("Restore cluster-wide snapshot.")
+ public void restoreSnapshot(
+ @MXBeanParameter(name = "snpName", description = "Snapshot name.")
+ String name,
+ @MXBeanParameter(name = "cacheGroupNames", description = "Optional comma-separated list of cache group names.")
+ String cacheGroupNames
+ );
+
+ /**
+ * Cancel previously started snapshot restore operation.
+ *
+ * @param name Snapshot name.
+ * @see IgniteSnapshot#cancelSnapshotRestore(String)
+ */
+ @MXBeanDescription("Cancel previously started snapshot restore operation.")
+ public void cancelSnapshotRestore(@MXBeanParameter(name = "snpName", description = "Snapshot name.") String name);
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java
index c662a5a..22d0ff0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java
@@ -22,19 +22,30 @@ import javax.management.AttributeNotFoundException;
import javax.management.DynamicMBean;
import javax.management.MBeanException;
import javax.management.ReflectionException;
+import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.util.distributed.SingleNodeMessage;
+import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.mxbean.SnapshotMXBean;
import org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi;
import org.apache.ignite.testframework.GridTestUtils;
import org.junit.Test;
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.SNAPSHOT_METRICS;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotRestoreProcess.SNAPSHOT_RESTORE_METRICS;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
/**
* Tests {@link SnapshotMXBean}.
*/
public class IgniteSnapshotMXBeanTest extends AbstractSnapshotSelfTest {
+ /** Metric group name. */
+ private static final String METRIC_GROUP = "Snapshot";
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
return super.getConfiguration(igniteInstanceName)
@@ -49,14 +60,14 @@ public class IgniteSnapshotMXBeanTest extends AbstractSnapshotSelfTest {
DynamicMBean snpMBean = metricRegistry(ignite.name(), null, SNAPSHOT_METRICS);
assertEquals("Snapshot end time must be undefined on first snapshot operation starts.",
- 0, getLastSnapshotEndTime(snpMBean));
+ 0, (long)getMetric("LastSnapshotEndTime", snpMBean));
- SnapshotMXBean mxBean = getMxBean(ignite.name(), "Snapshot", SnapshotMXBeanImpl.class, SnapshotMXBean.class);
+ SnapshotMXBean mxBean = getMxBean(ignite.name(), METRIC_GROUP, SnapshotMXBeanImpl.class, SnapshotMXBean.class);
mxBean.createSnapshot(SNAPSHOT_NAME);
assertTrue("Waiting for snapshot operation failed.",
- GridTestUtils.waitForCondition(() -> getLastSnapshotEndTime(snpMBean) > 0, 10_000));
+ GridTestUtils.waitForCondition(() -> (long)getMetric("LastSnapshotEndTime", snpMBean) > 0, TIMEOUT));
stopAllGrids();
@@ -72,7 +83,7 @@ public class IgniteSnapshotMXBeanTest extends AbstractSnapshotSelfTest {
IgniteEx startCli = startClientGrid(1);
IgniteEx killCli = startClientGrid(2);
- SnapshotMXBean mxBean = getMxBean(killCli.name(), "Snapshot", SnapshotMXBeanImpl.class,
+ SnapshotMXBean mxBean = getMxBean(killCli.name(), METRIC_GROUP, SnapshotMXBeanImpl.class,
SnapshotMXBean.class);
doSnapshotCancellationTest(startCli,
@@ -81,14 +92,89 @@ public class IgniteSnapshotMXBeanTest extends AbstractSnapshotSelfTest {
mxBean::cancelSnapshot);
}
- /**
+ /** @throws Exception If fails. */
+ @Test
+ public void testRestoreSnapshot() throws Exception {
+ // TODO IGNITE-14999 Support dynamic restoration of encrypted snapshots.
+ if (encryption)
+ return;
+
+ IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE, false);
+
+ DynamicMBean mReg0 = metricRegistry(grid(0).name(), null, SNAPSHOT_RESTORE_METRICS);
+ DynamicMBean mReg1 = metricRegistry(grid(1).name(), null, SNAPSHOT_RESTORE_METRICS);
+
+ assertEquals(0, (long)getMetric("endTime", mReg0));
+ assertEquals(0, (long)getMetric("endTime", mReg1));
+
+ getMxBean(ignite.name(), METRIC_GROUP, SnapshotMXBeanImpl.class, SnapshotMXBean.class)
+ .restoreSnapshot(SNAPSHOT_NAME, null);
+
+ assertTrue(GridTestUtils.waitForCondition(() -> (long)getMetric("endTime", mReg0) > 0, TIMEOUT));
+ assertTrue(GridTestUtils.waitForCondition(() -> (long)getMetric("endTime", mReg1) > 0, TIMEOUT));
+
+ assertCacheKeys(ignite.cache(DEFAULT_CACHE_NAME), CACHE_KEYS_RANGE);
+ }
+
+ /** @throws Exception If fails. */
+ @Test
+ public void testCancelRestoreSnapshot() throws Exception {
+ // TODO IGNITE-14999 Support dynamic restoration of encrypted snapshots.
+ if (encryption)
+ return;
+
+ IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE, false);
+ SnapshotMXBean mxBean = getMxBean(ignite.name(), METRIC_GROUP, SnapshotMXBeanImpl.class, SnapshotMXBean.class);
+ DynamicMBean mReg0 = metricRegistry(grid(0).name(), null, SNAPSHOT_RESTORE_METRICS);
+ DynamicMBean mReg1 = metricRegistry(grid(1).name(), null, SNAPSHOT_RESTORE_METRICS);
- * @param mBean Ignite snapshot MBean.
- * @return Value of snapshot end time.
+ assertEquals("", getMetric("error", mReg0));
+ assertEquals("", getMetric("error", mReg1));
+ assertEquals(0, (long)getMetric("endTime", mReg0));
+ assertEquals(0, (long)getMetric("endTime", mReg1));
+
+ TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(grid(1));
+
+ spi.blockMessages((node, msg) -> msg instanceof SingleNodeMessage &&
+ ((SingleNodeMessage<?>)msg).type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal());
+
+ IgniteFuture<Void> fut = ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null);
+
+ spi.waitForBlocked();
+
+ IgniteInternalFuture<Boolean> interruptFut = GridTestUtils.runAsync(() -> {
+ try {
+ return GridTestUtils.waitForCondition(
+ () -> !"".equals(getMetric("error", mReg0)) && !"".equals(getMetric("error", mReg1)), TIMEOUT);
+ } finally {
+ spi.stopBlock();
+ }
+ });
+
+ mxBean.cancelSnapshotRestore(SNAPSHOT_NAME);
+
+ assertTrue(interruptFut.get());
+
+ String expErrMsg = "Operation has been canceled by the user.";
+
+ assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expErrMsg);
+
+ assertTrue((long)getMetric("endTime", mReg0) > 0);
+ assertTrue((long)getMetric("endTime", mReg1) > 0);
+ assertTrue(((String)getMetric("error", mReg0)).contains(expErrMsg));
+ assertTrue(((String)getMetric("error", mReg1)).contains(expErrMsg));
+
+ assertNull(ignite.cache(DEFAULT_CACHE_NAME));
+ }
+
+ /**
+ * @param mBean Ignite snapshot restore MBean.
+ * @param name Metric name.
+ * @return Metric value.
*/
- private static long getLastSnapshotEndTime(DynamicMBean mBean) {
+ private static <T> T getMetric(String name, DynamicMBean mBean) {
try {
- return (long)mBean.getAttribute("LastSnapshotEndTime");
+ return (T)mBean.getAttribute(name);
}
catch (MBeanException | ReflectionException | AttributeNotFoundException e) {
throw new RuntimeException(e);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreMetricsTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreMetricsTest.java
new file mode 100644
index 0000000..55bb94a
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreMetricsTest.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Set;
+import javax.management.AttributeNotFoundException;
+import javax.management.DynamicMBean;
+import javax.management.MBeanException;
+import javax.management.ReflectionException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.FILE_SUFFIX;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_PREFIX;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotRestoreProcess.SNAPSHOT_RESTORE_METRICS;
+
+/**
+ * Tests snapshot restore metrics.
+ */
+public class IgniteClusterSnapshotRestoreMetricsTest extends IgniteClusterSnapshotRestoreBaseTest {
+ /** Separate working directory prefix. */
+ private static final String DEDICATED_DIR_PREFIX = "dedicated-";
+
+ /** Number of nodes using a separate working directory. */
+ private static final int DEDICATED_CNT = 2;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName)
+ .setMetricExporterSpi(new JmxMetricExporterSpi());
+
+ if (getTestIgniteInstanceIndex(igniteInstanceName) < DEDICATED_CNT) {
+ cfg.setWorkDirectory(Paths.get(U.defaultWorkDirectory(),
+ DEDICATED_DIR_PREFIX + U.maskForFileName(cfg.getIgniteInstanceName())).toString());
+ }
+
+ return cfg;
+ }
+
+ /**
+ * @param name Cache name.
+ * @return Cache configuration.
+ */
+ private CacheConfiguration<Integer, Object> cacheConfig(String name) {
+ return new CacheConfiguration<>(dfltCacheCfg)
+ .setName(name)
+ .setSqlSchema(name)
+ .setQueryEntities(Collections.singletonList(
+ new QueryEntity(Integer.class.getName(), Account.class.getName())
+ .setFields(new LinkedHashMap<>(F.asMap(
+ "id", Integer.class.getName(),
+ "balance", Integer.class.getName()))
+ )
+ .setIndexes(Collections.singletonList(new QueryIndex("id"))))
+ );
+ }
+
+ /** {@inheritDoc} */
+ @Before
+ @Override public void beforeTestSnapshot() throws Exception {
+ super.beforeTestSnapshot();
+
+ cleanuo();
+ }
+
+ /** {@inheritDoc} */
+ @After
+ @Override public void afterTestSnapshot() throws Exception {
+ super.afterTestSnapshot();
+
+ cleanuo();
+ }
+
+ /** @throws Exception If fails. */
+ @Test
+ public void testRestoreSnapshotProgress() throws Exception {
+ // Caches with differebt partition distribution.
+ CacheConfiguration<Integer, Object> ccfg1 = cacheConfig("cache1").setBackups(0);
+ CacheConfiguration<Integer, Object> ccfg2 = cacheConfig("cache2").setCacheMode(CacheMode.REPLICATED);
+
+ Ignite ignite = startGridsWithCache(DEDICATED_CNT, CACHE_KEYS_RANGE, key -> new Account(key, key), ccfg1, ccfg2);
+
+ ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ ignite.destroyCaches(F.asList(ccfg1.getName(), ccfg2.getName()));
+ awaitPartitionMapExchange();
+
+ // Add new empty node.
+ IgniteEx emptyNode = startGrid(DEDICATED_CNT);
+ resetBaselineTopology();
+
+ checkMetricsDefaults();
+
+ Set<String> grpNames = new HashSet<>(F.asList(ccfg1.getName(), ccfg2.getName()));
+
+ ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, grpNames);
+
+ for (Ignite grid : G.allGrids()) {
+ DynamicMBean mReg = metricRegistry(grid.name(), null, SNAPSHOT_RESTORE_METRICS);
+ String nodeNameMsg = "node=" + grid.name();
+
+ assertTrue(nodeNameMsg, GridTestUtils.waitForCondition(() -> getNumMetric("endTime", mReg) > 0, TIMEOUT));
+
+ int expParts = ((IgniteEx)grid).cachex(ccfg1.getName()).context().topology().localPartitions().size() +
+ ((IgniteEx)grid).cachex(ccfg2.getName()).context().topology().localPartitions().size();
+
+ // Cache2 is replicated - the index partition is being copied (on snapshot data nodes).
+ if (!emptyNode.name().equals(grid.name()))
+ expParts += 1;
+
+ assertEquals(nodeNameMsg, SNAPSHOT_NAME, mReg.getAttribute("snapshotName"));
+ assertEquals(nodeNameMsg, "", mReg.getAttribute("error"));
+
+ assertFalse(nodeNameMsg, ((String)mReg.getAttribute("requestId")).isEmpty());
+
+ assertEquals(nodeNameMsg, expParts, getNumMetric("totalPartitions", mReg));
+ assertEquals(nodeNameMsg, expParts, getNumMetric("processedPartitions", mReg));
+
+ long startTime = getNumMetric("startTime", mReg);
+ long endTime = getNumMetric("endTime", mReg);
+
+ assertTrue(nodeNameMsg, startTime > 0);
+ assertTrue(nodeNameMsg, endTime >= startTime);
+ }
+
+ assertSnapshotCacheKeys(ignite.cache(ccfg1.getName()));
+ assertSnapshotCacheKeys(ignite.cache(ccfg2.getName()));
+ }
+
+ /** @throws Exception If fails. */
+ @Test
+ public void testRestoreSnapshotError() throws Exception {
+ dfltCacheCfg.setCacheMode(CacheMode.REPLICATED);
+
+ IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE);
+
+ String failingFilePath = Paths.get(FilePageStoreManager.cacheDirName(dfltCacheCfg),
+ PART_FILE_PREFIX + (dfltCacheCfg.getAffinity().partitions() / 2) + FILE_SUFFIX).toString();
+
+ FileIOFactory ioFactory = new RandomAccessFileIOFactory();
+ String testErrMsg = "Test exception";
+
+ ignite.context().cache().context().snapshotMgr().ioFactory((file, modes) -> {
+ FileIO delegate = ioFactory.create(file, modes);
+
+ if (file.getPath().endsWith(failingFilePath))
+ throw new RuntimeException(testErrMsg);
+
+ return delegate;
+ });
+
+ checkMetricsDefaults();
+
+ ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null);
+
+ for (Ignite grid : G.allGrids()) {
+ DynamicMBean mReg = metricRegistry(grid.name(), null, SNAPSHOT_RESTORE_METRICS);
+
+ String nodeNameMsg = "node=" + grid.name();
+
+ assertTrue(nodeNameMsg, GridTestUtils.waitForCondition(() -> getNumMetric("endTime", mReg) > 0, TIMEOUT));
+
+ long startTime = getNumMetric("startTime", mReg);
+ long endTime = getNumMetric("endTime", mReg);
+
+ assertEquals(nodeNameMsg, SNAPSHOT_NAME, mReg.getAttribute("snapshotName"));
+
+ assertFalse(nodeNameMsg, ((String)mReg.getAttribute("requestId")).isEmpty());
+
+ assertTrue(nodeNameMsg, startTime > 0);
+ assertTrue(nodeNameMsg, endTime >= startTime);
+ assertTrue(nodeNameMsg, ((String)mReg.getAttribute("error")).contains(testErrMsg));
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void checkMetricsDefaults() throws Exception {
+ for (Ignite grid : G.allGrids()) {
+ String nodeNameMsg = "node=" + grid.name();
+
+ DynamicMBean mReg = metricRegistry(grid.name(), null, SNAPSHOT_RESTORE_METRICS);
+
+ assertEquals(nodeNameMsg, 0, getNumMetric("endTime", mReg));
+ assertEquals(nodeNameMsg, -1, getNumMetric("totalPartitions", mReg));
+ assertEquals(nodeNameMsg, 0, getNumMetric("processedPartitions", mReg));
+ assertTrue(nodeNameMsg, String.valueOf(mReg.getAttribute("snapshotName")).isEmpty());
+ }
+ }
+
+ /**
+ * @param mBean Ignite snapshot restore MBean.
+ * @param name Metric name.
+ * @return Metric value.
+ */
+ private long getNumMetric(String name, DynamicMBean mBean) {
+ try {
+ return ((Number)mBean.getAttribute(name)).longValue();
+ }
+ catch (MBeanException | ReflectionException | AttributeNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void cleanuo() throws Exception {
+ FilenameFilter filter = (file, name) -> file.isDirectory() && name.startsWith(DEDICATED_DIR_PREFIX);
+
+ for (File file : new File(U.defaultWorkDirectory()).listFiles(filter))
+ U.delete(file);
+ }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreWithIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreWithIndexingTest.java
index 7db3ff6..e1bc45d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreWithIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreWithIndexingTest.java
@@ -159,7 +159,7 @@ public class IgniteClusterSnapshotRestoreWithIndexingTest extends IgniteClusterS
SnapshotEvent startEvt = evts.get(0);
assertEquals(SNAPSHOT_NAME, startEvt.snapshotName());
- assertTrue(startEvt.message().contains("grps=[" + DEFAULT_CACHE_NAME + ']'));
+ assertTrue(startEvt.message().contains("caches=[" + DEFAULT_CACHE_NAME + ']'));
}
/** {@inheritDoc} */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
index acc21f4..965a881 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
@@ -32,7 +32,6 @@ import org.apache.ignite.internal.processors.cache.persistence.db.IgniteTcBotIni
import org.apache.ignite.internal.processors.cache.persistence.db.IndexingMultithreadedLoadContinuousRestartTest;
import org.apache.ignite.internal.processors.cache.persistence.db.LongDestroyDurableBackgroundTaskTest;
import org.apache.ignite.internal.processors.cache.persistence.db.MultipleParallelCacheDeleteDeadlockTest;
-import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotRestoreWithIndexingTest;
import org.apache.ignite.internal.processors.database.IgniteDbMultiNodeWithIndexingPutGetTest;
import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingPutGetTest;
import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingWalRestoreTest;
@@ -68,7 +67,6 @@ import org.junit.runners.Suite;
IgnitePdsIndexingDefragmentationTest.class,
StopRebuildIndexTest.class,
ForceRebuildIndexTest.class,
- IgniteClusterSnapshotRestoreWithIndexingTest.class,
ResumeRebuildIndexTest.class,
ResumeCreateIndexTest.class,
RenameIndexTreeTest.class,
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotWithIndexingTestSuite.java
index 4696a78..40d6654 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotWithIndexingTestSuite.java
@@ -18,6 +18,8 @@
package org.apache.ignite.testsuites;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotCheckWithIndexesTest;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotRestoreMetricsTest;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotRestoreWithIndexingTest;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotWithIndexesTest;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -26,7 +28,9 @@ import org.junit.runners.Suite;
@RunWith(Suite.class)
@Suite.SuiteClasses({
IgniteClusterSnapshotWithIndexesTest.class,
- IgniteClusterSnapshotCheckWithIndexesTest.class
+ IgniteClusterSnapshotCheckWithIndexesTest.class,
+ IgniteClusterSnapshotRestoreWithIndexingTest.class,
+ IgniteClusterSnapshotRestoreMetricsTest.class
})
public class IgniteSnapshotWithIndexingTestSuite {
}