You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by na...@apache.org on 2021/10/08 08:29:39 UTC
[ignite] branch master updated: IGNITE-15159: Provided the ability
to snapshot encrypted caches. (#9269)
This is an automated email from the ASF dual-hosted git repository.
namelchev pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 0aa8643 IGNITE-15159: Provided the ability to snapshot encrypted caches. (#9269)
0aa8643 is described below
commit 0aa86435472c5f9648231dd3a61f06d60c41abdb
Author: Vladimir Steshin <vl...@gmail.com>
AuthorDate: Fri Oct 8 11:29:08 2021 +0300
IGNITE-15159: Provided the ability to snapshot encrypted caches. (#9269)
---
.../managers/encryption/GridEncryptionManager.java | 13 +
.../managers/encryption/GroupKeyChangeProcess.java | 6 +
.../processors/cache/GridCacheProcessor.java | 8 +
.../cache/persistence/file/EncryptedFileIO.java | 78 +++---
.../persistence/file/EncryptedFileIOFactory.java | 15 +-
.../persistence/file/FilePageStoreManager.java | 29 +-
.../snapshot/IgniteSnapshotManager.java | 57 ++--
.../persistence/snapshot/SnapshotFutureTask.java | 21 +-
.../snapshot/SnapshotRestoreProcess.java | 10 +
.../processors/odbc/ClientListenerProcessor.java | 2 +
.../snapshot/AbstractSnapshotSelfTest.java | 148 ++++++++++
.../snapshot/EncryptedSnapshotTest.java | 298 +++++++++++++++++++++
.../snapshot/IgniteClusterSnapshotCheckTest.java | 89 +++++-
.../IgniteClusterSnapshotRestoreBaseTest.java | 86 +-----
.../IgniteClusterSnapshotRestoreSelfTest.java | 14 +-
.../snapshot/IgniteClusterSnapshotSelfTest.java | 2 +
.../snapshot/IgniteSnapshotManagerSelfTest.java | 99 +------
.../IgniteBasicWithPersistenceTestSuite.java | 2 +
.../IgniteClusterSnapshotCheckWithIndexesTest.java | 8 +
.../IgniteClusterSnapshotWithIndexesTest.java | 7 +
20 files changed, 715 insertions(+), 277 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GridEncryptionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GridEncryptionManager.java
index 1d8e93d..3d8e447 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GridEncryptionManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GridEncryptionManager.java
@@ -816,6 +816,13 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
}
/**
+ * @return {@code True} If reencryption is active in the cluster.
+ */
+ public boolean reencryptionInProgress() {
+ return grpKeyChangeProc.inProgress() || !reencryptGroups.isEmpty();
+ }
+
+ /**
* @return Re-encryption rate limit in megabytes per second ({@code 0} - unlimited).
*/
public double getReencryptionRate() {
@@ -1517,6 +1524,12 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
"The previous change was not completed."));
}
+ if (ctx.cache().context().snapshotMgr().isSnapshotCreating()
+ || ctx.cache().context().snapshotMgr().isRestoring()) {
+ return new GridFinishedFuture<>(new IgniteException("Master key change was rejected. Snapshot operation " +
+ "is in progress."));
+ }
+
masterKeyChangeRequest = req;
if (ctx.clientNode())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyChangeProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyChangeProcess.java
index 2fac162..387bb04 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyChangeProcess.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyChangeProcess.java
@@ -197,6 +197,12 @@ class GroupKeyChangeProcess {
"The previous change was not completed."));
}
+ if (ctx.cache().context().snapshotMgr().isSnapshotCreating()
+ || ctx.cache().context().snapshotMgr().isRestoring()) {
+ return new GridFinishedFuture<>(new IgniteException("Cache group key change was rejected. " +
+ "Snapshot operation is in progress."));
+ }
+
this.req = req;
try {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 9203a89..89ec474 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -123,6 +123,7 @@ import org.apache.ignite.internal.processors.cache.persistence.checkpoint.Checkp
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
import org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage;
import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
@@ -4062,6 +4063,13 @@ public class GridCacheProcessor extends GridProcessorAdapter {
}
/**
+ * @return {@code True} if cache group {@code cacheGrpId} is encrypted. {@code False} otherwise.
+ */
+ public boolean isEncrypted(int cacheGrpId) {
+ return cacheGrpId != MetaStorage.METASTORAGE_CACHE_ID && cacheGroup(cacheGrpId).config().isEncryptionEnabled();
+ }
+
+ /**
* Save cache configuration to persistent store if necessary.
*
* @param desc Cache descriptor.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java
index 01d9a06..0f0c1a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java
@@ -20,7 +20,7 @@ package org.apache.ignite.internal.processors.cache.persistence.file;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
-import org.apache.ignite.internal.managers.encryption.GridEncryptionManager;
+import org.apache.ignite.internal.managers.encryption.EncryptionCacheKeyProvider;
import org.apache.ignite.internal.managers.encryption.GroupKey;
import org.apache.ignite.spi.encryption.EncryptionSpi;
@@ -46,14 +46,14 @@ public class EncryptedFileIO implements FileIO {
private final int pageSize;
/**
- * Size of file header in bytes.
+ * Size of file header in bytes which is never encrypted.
*/
- private final int headerSize;
+ private final int plainHeaderSize;
/**
- * Shared database manager.
+ * Encryption keys provider.
*/
- private final GridEncryptionManager encMgr;
+ private final EncryptionCacheKeyProvider keyProvider;
/**
* Shared database manager.
@@ -67,16 +67,16 @@ public class EncryptedFileIO implements FileIO {
* @param plainFileIO Underlying file.
* @param groupId Group id.
* @param pageSize Size of plain data page in bytes.
- * @param headerSize Size of file header in bytes.
- * @param encMgr Encryption manager.
+ * @param plainHeaderSize Size of file header in bytes which is never encrypted.
+ * @param keyProvider Encryption keys provider.
*/
- EncryptedFileIO(FileIO plainFileIO, int groupId, int pageSize, int headerSize,
- GridEncryptionManager encMgr, EncryptionSpi encSpi) {
+ EncryptedFileIO(FileIO plainFileIO, int groupId, int pageSize, int plainHeaderSize, EncryptionCacheKeyProvider keyProvider,
+ EncryptionSpi encSpi) {
this.plainFileIO = plainFileIO;
this.groupId = groupId;
this.pageSize = pageSize;
- this.headerSize = headerSize;
- this.encMgr = encMgr;
+ this.plainHeaderSize = plainHeaderSize;
+ this.keyProvider = keyProvider;
this.encSpi = encSpi;
this.encUtil = new EncryptionUtil(encSpi, pageSize);
@@ -109,7 +109,7 @@ public class EncryptedFileIO implements FileIO {
/** {@inheritDoc} */
@Override public int read(ByteBuffer destBuf) throws IOException {
- assert position() == 0;
+ assert position() == 0 && plainHeaderSize > 0;
return plainFileIO.read(destBuf);
}
@@ -146,7 +146,7 @@ public class EncryptedFileIO implements FileIO {
/** {@inheritDoc} */
@Override public int readFully(ByteBuffer destBuf, long position) throws IOException {
assert destBuf.capacity() == pageSize;
- assert position() != 0;
+ assert position() >= plainHeaderSize;
ByteBuffer encrypted = ByteBuffer.allocate(pageSize);
@@ -179,10 +179,14 @@ public class EncryptedFileIO implements FileIO {
/** {@inheritDoc} */
@Override public int write(ByteBuffer srcBuf) throws IOException {
- assert position() == 0;
- assert headerSize == srcBuf.capacity();
+ if (plainHeaderSize > 0) {
+ assert position() == 0;
+ assert plainHeaderSize == srcBuf.capacity();
- return plainFileIO.write(srcBuf);
+ return plainFileIO.write(srcBuf);
+ }
+ else
+ return plainFileIO.writeFully(encrypt(srcBuf));
}
/** {@inheritDoc} */
@@ -192,37 +196,31 @@ public class EncryptedFileIO implements FileIO {
/** {@inheritDoc} */
@Override public int write(ByteBuffer srcBuf, long position) throws IOException {
- ByteBuffer encrypted = ByteBuffer.allocate(pageSize);
-
- encrypt(srcBuf, encrypted);
-
- encrypted.rewind();
-
- return plainFileIO.write(encrypted, position);
+ return plainFileIO.write(encrypt(srcBuf), position);
}
/** {@inheritDoc} */
@Override public int writeFully(ByteBuffer srcBuf, long position) throws IOException {
- ByteBuffer encrypted = ByteBuffer.allocate(pageSize);
-
- encrypt(srcBuf, encrypted);
-
- encrypted.rewind();
-
- return plainFileIO.writeFully(encrypted, position);
+ return plainFileIO.writeFully(encrypt(srcBuf), position);
}
/**
- * @param srcBuf Source buffer.
- * @param res Destination buffer.
- * @throws IOException If failed.
+ * @return Encrypted data.
*/
- private void encrypt(ByteBuffer srcBuf, ByteBuffer res) throws IOException {
- assert position() != 0;
+ private ByteBuffer encrypt(ByteBuffer srcBuf) throws IOException {
+ assert position() >= plainHeaderSize;
+
+ ByteBuffer encrypted = ByteBuffer.allocate(pageSize);
+
+ GroupKey key = keyProvider.getActiveKey(groupId);
+
+ assert key != null : "No active encryption key found for cache group " + groupId;
- GroupKey grpKey = encMgr.getActiveKey(groupId);
+ encUtil.encrypt(srcBuf, encrypted, key);
+
+ encrypted.rewind();
- encUtil.encrypt(srcBuf, res, grpKey);
+ return encrypted;
}
/**
@@ -232,11 +230,11 @@ public class EncryptedFileIO implements FileIO {
private void decrypt(ByteBuffer encrypted, ByteBuffer destBuf) throws IOException {
int keyId = encrypted.get(encryptedDataSize() + 4 /* CRC size. */) & 0xff;
- GroupKey grpKey = encMgr.groupKey(groupId, keyId);
+ GroupKey key = keyProvider.groupKey(groupId, keyId);
- assert grpKey != null : keyId;
+ assert key != null : "No encryption key found for cache group " + groupId + " by key id " + keyId;
- encUtil.decrypt(encrypted, destBuf, grpKey);
+ encUtil.decrypt(encrypted, destBuf, key);
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIOFactory.java
index c588981..c02bb65 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIOFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIOFactory.java
@@ -20,7 +20,7 @@ package org.apache.ignite.internal.processors.cache.persistence.file;
import java.io.File;
import java.io.IOException;
import java.nio.file.OpenOption;
-import org.apache.ignite.internal.managers.encryption.GridEncryptionManager;
+import org.apache.ignite.internal.managers.encryption.EncryptionCacheKeyProvider;
import org.apache.ignite.spi.encryption.EncryptionSpi;
/**
@@ -51,9 +51,9 @@ public class EncryptedFileIOFactory implements FileIOFactory {
private int groupId;
/**
- * Encryption manager.
+ * Encryption keys provider.
*/
- private GridEncryptionManager encMgr;
+ private EncryptionCacheKeyProvider encrKeyProvider;
/**
* Encryption spi.
@@ -64,14 +64,15 @@ public class EncryptedFileIOFactory implements FileIOFactory {
* @param plainIOFactory Underlying file factory.
* @param groupId Group id.
* @param pageSize Size of plain data page in bytes.
- * @param encMgr Encryption manager.
+ * @param encrKeyProvider Encryption keys provider.
*/
- EncryptedFileIOFactory(FileIOFactory plainIOFactory, int groupId, int pageSize, GridEncryptionManager encMgr,
+ EncryptedFileIOFactory(FileIOFactory plainIOFactory, int groupId, int pageSize,
+ EncryptionCacheKeyProvider encrKeyProvider,
EncryptionSpi encSpi) {
this.plainIOFactory = plainIOFactory;
this.groupId = groupId;
this.pageSize = pageSize;
- this.encMgr = encMgr;
+ this.encrKeyProvider = encrKeyProvider;
this.encSpi = encSpi;
}
@@ -79,7 +80,7 @@ public class EncryptedFileIOFactory implements FileIOFactory {
@Override public FileIO create(File file, OpenOption... modes) throws IOException {
FileIO io = plainIOFactory.create(file, modes);
- return new EncryptedFileIO(io, groupId, pageSize, headerSize, encMgr, encSpi);
+ return new EncryptedFileIO(io, groupId, pageSize, headerSize, encrKeyProvider, encSpi);
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index fb60b3e..f749bf4 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -679,19 +679,8 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
FileIOFactory pageStoreV1FileIoFactory = this.pageStoreV1FileIoFactory;
if (encrypted) {
- pageStoreFileIoFactory = new EncryptedFileIOFactory(
- this.pageStoreFileIoFactory,
- grpId,
- pageSize(),
- cctx.kernalContext().encryption(),
- cctx.gridConfig().getEncryptionSpi());
-
- pageStoreV1FileIoFactory = new EncryptedFileIOFactory(
- this.pageStoreV1FileIoFactory,
- grpId,
- pageSize(),
- cctx.kernalContext().encryption(),
- cctx.gridConfig().getEncryptionSpi());
+ pageStoreFileIoFactory = encryptedFileIoFactory(this.pageStoreFileIoFactory, grpId);
+ pageStoreV1FileIoFactory = encryptedFileIoFactory(this.pageStoreV1FileIoFactory, grpId);
}
FileVersionCheckingFactory pageStoreFactory = new FileVersionCheckingFactory(
@@ -711,6 +700,20 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
}
/**
+ * @param plainFileIOFactory Not-encrypting file io factory.
+ * @param cacheGrpId Cache group id.
+ * @return Encrypted file IO factory.
+ */
+ public EncryptedFileIOFactory encryptedFileIoFactory(FileIOFactory plainFileIOFactory, int cacheGrpId) {
+ return new EncryptedFileIOFactory(
+ plainFileIOFactory,
+ cacheGrpId,
+ pageSize(),
+ cctx.kernalContext().encryption(),
+ cctx.gridConfig().getEncryptionSpi());
+ }
+
+ /**
* @param cacheWorkDir Work directory.
* @param grpId Group ID.
* @param partitions Number of partitions.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
index 6c7aa2f..7fb70ca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
@@ -57,7 +57,6 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
-import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -95,7 +94,6 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
-import org.apache.ignite.internal.processors.cache.persistence.file.FileVersionCheckingFactory;
import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
@@ -292,8 +290,8 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
/** Factory to working with delta as file storage. */
private volatile FileIOFactory ioFactory = new RandomAccessFileIOFactory();
- /** Factory to create page store for restore. */
- private volatile BiFunction<Integer, Boolean, FileVersionCheckingFactory> storeFactory;
+ /** File store manager to create page store for restore. */
+ private volatile FilePageStoreManager storeMgr;
/** Snapshot thread pool to perform local partition snapshots. */
private ExecutorService snpRunner;
@@ -377,7 +375,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
assert cctx.pageStore() instanceof FilePageStoreManager;
- FilePageStoreManager storeMgr = (FilePageStoreManager)cctx.pageStore();
+ storeMgr = (FilePageStoreManager)cctx.pageStore();
pdsSettings = cctx.kernalContext().pdsFolderResolver().resolveFolders();
@@ -406,8 +404,6 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
"The list of names of all snapshots currently saved on the local node with respect to " +
"the configured via IgniteConfiguration snapshot working path.");
- storeFactory = storeMgr::getPageStoreFactory;
-
cctx.exchange().registerExchangeAwareComponent(this);
ctx.internalSubscriptionProcessor().registerMetastorageListener(this);
@@ -611,6 +607,16 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
"prior to snapshot operation start: " + leftNodes));
}
+ if (!cctx.localNode().isClient() && cctx.kernalContext().encryption().isMasterKeyChangeInProgress()) {
+ return new GridFinishedFuture<>(new IgniteCheckedException("Snapshot operation has been rejected. Master " +
+ "key changing process is not finished yet."));
+ }
+
+ if (!cctx.localNode().isClient() && cctx.kernalContext().encryption().reencryptionInProgress()) {
+ return new GridFinishedFuture<>(new IgniteCheckedException("Snapshot operation has been rejected. Caches " +
+ "re-encryption process is not finished yet."));
+ }
+
List<Integer> grpIds = new ArrayList<>(F.viewReadOnly(req.groups(), CU::cacheId));
Set<Integer> leftGrps = new HashSet<>(grpIds);
@@ -1306,7 +1312,6 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
List<String> grps = cctx.cache().persistentGroups().stream()
.filter(g -> cctx.cache().cacheType(g.cacheOrGroupName()) == CacheType.USER)
- .filter(g -> !g.config().isEncryptionEnabled())
.map(CacheGroupDescriptor::cacheOrGroupName)
.collect(Collectors.toList());
@@ -1534,12 +1539,10 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
File snpPart = getPartitionFile(new File(snapshotLocalDir(snpName), databaseRelativePath(folderName)),
grps.get(0).getName(), partId);
- FilePageStore pageStore = (FilePageStore)storeFactory
- .apply(CU.cacheId(grpName), false)
- .createPageStore(getTypeByPartId(partId),
- snpPart::toPath,
- val -> {
- });
+ int grpId = CU.cacheId(grpName);
+
+ FilePageStore pageStore = (FilePageStore)storeMgr.getPageStoreFactory(grpId, cctx.cache().isEncrypted(grpId)).
+ createPageStore(getTypeByPartId(partId), snpPart::toPath, val -> {});
GridCloseableIterator<CacheDataRow> partIter = partitionRowIterator(ctx, grpName, partId, pageStore);
@@ -1601,6 +1604,19 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
if (prev != null)
return new SnapshotFutureTask(new IgniteCheckedException("Snapshot with requested name is already scheduled: " + snpName));
+ if (!withMetaStorage) {
+ for (Integer grpId : parts.keySet()) {
+ if (!cctx.cache().isEncrypted(grpId))
+ continue;
+
+ snpFutTask.onDone(new IgniteCheckedException("Snapshot contains encrypted cache group " + grpId +
+ " but doesn't include metastore. Metastore is required because it contains encryption keys " +
+ "required to start with encrypted caches contained in the snapshot."));
+
+ return snpFutTask;
+ }
+ }
+
if (log.isInfoEnabled()) {
log.info("Snapshot task has been registered on local node [sctx=" + this +
", topVer=" + cctx.discovery().topologyVersionEx() + ']');
@@ -2223,12 +2239,15 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter
", delta=" + delta + ']');
}
+ boolean encrypted = cctx.cache().isEncrypted(pair.getGroupId());
+
+ FileIOFactory ioFactory = encrypted ? ((FilePageStoreManager)cctx.pageStore())
+ .encryptedFileIoFactory(IgniteSnapshotManager.this.ioFactory, pair.getGroupId()) :
+ IgniteSnapshotManager.this.ioFactory;
+
try (FileIO fileIo = ioFactory.create(delta, READ);
- FilePageStore pageStore = (FilePageStore)storeFactory
- .apply(pair.getGroupId(), false)
- .createPageStore(getTypeByPartId(pair.getPartitionId()),
- snpPart::toPath,
- val -> {})
+ FilePageStore pageStore = (FilePageStore)storeMgr.getPageStoreFactory(pair.getGroupId(), encrypted)
+ .createPageStore(getTypeByPartId(pair.getPartitionId()), snpPart::toPath, v -> {})
) {
ByteBuffer pageBuf = ByteBuffer.allocate(pageSize)
.order(ByteOrder.nativeOrder());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFutureTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFutureTask.java
index 1afecf6..2fefe35 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFutureTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFutureTask.java
@@ -344,9 +344,6 @@ class SnapshotFutureTask extends GridFutureAdapter<Set<GroupPartitionId>> implem
if (!CU.isPersistentCache(gctx.config(), cctx.kernalContext().config().getDataStorageConfiguration()))
throw new IgniteCheckedException("In-memory cache groups are not allowed to be snapshot: " + grpId);
- if (gctx.config().isEncryptionEnabled())
- throw new IgniteCheckedException("Encrypted cache groups are not allowed to be snapshot: " + grpId);
-
// Create cache group snapshot directory on start in a single thread.
U.ensureDirectory(cacheWorkDir(tmpConsIdDir, FilePageStoreManager.cacheDirName(gctx.config())),
"directory for snapshotting cache group",
@@ -625,14 +622,15 @@ class SnapshotFutureTask extends GridFutureAdapter<Set<GroupPartitionId>> implem
* @throws IgniteCheckedException If fails.
*/
private void addPartitionWriters(int grpId, Set<Integer> parts, String dirName) throws IgniteCheckedException {
+ Integer encGrpId = cctx.cache().isEncrypted(grpId) ? grpId : null;
+
for (int partId : parts) {
GroupPartitionId pair = new GroupPartitionId(grpId, partId);
PageStore store = pageStore.getStore(grpId, partId);
partDeltaWriters.put(pair,
- new PageStoreSerialWriter(store,
- partDeltaFile(cacheWorkDir(tmpConsIdDir, dirName), partId)));
+ new PageStoreSerialWriter(store, partDeltaFile(cacheWorkDir(tmpConsIdDir, dirName), partId), encGrpId));
partFileLengths.put(pair, store.size());
}
@@ -852,6 +850,9 @@ class SnapshotFutureTask extends GridFutureAdapter<Set<GroupPartitionId>> implem
/** Partition delta file to store delta pages into. */
private final File deltaFile;
+ /** Id of encrypted cache group. If {@code null}, no encrypted IO is used. */
+ private final Integer encryptedGrpId;
+
/** Busy lock to protect write operations. */
private final ReadWriteLock lock = new ReentrantReadWriteLock();
@@ -876,8 +877,9 @@ class SnapshotFutureTask extends GridFutureAdapter<Set<GroupPartitionId>> implem
/**
* @param store Partition page store.
* @param deltaFile Destination file to write pages to.
+ * @param encryptedGrpId Id of encrypted cache group. If {@code null}, no encrypted IO is used.
*/
- public PageStoreSerialWriter(PageStore store, File deltaFile) {
+ public PageStoreSerialWriter(PageStore store, File deltaFile, @Nullable Integer encryptedGrpId) {
assert store != null;
assert cctx.database().checkpointLockIsHeldByThread();
@@ -887,6 +889,7 @@ class SnapshotFutureTask extends GridFutureAdapter<Set<GroupPartitionId>> implem
// This guarantee us that no pages will be modified and it's safe to init pages
// list which needs to be processed.
writtenPages = new AtomicBitSet(store.pages());
+ this.encryptedGrpId = encryptedGrpId;
store.addWriteListener(this);
}
@@ -924,8 +927,10 @@ class SnapshotFutureTask extends GridFutureAdapter<Set<GroupPartitionId>> implem
if (stopped())
return;
- if (deltaFileIo == null)
- deltaFileIo = ioFactory.create(deltaFile);
+ if (deltaFileIo == null) {
+ deltaFileIo = (encryptedGrpId == null ? ioFactory :
+ pageStore.encryptedFileIoFactory(ioFactory, encryptedGrpId)).create(deltaFile);
+ }
}
catch (IOException e) {
acceptException(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
index 8589748..99a730a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
@@ -511,6 +511,16 @@ public class SnapshotRestoreProcess {
if (ctx.cache().context().snapshotMgr().isSnapshotCreating())
throw new IgniteCheckedException(OP_REJECT_MSG + "A cluster snapshot operation is in progress.");
+ if (ctx.encryption().isMasterKeyChangeInProgress()) {
+ return new GridFinishedFuture<>(new IgniteCheckedException(OP_REJECT_MSG + "Master key changing " +
+ "process is not finished yet."));
+ }
+
+ if (ctx.encryption().reencryptionInProgress()) {
+ return new GridFinishedFuture<>(new IgniteCheckedException(OP_REJECT_MSG + "Caches re-encryption " +
+ "process is not finished yet."));
+ }
+
for (UUID nodeId : req.nodes()) {
ClusterNode node = ctx.discovery().node(nodeId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java
index 6621b80..6c85440 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java
@@ -157,6 +157,8 @@ public class ClientListenerProcessor extends GridProcessorAdapter {
int selectorCnt = cliConnCfg.getSelectorCount();
+ ctx.metric().registry(CLIENT_CONNECTOR_METRICS);
+
for (int port = cliConnCfg.getPort(); port <= portTo && port <= 65535; port++) {
try {
srv = GridNioServer.<ClientMessage>builder()
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java
index 0175513..93ef0e2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java
@@ -56,15 +56,20 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.encryption.AbstractEncryptionTest;
import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
import org.apache.ignite.internal.processors.marshaller.MappedName;
+import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -74,11 +79,14 @@ import org.apache.ignite.lang.IgniteFutureCancelledException;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionSpi;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import static java.nio.file.Files.newDirectoryStream;
import static org.apache.ignite.cluster.ClusterState.ACTIVE;
@@ -87,6 +95,7 @@ import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE
import static org.apache.ignite.events.EventType.EVTS_CLUSTER_SNAPSHOT;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.FILE_SUFFIX;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_PREFIX;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.CP_SNAPSHOT_REASON;
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.DFLT_SNAPSHOT_TMP_DIR;
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.resolveSnapshotWorkDirectory;
import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
@@ -95,6 +104,7 @@ import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
/**
* Base snapshot tests.
*/
+@RunWith(Parameterized.class)
public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
/** Default snapshot name. */
protected static final String SNAPSHOT_NAME = "testSnapshot";
@@ -114,6 +124,29 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
/** Enable default data region persistence. */
protected boolean persistence = true;
+ /** Master key name. */
+ protected String masterKeyName;
+
+ /** Cache value builder. */
+ protected Function<Integer, Object> valBuilder = String::valueOf;
+
+ /**
+ * @return Cache value builder.
+ */
+ protected Function<Integer, Object> valueBuilder() {
+ return valBuilder;
+ }
+
+ /** Enable encryption of all caches in {@code IgniteConfiguration} before start. */
+ @Parameterized.Parameter
+ public boolean encryption;
+
+ /** Parameters. */
+ @Parameterized.Parameters(name = "Encryption={0}")
+ public static Iterable<Boolean> encryptionParams() {
+ return Arrays.asList(false, true);
+ }
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -138,6 +171,30 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
.setDiscoverySpi(discoSpi);
}
+ /** {@inheritDoc} */
+ @Override protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg,
+ GridSpringResourceContext ctx) throws Exception {
+
+ if (encryption && persistence) {
+ KeystoreEncryptionSpi encSpi = new KeystoreEncryptionSpi();
+
+ encSpi.setKeyStorePath(AbstractEncryptionTest.KEYSTORE_PATH);
+ encSpi.setKeyStorePassword(AbstractEncryptionTest.KEYSTORE_PASSWORD.toCharArray());
+
+ if (masterKeyName != null)
+ encSpi.setMasterKeyName(masterKeyName);
+
+ cfg.setEncryptionSpi(encSpi);
+
+ if (cfg.getCacheConfiguration() != null) {
+ for (CacheConfiguration<?, ?> cacheCfg : cfg.getCacheConfiguration())
+ cacheCfg.setEncryptionEnabled(true);
+ }
+ }
+
+ return super.startGrid(igniteInstanceName, cfg, ctx);
+ }
+
/** @throws Exception If fails. */
@Before
public void beforeTestSnapshot() throws Exception {
@@ -182,6 +239,38 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
}
/**
+ * @param ccfg Cache configuration.
+ * @throws IgniteCheckedException if failed.
+ */
+ protected void ensureCacheAbsent(CacheConfiguration<?, ?> ccfg) throws IgniteCheckedException {
+ String cacheName = ccfg.getName();
+
+ for (Ignite ignite : G.allGrids()) {
+ GridKernalContext kctx = ((IgniteEx)ignite).context();
+
+ if (kctx.clientNode())
+ continue;
+
+ CacheGroupDescriptor desc = kctx.cache().cacheGroupDescriptors().get(CU.cacheId(cacheName));
+
+ assertNull("nodeId=" + kctx.localNodeId() + ", cache=" + cacheName, desc);
+
+ boolean success = GridTestUtils.waitForCondition(
+ () -> !kctx.cache().context().snapshotMgr().isRestoring(),
+ TIMEOUT);
+
+ assertTrue("The process has not finished on the node " + kctx.localNodeId(), success);
+
+ File dir = ((FilePageStoreManager)kctx.cache().context().pageStore()).cacheWorkDir(ccfg);
+
+ String errMsg = String.format("%s, dir=%s, exists=%b, files=%s",
+ ignite.name(), dir, dir.exists(), Arrays.toString(dir.list()));
+
+ assertTrue(errMsg, !dir.exists() || dir.list().length == 0);
+ }
+ }
+
+ /**
* @param ccfg Default cache configuration.
* @return Cache configuration.
*/
@@ -367,6 +456,28 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
}
/**
+ * @param nodesCnt Nodes count.
+ * @param keysCnt Number of keys to create.
+ * @param startClient {@code True} to start an additional client node.
+ * @return Ignite coordinator instance.
+ * @throws Exception if failed.
+ */
+ protected IgniteEx startGridsWithSnapshot(int nodesCnt, int keysCnt, boolean startClient) throws Exception {
+ IgniteEx ignite = startGridsWithCache(nodesCnt, keysCnt, valueBuilder(), dfltCacheCfg);
+
+ if (startClient)
+ ignite = startClientGrid("client");
+
+ ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ ignite.cache(dfltCacheCfg.getName()).destroy();
+
+ awaitPartitionMapExchange();
+
+ return ignite;
+ }
+
+ /**
* @param ignite Ignite instance.
* @return Snapshot manager related to given ignite instance.
*/
@@ -397,6 +508,43 @@ public abstract class AbstractSnapshotSelfTest extends GridCommonAbstractTest {
}
/**
+ * @param cache Cache.
+ * @param keysCnt Expected number of keys.
+ */
+ protected void assertCacheKeys(IgniteCache<Object, Object> cache, int keysCnt) {
+ assertEquals(keysCnt, cache.size());
+
+ for (int i = 0; i < keysCnt; i++)
+ assertEquals(valueBuilder().apply(i), cache.get(i));
+ }
+
+ /**
+ * @param snpName Unique snapshot name.
+ * @param parts Collection of pairs group and appropriate cache partition to be snapshot.
+ * @param snpSndr Sender which used for snapshot sub-task processing.
+ * @return Future which will be completed when snapshot is done.
+ */
+ protected SnapshotFutureTask startLocalSnapshotTask(
+ GridCacheSharedContext<?, ?> cctx,
+ String snpName,
+ Map<Integer, Set<Integer>> parts,
+ SnapshotSender snpSndr
+ ) throws IgniteCheckedException {
+ SnapshotFutureTask snpFutTask = cctx.snapshotMgr().registerSnapshotTask(snpName, cctx.localNodeId(), parts, encryption, snpSndr);
+
+ snpFutTask.start();
+
+ // Snapshot is still in the INIT state. beforeCheckpoint has been skipped
+ // due to checkpoint already running and we need to schedule the next one
+ // right after current will be completed.
+ cctx.database().forceCheckpoint(String.format(CP_SNAPSHOT_REASON, snpName));
+
+ snpFutTask.started().get();
+
+ return snpFutTask;
+ }
+
+ /**
* @param grids Grids to block snapshot executors.
* @return Wrapped snapshot executor list.
*/
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java
new file mode 100644
index 0000000..cada253
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import java.util.Collections;
+import java.util.function.Function;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.encryption.AbstractEncryptionTest;
+import org.apache.ignite.internal.util.distributed.FullMessage;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.Test;
+import org.junit.runners.Parameterized;
+
+/**
+ * Snapshot test for encrypted snapshots.
+ */
+public class EncryptedSnapshotTest extends AbstractSnapshotSelfTest {
+ /** Second cache name. */
+ private static final String CACHE2 = "cache2";
+
+ /** Parameters. */
+ @Parameterized.Parameters(name = "Encryption is enabled.")
+ public static Iterable<Boolean> enableEncryption() {
+ return Collections.singletonList(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Function<Integer, Object> valueBuilder() {
+ return (i -> new Account(i, i));
+ }
+
+ /** Checks creation of encrypted cache with same name after putting plain cache in snapshot. */
+ @Test
+ public void testEncryptedCacheCreatedAfterPlainCacheSnapshotting() throws Exception {
+ testCacheCreatedAfterSnapshotting(true);
+ }
+
+ /** Checks creation of plain cache with same name after putting encrypted cache in snapshot. */
+ @Test
+ public void testPlainCacheCreatedAfterEncryptedCacheSnapshotting() throws Exception {
+ testCacheCreatedAfterSnapshotting(false);
+ }
+
+ /** Checks re-encryption fails during snapshot restoration. */
+ @Test
+ public void testReencryptDuringRestore() throws Exception {
+ checkActionFailsDuringSnapshotOperation(true, this::chageCacheKey, "Cache group key change was rejected.",
+ IgniteException.class);
+ }
+
+ /** Checks master key changing fails during snapshot restoration. */
+ @Test
+ public void testMasterKeyChangeDuringRestore() throws Exception {
+ checkActionFailsDuringSnapshotOperation(true, this::chageMasterKey, "Master key change was rejected.",
+ IgniteException.class);
+ }
+
+ /** Checks re-encryption fails during snapshot creation. */
+ @Test
+ public void testReencryptDuringSnapshot() throws Exception {
+ checkActionFailsDuringSnapshotOperation(false, this::chageCacheKey, "Cache group key change was rejected.",
+ IgniteException.class);
+ }
+
+ /** Checks master key changing fails during snapshot creation. */
+ @Test
+ public void testMasterKeyChangeDuringSnapshot() throws Exception {
+ checkActionFailsDuringSnapshotOperation(false, this::chageMasterKey, "Master key change was rejected.",
+ IgniteException.class);
+ }
+
+ /** Checks snapshot action fail during cache group key change. */
+ @Test
+ public void testSnapshotFailsDuringCacheKeyChange() throws Exception {
+ checkSnapshotActionFailsDuringReencryption(this::chageCacheKey, "Caches re-encryption process is not " +
+ "finished yet");
+ }
+
+ /** Checks snapshot action fail during master key change. */
+ @Test
+ public void testSnapshotFailsDuringMasterKeyChange() throws Exception {
+ checkSnapshotActionFailsDuringReencryption(this::chageMasterKey, "Master key changing process is not " +
+ "finished yet.");
+ }
+
+ /** Checks snapshot restoration fails if different master key is contained in the snapshot. */
+ @Test
+ public void testStartFromSnapshotFailedWithOtherMasterKey() throws Exception {
+ IgniteEx ig = startGridsWithCache(1, CACHE_KEYS_RANGE, valueBuilder(), dfltCacheCfg);
+
+ ig.snapshot().createSnapshot(SNAPSHOT_NAME).get();
+
+ ig.destroyCache(dfltCacheCfg.getName());
+
+ ensureCacheAbsent(dfltCacheCfg);
+
+ stopAllGrids(false);
+
+ masterKeyName = AbstractEncryptionTest.MASTER_KEY_NAME_2;
+
+ GridTestUtils.assertThrowsAnyCause(
+ log,
+ () -> startGridsFromSnapshot(1, SNAPSHOT_NAME),
+ IgniteSpiException.class,
+ "bad key is used during decryption"
+ );
+ }
+
+ /** Checks it is unavailable to register snapshot task for encrypted caches without metastore. */
+ @Test
+ public void testSnapshotTaskIsBlockedWithoutMetastore() throws Exception {
+ // Start grid node with data before each test.
+ IgniteEx ig = startGridsWithCache(1, CACHE_KEYS_RANGE, valueBuilder(), dfltCacheCfg);
+
+ GridTestUtils.assertThrowsAnyCause(log,
+ () -> snp(ig).registerSnapshotTask(SNAPSHOT_NAME, ig.localNode().id(),
+ F.asMap(CU.cacheId(dfltCacheCfg.getName()), null), false,
+ snp(ig).localSnapshotSenderFactory().apply(SNAPSHOT_NAME)).get(TIMEOUT),
+ IgniteCheckedException.class,
+ "Metastore is required because it contains encryption keys");
+ }
+
+ /**
+ * Ensures that same-name-cache is created after putting cache into snapshot and deleting.
+ *
+ * @param encryptedFirst If {@code true}, creates encrypted cache before snapshoting and deleting. In reverse
+ * order if {@code false}.
+ */
+ private void testCacheCreatedAfterSnapshotting(boolean encryptedFirst) throws Exception {
+ CacheConfiguration<Integer, Object> ccfg = new CacheConfiguration<>(dfltCacheCfg).setName(CACHE2);
+
+ if (encryptedFirst)
+ ccfg.setEncryptionEnabled(true);
+
+ startGridsWithCache(3, ccfg, CACHE_KEYS_RANGE);
+
+ grid(1).snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ grid(2).destroyCache(CACHE2);
+
+ awaitPartitionMapExchange();
+
+ addCache(!encryptedFirst);
+ }
+
+ /**
+ * Checks {@code action} is blocked with {@code errPrefix} and {@code errEncrypType} during active snapshot.
+ *
+ * @param restore If {@code true}, snapshot restoration is activated during the test. Snapshot creation otherwise.
+ * @param action Action to call during snapshot operation. Its param is the grid num.
+ * @param errPrefix Prefix of error message text to search for.
+ * @param errType Type of exception to search for.
+ */
+ private void checkActionFailsDuringSnapshotOperation(boolean restore, Function<Integer, IgniteFuture<?>> action,
+ String errPrefix, Class<? extends Exception> errType) throws Exception {
+ startGridsWithCache(3, CACHE_KEYS_RANGE, valueBuilder(), dfltCacheCfg);
+
+ BlockingCustomMessageDiscoverySpi spi0 = discoSpi(grid(0));
+
+ IgniteFuture<Void> fut;
+
+ if (restore) {
+ CacheConfiguration<?, ?> notEncrCacheCfg = addCache(false);
+
+ grid(1).snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ grid(2).cache(notEncrCacheCfg.getName()).destroy();
+
+ awaitPartitionMapExchange();
+
+ ensureCacheAbsent(notEncrCacheCfg);
+
+ spi0.block((msg) -> msg instanceof FullMessage && ((FullMessage<?>)msg).error().isEmpty());
+
+ fut = grid(1).snapshot().restoreSnapshot(SNAPSHOT_NAME,
+ Collections.singletonList(notEncrCacheCfg.getName()));
+ }
+ else {
+ spi0.block((msg) -> msg instanceof FullMessage && ((FullMessage<?>)msg).error().isEmpty());
+
+ fut = grid(1).snapshot().createSnapshot(SNAPSHOT_NAME);
+ }
+
+ spi0.waitBlocked(TIMEOUT);
+
+ GridTestUtils.assertThrowsAnyCause(log, () -> action.apply(2).get(TIMEOUT), errType,
+ errPrefix + " Snapshot operation is in progress.");
+
+ spi0.unblock();
+
+ fut.get(TIMEOUT);
+ }
+
+ /**
+ * Checks snapshot action is blocked during {@code reencryption}.
+ *
+ * @param reencryption Any kind of re-encryption action.
+ * @param expectedError Expected error text.
+ */
+ private void checkSnapshotActionFailsDuringReencryption(Function<Integer, IgniteFuture<?>> reencryption,
+ String expectedError) throws Exception {
+ startGridsWithCache(3, CACHE_KEYS_RANGE, valueBuilder(), dfltCacheCfg);
+
+ CacheConfiguration<?, ?> notEncrCacheCfg = addCache(false);
+
+ grid(1).snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
+
+ grid(2).destroyCache(notEncrCacheCfg.getName());
+
+ awaitPartitionMapExchange();
+
+ ensureCacheAbsent(notEncrCacheCfg);
+
+ BlockingCustomMessageDiscoverySpi discoSpi = discoSpi(grid(0));
+
+ discoSpi.block(msg -> msg instanceof FullMessage && ((FullMessage<?>)msg).error().isEmpty());
+
+ IgniteFuture<?> fut = reencryption.apply(1);
+
+ discoSpi.waitBlocked(TIMEOUT);
+
+ GridTestUtils.assertThrowsAnyCause(log,
+ () -> grid(1).snapshot().restoreSnapshot(SNAPSHOT_NAME, Collections.singletonList(CACHE2)).get(TIMEOUT),
+ IgniteCheckedException.class,
+ expectedError);
+
+ GridTestUtils.assertThrowsAnyCause(log,
+ () -> grid(2).snapshot().createSnapshot(SNAPSHOT_NAME + "_v2").get(TIMEOUT), IgniteCheckedException.class,
+ expectedError);
+
+ discoSpi.unblock();
+
+ fut.get(TIMEOUT);
+ }
+
+ /**
+ * Adds cache to the grid. Fills it and waits for PME.
+ *
+ * @param encrypted If {@code true}, created encrypted cache.
+ * @return CacheConfiguration of the created cache.
+ */
+ private CacheConfiguration<?, ?> addCache(boolean encrypted) throws InterruptedException {
+ CacheConfiguration<?, ?> cacheCfg = new CacheConfiguration<>(dfltCacheCfg).setName(CACHE2).
+ setEncryptionEnabled(encrypted);
+
+ grid(0).createCache(cacheCfg);
+
+ Function<Integer, Object> valBuilder = valueBuilder();
+
+ IgniteDataStreamer<Integer, Object> streamer = grid(0).dataStreamer(CACHE2);
+
+ for (int i = 0; i < CACHE_KEYS_RANGE; i++)
+ streamer.addData(i, valBuilder.apply(i));
+
+ streamer.flush();
+
+ awaitPartitionMapExchange();
+
+ return cacheCfg;
+ }
+
+ /**
+ * @return Cache group key change action.
+ */
+ private IgniteFuture<?> chageCacheKey(int gridNum) {
+ return grid(gridNum).encryption().changeCacheGroupKey(Collections.singletonList(dfltCacheCfg.getName()));
+ }
+
+ /**
+ * @return Master key change action.
+ */
+ private IgniteFuture<?> chageMasterKey(int gridNum) {
+ return grid(gridNum).encryption().changeMasterKey(AbstractEncryptionTest.MASTER_KEY_NAME_2);
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java
index 966ac89..d1b3fa89 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java
@@ -42,9 +42,11 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.compute.ComputeJobResult;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.GridJobExecuteRequest;
import org.apache.ignite.internal.GridTopic;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.binary.BinaryContext;
import org.apache.ignite.internal.binary.BinaryObjectImpl;
import org.apache.ignite.internal.managers.communication.GridMessageListener;
@@ -59,6 +61,7 @@ import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabase
import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
@@ -78,12 +81,15 @@ import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
import org.jetbrains.annotations.Nullable;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runners.Parameterized;
import static java.util.Collections.singletonList;
import static org.apache.ignite.cluster.ClusterState.ACTIVE;
import static org.apache.ignite.configuration.IgniteConfiguration.DFLT_SNAPSHOT_DIRECTORY;
+import static org.apache.ignite.internal.MarshallerContextImpl.mappingFileStoreWorkDir;
import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
import static org.apache.ignite.internal.processors.cache.GridCacheUtils.TTL_ETERNAL;
+import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.binaryWorkDir;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheDirName;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.getPartitionFileName;
import static org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId.getTypeByPartId;
@@ -107,6 +113,12 @@ public class IgniteClusterSnapshotCheckTest extends AbstractSnapshotSelfTest {
/** Optional cache name to be created on demand. */
private static final String OPTIONAL_CACHE_NAME = "CacheName";
+ /** Parameters. Encryption is not supported by snapshot validation. */
+ @Parameterized.Parameters(name = "Encryption is disabled")
+ public static Iterable<Boolean> disabledEncryption() {
+ return Collections.singletonList(false);
+ }
+
/** Cleanup data of task execution results if need. */
@Before
public void beforeCheck() {
@@ -119,7 +131,7 @@ public class IgniteClusterSnapshotCheckTest extends AbstractSnapshotSelfTest {
IgniteEx ignite = startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE);
startClientGrid();
-
+
ignite.snapshot().createSnapshot(SNAPSHOT_NAME)
.get();
@@ -559,6 +571,77 @@ public class IgniteClusterSnapshotCheckTest extends AbstractSnapshotSelfTest {
assertTrue("Threads created: " + createdThreads, createdThreads < iterations);
}
+ /** Checks bytes, signature of partion files. Compares before and after snapshot. Assumes the files are equal. */
+ @Test
+ public void testSnapshotLocalPartitions() throws Exception {
+ // Note: this test is valid only for not-encrypted snapshots. Writing snapshots deltas causes several page writes into file.
+ // Every page write calls encrypt(). Every repeatable encrypt() produces different record (different bytes) even for same original
+ // data. Re-writting pages from delta to partition file in the shanpshot leads to additional encryption before writing to the
+ // snapshot partition file. Thus, page in original partition and in snapshot partiton has different encrypted CRC and same
+ // de-crypted CRC. Different encrypted CRC looks like different data in point of view of third-party observer.
+
+ IgniteEx ig = startGridsWithCache(1, 4096, key -> new Account(key, key),
+ new CacheConfiguration<>(DEFAULT_CACHE_NAME));
+
+ for (int i = 4096; i < 8192; i++) {
+ ig.cache(DEFAULT_CACHE_NAME).put(i, new Account(i, i) {
+ @Override public String toString() {
+ return "_" + super.toString();
+ }
+ });
+ }
+
+ GridCacheSharedContext<?, ?> cctx = ig.context().cache().context();
+ IgniteSnapshotManager mgr = snp(ig);
+
+ // Collection of pairs group and appropriate cache partition to be snapshot.
+ IgniteInternalFuture<?> snpFut = startLocalSnapshotTask(cctx,
+ SNAPSHOT_NAME,
+ F.asMap(CU.cacheId(DEFAULT_CACHE_NAME), null),
+ mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME));
+
+ snpFut.get();
+
+ File cacheWorkDir = ((FilePageStoreManager)ig.context()
+ .cache()
+ .context()
+ .pageStore())
+ .cacheWorkDir(dfltCacheCfg);
+
+ // Checkpoint forces on cluster deactivation (currently only single node in cluster),
+ // so we must have the same data in snapshot partitions and those which left
+ // after node stop.
+ stopGrid(ig.name());
+
+ // Calculate CRCs.
+ IgniteConfiguration cfg = ig.context().config();
+ PdsFolderSettings settings = ig.context().pdsFolderResolver().resolveFolders();
+ String nodePath = databaseRelativePath(settings.folderName());
+ File binWorkDir = binaryWorkDir(cfg.getWorkDirectory(), settings.folderName());
+ File marshWorkDir = mappingFileStoreWorkDir(U.workDirectory(cfg.getWorkDirectory(), cfg.getIgniteHome()));
+ File snpBinWorkDir = binaryWorkDir(mgr.snapshotLocalDir(SNAPSHOT_NAME).getAbsolutePath(), settings.folderName());
+ File snpMarshWorkDir = mappingFileStoreWorkDir(mgr.snapshotLocalDir(SNAPSHOT_NAME).getAbsolutePath());
+
+ final Map<String, Integer> origPartCRCs = calculateCRC32Partitions(cacheWorkDir);
+ final Map<String, Integer> snpPartCRCs = calculateCRC32Partitions(
+ FilePageStoreManager.cacheWorkDir(U.resolveWorkDirectory(mgr.snapshotLocalDir(SNAPSHOT_NAME)
+ .getAbsolutePath(),
+ nodePath,
+ false),
+ cacheDirName(dfltCacheCfg)));
+
+ assertEquals("Partitions must have the same CRC after file copying and merging partition delta files",
+ origPartCRCs, snpPartCRCs);
+ assertEquals("Binary object mappings must be the same for local node and created snapshot",
+ calculateCRC32Partitions(binWorkDir), calculateCRC32Partitions(snpBinWorkDir));
+ assertEquals("Marshaller meta mast be the same for local node and created snapshot",
+ calculateCRC32Partitions(marshWorkDir), calculateCRC32Partitions(snpMarshWorkDir));
+
+ File snpWorkDir = mgr.snapshotTmpDir();
+
+ assertEquals("Snapshot working directory must be cleaned after usage", 0, snpWorkDir.listFiles().length);
+ }
+
/**
* @param cls Class of running task.
* @param results Results of compute.
@@ -621,8 +704,10 @@ public class IgniteClusterSnapshotCheckTest extends AbstractSnapshotSelfTest {
Path part0 = U.searchFileRecursively(cachePath, getPartitionFileName(partId));
+ int grpId = CU.cacheId(ccfg.getName());
+
try (FilePageStore pageStore = (FilePageStore)((FilePageStoreManager)ignite.context().cache().context().pageStore())
- .getPageStoreFactory(CU.cacheId(ccfg.getName()), false)
+ .getPageStoreFactory(grpId, ignite.context().cache().isEncrypted(grpId))
.createPageStore(getTypeByPartId(partId),
() -> part0,
val -> {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreBaseTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreBaseTest.java
index 9368ef3..579e9c3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreBaseTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreBaseTest.java
@@ -17,28 +17,21 @@
package org.apache.ignite.internal.processors.cache.persistence.snapshot;
-import java.io.File;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.function.Function;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.binary.BinaryObjectBuilder;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
-import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
-import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.internal.util.typedef.internal.CU;
-import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.runners.Parameterized;
/**
* Snapshot restore test base.
*/
public abstract class IgniteClusterSnapshotRestoreBaseTest extends AbstractSnapshotSelfTest {
- /** Cache value builder. */
- protected abstract Function<Integer, Object> valueBuilder();
+ /** Parameters. Encrypted snapshots are not supported. */
+ @Parameterized.Parameters(name = "Encryption is disabled")
+ public static Iterable<Boolean> disabledEncryption() {
+ return Collections.singletonList(false);
+ }
/**
* @param nodesCnt Nodes count.
@@ -50,71 +43,6 @@ public abstract class IgniteClusterSnapshotRestoreBaseTest extends AbstractSnaps
return startGridsWithSnapshot(nodesCnt, keysCnt, false);
}
- /**
- * @param nodesCnt Nodes count.
- * @param keysCnt Number of keys to create.
- * @param startClient {@code True} to start an additional client node.
- * @return Ignite coordinator instance.
- * @throws Exception if failed.
- */
- protected IgniteEx startGridsWithSnapshot(int nodesCnt, int keysCnt, boolean startClient) throws Exception {
- IgniteEx ignite = startGridsWithCache(nodesCnt, keysCnt, valueBuilder(), dfltCacheCfg);
-
- if (startClient)
- ignite = startClientGrid("client");
-
- ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
-
- ignite.cache(dfltCacheCfg.getName()).destroy();
-
- awaitPartitionMapExchange();
-
- return ignite;
- }
-
- /**
- * @param cache Cache.
- * @param keysCnt Expected number of keys.
- */
- protected void assertCacheKeys(IgniteCache<Object, Object> cache, int keysCnt) {
- assertEquals(keysCnt, cache.size());
-
- for (int i = 0; i < keysCnt; i++)
- assertEquals(valueBuilder().apply(i), cache.get(i));
- }
-
- /**
- * @param ccfg Cache configuration.
- * @throws IgniteCheckedException if failed.
- */
- protected void ensureCacheAbsent(CacheConfiguration<?, ?> ccfg) throws IgniteCheckedException {
- String cacheName = ccfg.getName();
-
- for (Ignite ignite : G.allGrids()) {
- GridKernalContext kctx = ((IgniteEx)ignite).context();
-
- if (kctx.clientNode())
- continue;
-
- CacheGroupDescriptor desc = kctx.cache().cacheGroupDescriptors().get(CU.cacheId(cacheName));
-
- assertNull("nodeId=" + kctx.localNodeId() + ", cache=" + cacheName, desc);
-
- boolean success = GridTestUtils.waitForCondition(
- () -> !kctx.cache().context().snapshotMgr().isRestoring(),
- TIMEOUT);
-
- assertTrue("The process has not finished on the node " + kctx.localNodeId(), success);
-
- File dir = ((FilePageStoreManager)kctx.cache().context().pageStore()).cacheWorkDir(ccfg);
-
- String errMsg = String.format("%s, dir=%s, exists=%b, files=%s",
- ignite.name(), dir, dir.exists(), Arrays.toString(dir.list()));
-
- assertTrue(errMsg, !dir.exists() || dir.list().length == 0);
- }
- }
-
/** */
protected class BinaryValueBuilder implements Function<Integer, Object> {
/** Binary type name. */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java
index 526093e..e6f93b5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java
@@ -95,9 +95,6 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR
/** Default shared cache group name. */
private static final String SHARED_GRP = "shared";
- /** Cache value builder. */
- private Function<Integer, Object> valBuilder = String::valueOf;
-
/** Reset consistent ID flag. */
private boolean resetConsistentId;
@@ -111,11 +108,6 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR
return cfg;
}
- /** {@inheritDoc} */
- @Override protected Function<Integer, Object> valueBuilder() {
- return valBuilder;
- }
-
/**
* Ensures that system partition verification task is invoked before restoring the snapshot.
*
@@ -161,7 +153,7 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR
CacheConfiguration<Integer, Object> cacheCfg2 =
txCacheConfig(new CacheConfiguration<Integer, Object>(CACHE2)).setGroupName(SHARED_GRP);
- IgniteEx ignite = startGridsWithCache(2, CACHE_KEYS_RANGE, valBuilder,
+ IgniteEx ignite = startGridsWithCache(2, CACHE_KEYS_RANGE, valueBuilder(),
dfltCacheCfg.setBackups(0), cacheCfg1, cacheCfg2);
ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
@@ -348,7 +340,7 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR
/** @throws Exception If failed. */
@Test
public void testClusterSnapshotRestoreRejectOnInActiveCluster() throws Exception {
- IgniteEx ignite = startGridsWithCache(2, CACHE_KEYS_RANGE, valBuilder, dfltCacheCfg);
+ IgniteEx ignite = startGridsWithCache(2, CACHE_KEYS_RANGE, valueBuilder(), dfltCacheCfg);
ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT);
@@ -391,7 +383,7 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR
CacheConfiguration<Integer, Object> cacheCfg2 =
txCacheConfig(new CacheConfiguration<Integer, Object>(CACHE2)).setGroupName(SHARED_GRP);
- IgniteEx ignite = startGridsWithCache(2, CACHE_KEYS_RANGE, valBuilder, cacheCfg1, cacheCfg2);
+ IgniteEx ignite = startGridsWithCache(2, CACHE_KEYS_RANGE, valueBuilder(), cacheCfg1, cacheCfg2);
ignite.cluster().state(ClusterState.ACTIVE);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java
index 28e1dea..366ec58 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java
@@ -329,6 +329,8 @@ public class IgniteClusterSnapshotSelfTest extends AbstractSnapshotSelfTest {
CacheConfiguration<Integer, Account> eastCcfg = txCacheConfig(new CacheConfiguration<>("east"));
CacheConfiguration<Integer, Account> westCcfg = txCacheConfig(new CacheConfiguration<>("west"));
+ dfltCacheCfg = null;
+
startGridsWithCache(grids, clientsCnt, key -> new Account(key, balance), eastCcfg, westCcfg);
Ignite client = startClientGrid(grids);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManagerSelfTest.java
index e31b175..55468f4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManagerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManagerSelfTest.java
@@ -51,10 +51,8 @@ import org.apache.ignite.internal.processors.cache.persistence.checkpoint.Checkp
import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
-import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.file.FileVersionCheckingFactory;
import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
-import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
import org.apache.ignite.internal.util.lang.GridCloseableIterator;
import org.apache.ignite.internal.util.typedef.F;
@@ -67,11 +65,7 @@ import org.apache.ignite.testframework.LogListener;
import org.junit.Test;
import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE;
-import static org.apache.ignite.internal.MarshallerContextImpl.mappingFileStoreWorkDir;
-import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.binaryWorkDir;
-import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheDirName;
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.CP_SNAPSHOT_REASON;
-import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.databaseRelativePath;
import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
import static org.apache.ignite.testframework.GridTestUtils.setFieldValue;
@@ -99,71 +93,6 @@ public class IgniteSnapshotManagerSelfTest extends AbstractSnapshotSelfTest {
return cfg;
}
- /** @throws Exception If fails. */
- @Test
- public void testSnapshotLocalPartitions() throws Exception {
- IgniteEx ig = startGridsWithCache(1, 4096, key -> new Account(key, key),
- new CacheConfiguration<>(DEFAULT_CACHE_NAME));
-
- for (int i = 4096; i < 8192; i++) {
- ig.cache(DEFAULT_CACHE_NAME).put(i, new Account(i, i) {
- @Override public String toString() {
- return "_" + super.toString();
- }
- });
- }
-
- GridCacheSharedContext<?, ?> cctx = ig.context().cache().context();
- IgniteSnapshotManager mgr = snp(ig);
-
- // Collection of pairs group and appropriate cache partition to be snapshot.
- IgniteInternalFuture<?> snpFut = startLocalSnapshotTask(cctx,
- SNAPSHOT_NAME,
- F.asMap(CU.cacheId(DEFAULT_CACHE_NAME), null),
- mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME));
-
- snpFut.get();
-
- File cacheWorkDir = ((FilePageStoreManager)ig.context()
- .cache()
- .context()
- .pageStore())
- .cacheWorkDir(dfltCacheCfg);
-
- // Checkpoint forces on cluster deactivation (currently only single node in cluster),
- // so we must have the same data in snapshot partitions and those which left
- // after node stop.
- stopGrid(ig.name());
-
- // Calculate CRCs.
- IgniteConfiguration cfg = ig.context().config();
- PdsFolderSettings settings = ig.context().pdsFolderResolver().resolveFolders();
- String nodePath = databaseRelativePath(settings.folderName());
- File binWorkDir = binaryWorkDir(cfg.getWorkDirectory(), settings.folderName());
- File marshWorkDir = mappingFileStoreWorkDir(U.workDirectory(cfg.getWorkDirectory(), cfg.getIgniteHome()));
- File snpBinWorkDir = binaryWorkDir(mgr.snapshotLocalDir(SNAPSHOT_NAME).getAbsolutePath(), settings.folderName());
- File snpMarshWorkDir = mappingFileStoreWorkDir(mgr.snapshotLocalDir(SNAPSHOT_NAME).getAbsolutePath());
-
- final Map<String, Integer> origPartCRCs = calculateCRC32Partitions(cacheWorkDir);
- final Map<String, Integer> snpPartCRCs = calculateCRC32Partitions(
- FilePageStoreManager.cacheWorkDir(U.resolveWorkDirectory(mgr.snapshotLocalDir(SNAPSHOT_NAME)
- .getAbsolutePath(),
- nodePath,
- false),
- cacheDirName(dfltCacheCfg)));
-
- assertEquals("Partitions must have the same CRC after file copying and merging partition delta files",
- origPartCRCs, snpPartCRCs);
- assertEquals("Binary object mappings must be the same for local node and created snapshot",
- calculateCRC32Partitions(binWorkDir), calculateCRC32Partitions(snpBinWorkDir));
- assertEquals("Marshaller meta mast be the same for local node and created snapshot",
- calculateCRC32Partitions(marshWorkDir), calculateCRC32Partitions(snpMarshWorkDir));
-
- File snpWorkDir = mgr.snapshotTmpDir();
-
- assertEquals("Snapshot working directory must be cleaned after usage", 0, snpWorkDir.listFiles().length);
- }
-
/**
* Test that all partitions are copied successfully even after multiple checkpoints occur during
* the long copy of cache partition files.
@@ -205,7 +134,7 @@ public class IgniteSnapshotManagerSelfTest extends AbstractSnapshotSelfTest {
SnapshotFutureTask snpFutTask = mgr.registerSnapshotTask(SNAPSHOT_NAME,
cctx.localNodeId(),
F.asMap(CU.cacheId(DEFAULT_CACHE_NAME), null),
- false,
+ encryption,
new DelegateSnapshotSender(log, mgr.snapshotExecutorService(), mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME)) {
@Override public void sendPart0(File part, String cacheDirName, GroupPartitionId pair, Long length) {
try {
@@ -615,32 +544,6 @@ public class IgniteSnapshotManagerSelfTest extends AbstractSnapshotSelfTest {
setFieldValue(snp(ignite), "storeFactory", factory);
}
- /**
- * @param snpName Unique snapshot name.
- * @param parts Collection of pairs group and appropriate cache partition to be snapshot.
- * @param snpSndr Sender which used for snapshot sub-task processing.
- * @return Future which will be completed when snapshot is done.
- */
- private static SnapshotFutureTask startLocalSnapshotTask(
- GridCacheSharedContext<?, ?> cctx,
- String snpName,
- Map<Integer, Set<Integer>> parts,
- SnapshotSender snpSndr
- ) throws IgniteCheckedException {
- SnapshotFutureTask snpFutTask = cctx.snapshotMgr().registerSnapshotTask(snpName, cctx.localNodeId(), parts, false, snpSndr);
-
- snpFutTask.start();
-
- // Snapshot is still in the INIT state. beforeCheckpoint has been skipped
- // due to checkpoint already running and we need to schedule the next one
- // right after current will be completed.
- cctx.database().forceCheckpoint(String.format(CP_SNAPSHOT_REASON, snpName));
-
- snpFutTask.started().get();
-
- return snpFutTask;
- }
-
/** */
private static class ZeroPartitionAffinityFunction extends RendezvousAffinityFunction {
/** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
index 2b8dfe1..402f2f7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
@@ -40,6 +40,7 @@ import org.apache.ignite.internal.encryption.MasterKeyChangeTest;
import org.apache.ignite.internal.processors.cache.persistence.CheckpointReadLockFailureTest;
import org.apache.ignite.internal.processors.cache.persistence.CommonPoolStarvationCheckpointTest;
import org.apache.ignite.internal.processors.cache.persistence.SingleNodePersistenceSslTest;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.EncryptedSnapshotTest;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotCheckTest;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotHandlerTest;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotRestoreSelfTest;
@@ -105,6 +106,7 @@ import org.junit.runners.Suite;
IgniteSnapshotMXBeanTest.class,
IgniteClusterSnapshotRestoreSelfTest.class,
IgniteClusterSnapshotHandlerTest.class,
+ EncryptedSnapshotTest.class,
IgniteClusterIdTagTest.class,
FullyConnectedComponentSearcherTest.class,
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckWithIndexesTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckWithIndexesTest.java
index cd2b0ca..b43fa72 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckWithIndexesTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckWithIndexesTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+import java.util.Collections;
import java.util.UUID;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheMode;
@@ -28,6 +29,7 @@ import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.lang.IgnitePredicate;
import org.junit.Test;
+import org.junit.runners.Parameterized;
import static java.util.Collections.singletonList;
import static org.apache.ignite.testframework.GridTestUtils.assertContains;
@@ -36,6 +38,12 @@ import static org.apache.ignite.testframework.GridTestUtils.assertContains;
* Cluster-wide snapshot test check command with indexes.
*/
public class IgniteClusterSnapshotCheckWithIndexesTest extends AbstractSnapshotSelfTest {
+ /** Parameters. Encryption is not supported by snapshot validation. */
+ @Parameterized.Parameters(name = "Encryption is disabled")
+ public static Iterable<Boolean> disabledEncryption() {
+ return Collections.singletonList(false);
+ }
+
/** @throws Exception If fails. */
@Test
public void testClusterSnapshotCheckEmptyCache() throws Exception {
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotWithIndexesTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotWithIndexesTest.java
index 89ec92c..3c7e144 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotWithIndexesTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotWithIndexesTest.java
@@ -34,6 +34,7 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.visor.verify.ValidateIndexesClosure;
import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.testframework.GridTestUtils;
import org.junit.Test;
import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_CHECKPOINT_FREQ;
@@ -156,6 +157,12 @@ public class IgniteClusterSnapshotWithIndexesTest extends AbstractSnapshotSelfTe
IgniteEx snp = startGridsFromSnapshot(grids, SNAPSHOT_NAME);
+ for (Ignite ig : G.allGrids()) {
+ GridTestUtils.waitForCondition(
+ () -> ((IgniteEx)ig).context().cache().publicCaches().stream().allMatch(c -> c.indexReadyFuture().isDone()),
+ TIMEOUT);
+ }
+
List<String> currIdxNames = executeSql(snp, "SELECT * FROM SYS.INDEXES").stream().
map(l -> (String)l.get(6))
.collect(Collectors.toList());