You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by al...@apache.org on 2020/10/27 12:06:05 UTC
[ignite] branch master updated: IGNITE-12843 TDE - Phase-3. Cache
key rotation - Fixes #7941.
This is an automated email from the ASF dual-hosted git repository.
alexpl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 6f3d633 IGNITE-12843 TDE - Phase-3. Cache key rotation - Fixes #7941.
6f3d633 is described below
commit 6f3d6336e1a1e068632cce05aa1973905d5873ce
Author: Pavel Pereslegin <xx...@gmail.com>
AuthorDate: Tue Oct 27 14:18:42 2020 +0300
IGNITE-12843 TDE - Phase-3. Cache key rotation - Fixes #7941.
Signed-off-by: Aleksey Plekhanov <pl...@gmail.com>
---
.../java/org/apache/ignite/IgniteEncryption.java | 16 +
.../configuration/DataStorageConfiguration.java | 24 +
.../configuration/EncryptionConfiguration.java | 108 ++
.../org/apache/ignite/internal/IgniteFeatures.java | 7 +-
.../encryption/CacheGroupEncryptionKeys.java | 376 +++++++
.../managers/encryption/CacheGroupPageScanner.java | 479 +++++++++
.../encryption/ChangeCacheEncryptionRequest.java | 108 ++
.../managers/encryption/EncryptionMXBeanImpl.java | 6 +
.../managers/encryption/GridEncryptionManager.java | 754 +++++++++++---
.../internal/managers/encryption/GroupKey.java | 85 ++
.../managers/encryption/GroupKeyChangeProcess.java | 356 +++++++
...ptionMXBeanImpl.java => GroupKeyEncrypted.java} | 43 +-
...ionMXBeanImpl.java => ReencryptStateUtils.java} | 40 +-
.../pagemem/wal/IgniteWriteAheadLogManager.java | 5 +
.../pagemem/wal/record/MasterKeyChangeRecord.java | 3 +
...ngeRecord.java => MasterKeyChangeRecordV2.java} | 24 +-
.../wal/record/ReencryptionStartRecord.java | 52 +
.../internal/pagemem/wal/record/WALRecord.java | 20 +-
.../delta/MetaPageUpdateIndexDataRecord.java | 109 ++
.../delta/MetaPageUpdatePartitionDataRecordV3.java | 123 +++
.../processors/cache/CacheGroupMetricsImpl.java | 50 +
.../internal/processors/cache/GridCacheUtils.java | 6 +-
.../processors/cache/mvcc/txlog/TxLog.java | 3 +-
.../GridCacheDatabaseSharedManager.java | 20 +-
.../cache/persistence/GridCacheOffheapManager.java | 140 ++-
.../cache/persistence/file/EncryptedFileIO.java | 70 +-
.../cache/persistence/tree/io/PageMetaIO.java | 3 +-
.../cache/persistence/tree/io/PageMetaIOV2.java | 106 ++
.../persistence/tree/io/PagePartitionMetaIO.java | 20 +-
.../persistence/tree/io/PagePartitionMetaIOV2.java | 17 +-
.../persistence/tree/io/PagePartitionMetaIOV3.java | 123 +++
.../persistence/wal/FileWriteAheadLogManager.java | 7 +
.../wal/serializer/RecordDataV1Serializer.java | 146 ++-
.../wal/serializer/RecordDataV2Serializer.java | 5 +-
.../ignite/internal/util/BasicRateLimiter.java | 153 +++
.../util/distributed/DistributedProcess.java | 12 +-
.../org/apache/ignite/mxbean/EncryptionMXBean.java | 12 +
.../encryption/AbstractEncryptionTest.java | 272 ++++-
.../encryption/CacheGroupKeyChangeTest.java | 1063 ++++++++++++++++++++
.../encryption/CacheGroupReencryptionTest.java | 867 ++++++++++++++++
.../encryption/EncryptedCacheBigEntryTest.java | 9 +-
.../encryption/EncryptedCacheCreateTest.java | 8 +-
.../encryption/EncryptedCacheDestroyTest.java | 10 +-
.../encryption/EncryptedCacheGroupCreateTest.java | 14 +-
.../encryption/EncryptedCacheRestartTest.java | 10 +-
.../internal/encryption/EncryptionMXBeanTest.java | 24 +
.../internal/encryption/MasterKeyChangeTest.java | 9 +-
.../cache/persistence/pagemem/NoOpWALManager.java | 5 +
.../ignite/internal/util/BasicRateLimiterTest.java | 108 ++
.../junits/common/GridCommonAbstractTest.java | 9 +-
.../testframework/wal/record/RecordUtils.java | 51 +-
.../IgniteBasicWithPersistenceTestSuite.java | 5 +
.../ignite/testsuites/IgniteUtilSelfTestSuite.java | 5 +-
.../testsuites/IgnitePdsWithIndexingTestSuite.java | 4 +-
.../SpringEncryptedCacheRestartTest.java | 33 +-
55 files changed, 5760 insertions(+), 377 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteEncryption.java b/modules/core/src/main/java/org/apache/ignite/IgniteEncryption.java
index 68439b7..debc779 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteEncryption.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteEncryption.java
@@ -17,6 +17,7 @@
package org.apache.ignite;
+import java.util.Collection;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.lang.IgniteFuture;
@@ -70,4 +71,19 @@ public interface IgniteEncryption {
* @return Future for this operation.
*/
public IgniteFuture<Void> changeMasterKey(String masterKeyName);
+
+ /**
+ * Starts cache group encryption key change process.
+ * <p>
+ * <b>NOTE:</b> Node join is rejected during rotation of cache group encryption key. Background re-encryption of
+ * existing data in the specified cache group(s) begins after the encryption key(s) is changed. During
+ * re-encryption, node join is not rejected, the cluster remains fully functional, it is fault-tolerant operation
+ * that automatically continues after restart. Secondary rotation of the encryption key of a cache group is only
+ * possible after background re-encryption of existing data in this cache group is completed.
+ *
+ * @param cacheOrGrpNames Cache or group names.
+ * @return Future which will be completed when new encryption key(s) are set for writing on all nodes in the cluster
+ * and re-encryption of existing cache data is initiated.
+ */
+ public IgniteFuture<Void> changeCacheGroupKey(Collection<String> cacheOrGrpNames);
}
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
index 3b70891..2a1927b 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
@@ -314,6 +314,9 @@ public class DataStorageConfiguration implements Serializable {
/** Default warm-up configuration. */
@Nullable private WarmUpConfiguration dfltWarmUpCfg;
+ /** Encryption configuration. */
+ private EncryptionConfiguration encCfg = new EncryptionConfiguration();
+
/**
* Creates valid durable memory configuration with all default values.
*/
@@ -1119,6 +1122,27 @@ public class DataStorageConfiguration implements Serializable {
}
/**
+ * Gets encryyption configuration.
+ *
+ * @return Encryption configuration.
+ */
+ public EncryptionConfiguration getEncryptionConfiguration() {
+ return encCfg;
+ }
+
+ /**
+ * Sets encryption configuration.
+ *
+ * @param encCfg Encryption configuration.
+ * @return {@code this} for chaining.
+ */
+ public DataStorageConfiguration setEncryptionConfiguration(EncryptionConfiguration encCfg) {
+ this.encCfg = encCfg;
+
+ return this;
+ }
+
+ /**
* Sets default warm-up configuration.
*
* @param dfltWarmUpCfg Default warm-up configuration. To assign a special
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/EncryptionConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/EncryptionConfiguration.java
new file mode 100644
index 0000000..79e205e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/EncryptionConfiguration.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.typedef.internal.A;
+
+/**
+ * Encryption configuration.
+ */
+public class EncryptionConfiguration implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Default re-encryption rate limit. The value is {@code 0}, which means that scan speed is not limited. */
+ public static final double DFLT_REENCRYPTION_RATE_MBPS = 0.0;
+
+ /** Default number of pages that is scanned during reencryption under checkpoint lock. The value is {@code 100}. */
+ public static final int DFLT_REENCRYPTION_BATCH_SIZE = 100;
+
+ /** Re-encryption rate limit in megabytes per second (set {@code 0} for unlimited scanning). */
+ private double reencryptionRateLimit = DFLT_REENCRYPTION_RATE_MBPS;
+
+ /** The number of pages that is scanned during re-encryption under checkpoint lock. */
+ private int reencryptionBatchSize = DFLT_REENCRYPTION_BATCH_SIZE;
+
+ /**
+ * Creates valid encryption configuration with all default values.
+ */
+ public EncryptionConfiguration() {
+ // No-op.
+ }
+
+ /**
+ * Constructs the copy of the configuration.
+ *
+ * @param cfg Configuration to copy.
+ */
+ public EncryptionConfiguration(EncryptionConfiguration cfg) {
+ assert cfg != null;
+
+ reencryptionBatchSize = cfg.getReencryptionBatchSize();
+ reencryptionRateLimit = cfg.getReencryptionRateLimit();
+ }
+
+ /**
+ * Gets re-encryption rate limit.
+ *
+ * @return Re-encryption rate limit in megabytes per second.
+ */
+ public double getReencryptionRateLimit() {
+ return reencryptionRateLimit;
+ }
+
+ /**
+ * Sets re-encryption rate limit.
+ *
+ * @param reencryptionRateLimit Re-encryption rate limit in megabytes per second.
+ * @return {@code this} for chaining.
+ */
+ public EncryptionConfiguration setReencryptionRateLimit(double reencryptionRateLimit) {
+ A.ensure(reencryptionRateLimit >= 0,
+ "Reencryption rate limit (" + reencryptionRateLimit + ") must be non-negative.");
+
+ this.reencryptionRateLimit = reencryptionRateLimit;
+
+ return this;
+ }
+
+ /**
+ * Gets the number of pages that is scanned during re-encryption under checkpoint lock.
+ *
+ * @return The number of pages that is scanned during re-encryption under checkpoint lock.
+ */
+ public int getReencryptionBatchSize() {
+ return reencryptionBatchSize;
+ }
+
+ /**
+ * Sets the number of pages that is scanned during re-encryption under checkpoint lock.
+ *
+ * @param reencryptionBatchSize The number of pages that is scanned during re-encryption under checkpoint lock.
+ * @return {@code this} for chaining.
+ */
+ public EncryptionConfiguration setReencryptionBatchSize(int reencryptionBatchSize) {
+ A.ensure(reencryptionBatchSize > 0,
+ "Reencryption batch size(" + reencryptionBatchSize + ") must be positive.");
+
+ this.reencryptionBatchSize = reencryptionBatchSize;
+
+ return this;
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteFeatures.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteFeatures.java
index 11ef198..e1f09e5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteFeatures.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteFeatures.java
@@ -18,6 +18,8 @@
package org.apache.ignite.internal;
import java.util.BitSet;
+import java.util.Collection;
+import org.apache.ignite.IgniteEncryption;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.cluster.ClusterState;
import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
@@ -128,7 +130,10 @@ public enum IgniteFeatures {
SPECIFIED_SEQ_PK_KEYS(45),
/** Compatibility support for new fields which are configured split. */
- SPLITTED_CACHE_CONFIGURATIONS_V2(46);
+ SPLITTED_CACHE_CONFIGURATIONS_V2(46),
+
+ /** Cache encryption key change. See {@link IgniteEncryption#changeCacheGroupKey(Collection)}. */
+ CACHE_GROUP_KEY_CHANGE(47);
/**
* Unique feature identifier.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/CacheGroupEncryptionKeys.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/CacheGroupEncryptionKeys.java
new file mode 100644
index 0000000..03b884b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/CacheGroupEncryptionKeys.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.encryption;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CopyOnWriteArrayList;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.encryption.EncryptionSpi;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Serves for managing encryption keys and related datastructure located in the heap.
+ */
+class CacheGroupEncryptionKeys {
+ /** Group encryption keys. */
+ private final Map<Integer, List<GroupKey>> grpKeys = new ConcurrentHashMap<>();
+
+ /**
+ * WAL segments encrypted with previous encryption keys prevent keys from being deleted
+ * until the associated segment is deleted.
+ */
+ private final Collection<TrackedWalSegment> trackedWalSegments = new ConcurrentLinkedQueue<>();
+
+ /** Encryption spi. */
+ private final EncryptionSpi encSpi;
+
+ /**
+ * @param encSpi Encryption spi.
+ */
+ CacheGroupEncryptionKeys(EncryptionSpi encSpi) {
+ this.encSpi = encSpi;
+ }
+
+ /**
+ * Returns group encryption key, that was set for writing.
+ *
+ * @param grpId Cache group ID.
+ * @return Group encryption key with ID, that was set for writing.
+ */
+ @Nullable GroupKey getActiveKey(int grpId) {
+ List<GroupKey> keys = grpKeys.get(grpId);
+
+ if (F.isEmpty(keys))
+ return null;
+
+ return keys.get(0);
+ }
+
+ /**
+ * Returns group encryption key with specified ID.
+ *
+ * @param grpId Cache group ID.
+ * @param keyId Encryption key ID.
+ * @return Group encryption key.
+ */
+ @Nullable GroupKey getKey(int grpId, int keyId) {
+ List<GroupKey> keys = grpKeys.get(grpId);
+
+ if (keys == null)
+ return null;
+
+ for (GroupKey groupKey : keys) {
+ if (groupKey.unsignedId() == keyId)
+ return groupKey;
+ }
+
+ return null;
+ }
+
+ /**
+ * Gets the existing encryption key IDs for the specified cache group.
+ *
+ * @param grpId Cache group ID.
+ * @return List of the key IDs.
+ */
+ @Nullable List<Integer> keyIds(int grpId) {
+ List<GroupKey> keys = grpKeys.get(grpId);
+
+ if (keys == null)
+ return null;
+
+ List<Integer> keyIds = new ArrayList<>(keys.size());
+
+ for (GroupKey groupKey : keys)
+ keyIds.add(groupKey.unsignedId());
+
+ return keyIds;
+ }
+
+ /**
+ * @return Cache group IDs for which encryption keys are stored.
+ */
+ Set<Integer> groupIds() {
+ return grpKeys.keySet();
+ }
+
+ /**
+ * @return Local encryption keys.
+ */
+ @Nullable HashMap<Integer, GroupKeyEncrypted> getAll() {
+ if (F.isEmpty(grpKeys))
+ return null;
+
+ HashMap<Integer, GroupKeyEncrypted> keys = U.newHashMap(grpKeys.size());
+
+ for (Map.Entry<Integer, List<GroupKey>> entry : grpKeys.entrySet()) {
+ int grpId = entry.getKey();
+ GroupKey grpKey = entry.getValue().get(0);
+
+ keys.put(grpId, new GroupKeyEncrypted(grpKey.unsignedId(), encSpi.encryptKey(grpKey.key())));
+ }
+
+ return keys;
+ }
+
+ /**
+ * @param grpId Cache group ID.
+ *
+ * @return Local encryption keys used for specified cache group.
+ */
+ @Nullable List<GroupKeyEncrypted> getAll(int grpId) {
+ List<GroupKey> grpKeys = this.grpKeys.get(grpId);
+
+ if (F.isEmpty(grpKeys))
+ return null;
+
+ List<GroupKeyEncrypted> encryptedKeys = new ArrayList<>(grpKeys.size());
+
+ for (GroupKey grpKey : grpKeys)
+ encryptedKeys.add(new GroupKeyEncrypted(grpKey.unsignedId(), encSpi.encryptKey(grpKey.key())));
+
+ return encryptedKeys;
+ }
+
+ /**
+ * Sets new encryption key for writing.
+ *
+ * @param grpId Cache group ID.
+ * @param keyId ID of the existing encryption key to be set for writing..
+ * @return Previous encryption key used for writing.
+ */
+ GroupKey changeActiveKey(int grpId, int keyId) {
+ List<GroupKey> keys = grpKeys.get(grpId);
+
+ assert !F.isEmpty(keys) : "grpId=" + grpId;
+
+ GroupKey prevKey = keys.get(0);
+
+ assert prevKey.unsignedId() != keyId : "keyId=" + keyId;
+
+ GroupKey newKey = null;
+
+ for (ListIterator<GroupKey> itr = keys.listIterator(keys.size()); itr.hasPrevious(); ) {
+ GroupKey key = itr.previous();
+
+ if (key.unsignedId() != keyId)
+ continue;
+
+ newKey = key;
+
+ break;
+ }
+
+ assert newKey != null : "exp=" + keyId + ", act=" + keys;
+
+ keys.add(0, newKey);
+
+ // Remove the duplicate key(s) from the tail of the list.
+ keys.subList(1, keys.size()).removeIf(k -> k.unsignedId() == keyId);
+
+ return prevKey;
+ }
+
+ /**
+ * Adds new encryption key.
+ *
+ * @param grpId Cache group ID.
+ * @param newEncKey New encrypted key for writing.
+ * @return {@code True} If a key has been added, {@code False} if the specified key is already present.
+ */
+ boolean addKey(int grpId, GroupKeyEncrypted newEncKey) {
+ List<GroupKey> keys = grpKeys.computeIfAbsent(grpId, v -> new CopyOnWriteArrayList<>());
+
+ GroupKey grpKey = new GroupKey(newEncKey.id(), encSpi.decryptKey(newEncKey.key()));
+
+ if (!keys.contains(grpKey))
+ return keys.add(grpKey);
+
+ return false;
+ }
+
+ /**
+ * @param grpId Cache group ID.
+ * @param encryptedKeys Encrypted keys.
+ */
+ void setGroupKeys(int grpId, List<GroupKeyEncrypted> encryptedKeys) {
+ List<GroupKey> keys = new CopyOnWriteArrayList<>();
+
+ for (GroupKeyEncrypted grpKey : encryptedKeys)
+ keys.add(new GroupKey(grpKey.id(), encSpi.decryptKey(grpKey.key())));
+
+ grpKeys.put(grpId, keys);
+ }
+
+ /**
+ * Remove encrytion keys associated with the specified cache group.
+ *
+ * @param grpId Cache group ID.
+ * @return List of encryption keys of the removed cache group.
+ */
+ List<GroupKey> remove(int grpId) {
+ return grpKeys.remove(grpId);
+ }
+
+ /**
+ * @param grpId Cache group ID.
+ * @param ids Key IDs for deletion.
+ * @return {@code True} if the keys have been deleted.
+ */
+ boolean removeKeysById(int grpId, Set<Integer> ids) {
+ List<GroupKey> keys = grpKeys.get(grpId);
+
+ if (F.isEmpty(keys))
+ return false;
+
+ return keys.subList(1, keys.size()).removeIf(key -> ids.contains(key.unsignedId()));
+ }
+
+ /**
+ * Remove unused keys.
+ *
+ * @param grpId Cache group ID.
+ * @return Removed key IDs,
+ */
+ Set<Integer> removeUnusedKeys(int grpId) {
+ List<GroupKey> keys = grpKeys.get(grpId);
+ Set<Integer> rmvKeyIds = U.newHashSet(keys.size() - 1);
+
+ rmvKeyIds.addAll(F.viewReadOnly(keys.subList(1, keys.size()), GroupKey::unsignedId));
+
+ for (TrackedWalSegment segment : trackedWalSegments) {
+ if (segment.grpId != grpId)
+ continue;
+
+ rmvKeyIds.remove(segment.keyId);
+ }
+
+ if (keys.removeIf(key -> rmvKeyIds.contains(key.unsignedId())))
+ return rmvKeyIds;
+
+ return Collections.emptySet();
+ }
+
+ /**
+ * @return A collection of tracked (encrypted with previous encryption keys) WAL segments.
+ */
+ Collection<TrackedWalSegment> trackedWalSegments() {
+ return Collections.unmodifiableCollection(trackedWalSegments);
+ }
+
+ /**
+ * @param segments WAL segments, mapped to cache group encryption key IDs.
+ */
+ void trackedWalSegments(Collection<TrackedWalSegment> segments) {
+ trackedWalSegments.addAll(segments);
+ }
+
+ /**
+ * Associate WAL segment index with the specified key ID
+ * to prevent deletion of that encryption key before deleting the segment.
+ *
+ * @param grpId Cache group ID.
+ * @param keyId Encryption key ID.
+ * @param walIdx WAL segment index.
+ */
+ void reserveWalKey(int grpId, int keyId, long walIdx) {
+ trackedWalSegments.add(new TrackedWalSegment(walIdx, grpId, keyId));
+ }
+
+ /**
+ * @param grpId Cache group ID.
+ * @param keyId Encryption key ID.
+ * @return Wal segment index or null if there no segment associated with the specified cache group ID and key ID.
+ */
+ @Nullable Long reservedSegment(int grpId, int keyId) {
+ for (TrackedWalSegment segment : trackedWalSegments) {
+ if (segment.grpId != grpId)
+ continue;
+
+ if (segment.keyId == keyId)
+ return segment.idx;
+ }
+
+ return null;
+ }
+
+ /**
+ * Remove all of the segments that are not greater than the specified index.
+ *
+ * @param walIdx WAL segment index.
+ * @return Map of group IDs with key IDs that were associated with removed WAL segments.
+ */
+ Map<Integer, Set<Integer>> releaseWalKeys(long walIdx) {
+ Map<Integer, Set<Integer>> rmvKeys = new HashMap<>();
+ Iterator<TrackedWalSegment> iter = trackedWalSegments.iterator();
+
+ while (iter.hasNext()) {
+ TrackedWalSegment segment = iter.next();
+
+ if (segment.idx > walIdx)
+ break;
+
+ iter.remove();
+
+ rmvKeys.computeIfAbsent(segment.grpId, v -> new HashSet<>()).add(segment.keyId);
+ }
+
+ return rmvKeys;
+ }
+
+ /**
+ * A WAL segment encrypted with a specific encryption key ID.
+ */
+ protected static final class TrackedWalSegment implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** WAL segment index. */
+ private final long idx;
+
+ /** Cache group ID. */
+ private final int grpId;
+
+ /** Encryption key ID. */
+ private final int keyId;
+
+ /**
+ * @param idx WAL segment index.
+ * @param grpId Cache group ID.
+ * @param keyId Encryption key ID.
+ */
+ public TrackedWalSegment(long idx, int grpId, int keyId) {
+ this.idx = idx;
+ this.grpId = grpId;
+ this.keyId = keyId;
+ }
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/CacheGroupPageScanner.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/CacheGroupPageScanner.java
new file mode 100644
index 0000000..dc0a29b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/CacheGroupPageScanner.java
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.encryption;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.locks.ReentrantLock;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.NodeStoppingException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
+import org.apache.ignite.internal.util.BasicRateLimiter;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.thread.IgniteThreadPoolExecutor;
+import org.apache.ignite.thread.OomExceptionHandler;
+
+import static org.apache.ignite.internal.util.IgniteUtils.MB;
+
+/**
+ * Cache group page stores scanner.
+ * Scans a range of pages and marks them as dirty to re-encrypt them with the last encryption key on disk.
+ */
+public class CacheGroupPageScanner implements CheckpointListener {
+ /** Thread prefix for scanning tasks. */
+ private static final String REENCRYPT_THREAD_PREFIX = "reencrypt";
+
+ /** Kernal context. */
+ private final GridKernalContext ctx;
+
+ /** Logger. */
+ private final IgniteLogger log;
+
+ /** Lock. */
+ private final ReentrantLock lock = new ReentrantLock();
+
+ /** Mapping of cache group ID to group scanning task. */
+ private final Map<Integer, GroupScanTask> grps = new ConcurrentHashMap<>();
+
+ /** Collection of groups waiting for a checkpoint. */
+ private final Collection<GroupScanTask> cpWaitGrps = new ConcurrentLinkedQueue<>();
+
+ /** Page scanning speed limiter. */
+ private final BasicRateLimiter limiter;
+
+ /** Single-threaded executor to run cache group scan task. */
+ private final ThreadPoolExecutor singleExecSvc;
+
+ /** Number of pages that is scanned during reencryption under checkpoint lock. */
+ private final int batchSize;
+
+ /** Stop flag. */
+ private boolean stopped;
+
+ /**
+ * @param ctx Grid kernal context.
+ */
+ public CacheGroupPageScanner(GridKernalContext ctx) {
+ this.ctx = ctx;
+
+ log = ctx.log(getClass());
+
+ DataStorageConfiguration dsCfg = ctx.config().getDataStorageConfiguration();
+
+ if (ctx.clientNode() || !CU.isPersistenceEnabled(dsCfg)) {
+ batchSize = -1;
+ limiter = null;
+ singleExecSvc = null;
+
+ return;
+ }
+
+ double rateLimit = dsCfg.getEncryptionConfiguration().getReencryptionRateLimit();
+
+ limiter = rateLimit > 0 ? new BasicRateLimiter(rateLimit * MB /
+ (dsCfg.getPageSize() == 0 ? DataStorageConfiguration.DFLT_PAGE_SIZE : dsCfg.getPageSize())) : null;
+
+ batchSize = dsCfg.getEncryptionConfiguration().getReencryptionBatchSize();
+
+ singleExecSvc = new IgniteThreadPoolExecutor(REENCRYPT_THREAD_PREFIX,
+ ctx.igniteInstanceName(),
+ 1,
+ 1,
+ IgniteConfiguration.DFLT_THREAD_KEEP_ALIVE_TIME,
+ new LinkedBlockingQueue<>(),
+ GridIoPolicy.SYSTEM_POOL,
+ new OomExceptionHandler(ctx));
+
+ singleExecSvc.allowCoreThreadTimeOut(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onCheckpointBegin(Context cpCtx) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void beforeCheckpointBegin(Context cpCtx) {
+ Set<GroupScanTask> completeCandidates = new HashSet<>();
+
+ cpWaitGrps.removeIf(completeCandidates::add);
+
+ cpCtx.finishedStateFut().listen(
+ f -> {
+ // Retry if error occurs.
+ if (f.error() != null || f.isCancelled()) {
+ cpWaitGrps.addAll(completeCandidates);
+
+ return;
+ }
+
+ lock.lock();
+
+ try {
+ for (GroupScanTask grpScanTask : completeCandidates) {
+ grps.remove(grpScanTask.groupId());
+
+ grpScanTask.onDone();
+
+ if (log.isInfoEnabled())
+ log.info("Cache group reencryption is finished [grpId=" + grpScanTask.groupId() + "]");
+ }
+
+ if (!grps.isEmpty())
+ return;
+
+ ((GridCacheDatabaseSharedManager)ctx.cache().context().database()).
+ removeCheckpointListener(this);
+ }
+ finally {
+ lock.unlock();
+ }
+ }
+ );
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onMarkCheckpointBegin(Context ctx) {
+ // No-op.
+ }
+
+ /**
+ * Schedule scanning partitions.
+ *
+ * @param grpId Cache group ID.
+ */
+ public IgniteInternalFuture<Void> schedule(int grpId) throws IgniteCheckedException {
+ CacheGroupContext grp = ctx.cache().cacheGroup(grpId);
+
+ if (grp == null || !grp.affinityNode()) {
+ if (log.isInfoEnabled())
+ log.info("Skip reencryption, cache group doesn't exist on the local node [grp=" + grpId + "]");
+
+ return new GridFinishedFuture<>();
+ }
+
+ lock.lock();
+
+ try {
+ if (stopped)
+ throw new NodeStoppingException("Operation has been cancelled (node is stopping).");
+
+ if (grps.isEmpty())
+ ((GridCacheDatabaseSharedManager)ctx.cache().context().database()).addCheckpointListener(this);
+
+ GroupScanTask prevState = grps.get(grpId);
+
+ if (prevState != null && !prevState.isDone()) {
+ if (log.isDebugEnabled())
+ log.debug("Reencryption already scheduled [grpId=" + grpId + "]");
+
+ return prevState;
+ }
+
+ Set<Integer> parts = new HashSet<>();
+
+ forEachPageStore(grp, new IgniteInClosureX<Integer>() {
+ @Override public void applyx(Integer partId) {
+ if (ctx.encryption().getEncryptionState(grpId, partId) == 0) {
+ if (log.isDebugEnabled())
+ log.debug("Skipping partition reencryption [grp=" + grpId + ", p=" + partId + "]");
+
+ return;
+ }
+
+ parts.add(partId);
+ }
+ });
+
+ GroupScanTask grpScan = new GroupScanTask(grp, parts);
+
+ singleExecSvc.submit(grpScan);
+
+ if (log.isInfoEnabled())
+ log.info("Scheduled reencryption [grpId=" + grpId + "]");
+
+ grps.put(grpId, grpScan);
+
+ return grpScan;
+ }
+ finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * @param grpId Cache group ID.
+ * @return Future that will be completed when all partitions have been scanned and pages have been written to disk.
+ */
+ public IgniteInternalFuture<Void> statusFuture(int grpId) {
+ GroupScanTask grpScanTask = grps.get(grpId);
+
+ return grpScanTask == null ? new GridFinishedFuture<>() : grpScanTask;
+ }
+
+ /**
+ * Shutdown scanning and disable new tasks scheduling.
+ */
+ public void stop() throws IgniteCheckedException {
+ lock.lock();
+
+ try {
+ stopped = true;
+
+ for (GroupScanTask grpScanTask : grps.values())
+ grpScanTask.cancel();
+
+ if (singleExecSvc != null)
+ singleExecSvc.shutdownNow();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Stop scannig the specified partition.
+ *
+ * @param grpId Cache group ID.
+ * @param partId Partition ID.
+ * @return {@code True} if reencryption was cancelled.
+ */
+ public boolean excludePartition(int grpId, int partId) {
+ GroupScanTask grpScanTask = grps.get(grpId);
+
+ if (grpScanTask == null)
+ return false;
+
+ return grpScanTask.excludePartition(partId);
+ }
+
+ /**
+ * Collect current number of pages in the specified cache group.
+ *
+ * @param grp Cache group.
+ * @return Partitions with current page count.
+ * @throws IgniteCheckedException If failed.
+ */
+ public long[] pagesCount(CacheGroupContext grp) throws IgniteCheckedException {
+ // The last element of the array is used to store the status of the index partition.
+ long[] partStates = new long[grp.affinity().partitions() + 1];
+
+ ctx.cache().context().database().checkpointReadLock();
+
+ try {
+ forEachPageStore(grp, new IgniteInClosureX<Integer>() {
+ @Override public void applyx(Integer partId) throws IgniteCheckedException {
+ int pagesCnt = ctx.cache().context().pageStore().pages(grp.groupId(), partId);
+
+ partStates[Math.min(partId, partStates.length - 1)] = pagesCnt;
+ }
+ });
+ } finally {
+ ctx.cache().context().database().checkpointReadUnlock();
+ }
+
+ return partStates;
+ }
+
+ /**
+ * @param grp Cache group.
+ * @param hnd Partition handler.
+ */
+ private void forEachPageStore(CacheGroupContext grp, IgniteInClosureX<Integer> hnd) throws IgniteCheckedException {
+ int parts = grp.affinity().partitions();
+
+ IgnitePageStoreManager pageStoreMgr = ctx.cache().context().pageStore();
+
+ for (int p = 0; p < parts; p++) {
+ if (!pageStoreMgr.exists(grp.groupId(), p))
+ continue;
+
+ hnd.applyx(p);
+ }
+
+ hnd.applyx(PageIdAllocator.INDEX_PARTITION);
+ }
+
+ /**
+ * Cache group partition scanning task.
+ */
+ private class GroupScanTask extends GridFutureAdapter<Void> implements Runnable {
+ /** Cache group ID. */
+ private final CacheGroupContext grp;
+
+ /** Partition IDs. */
+ private final Set<Integer> parts;
+
+ /** Page memory. */
+ private final PageMemoryEx pageMem;
+
+ /**
+ * @param grp Cache group.
+ */
+ public GroupScanTask(CacheGroupContext grp, Set<Integer> parts) {
+ this.grp = grp;
+ this.parts = new GridConcurrentHashSet<>(parts);
+
+ pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized boolean cancel() throws IgniteCheckedException {
+ return onCancelled();
+ }
+
+ /**
+ * Stop reencryption of the specified partition.
+ *
+ * @param partId Partition ID.
+ * @return {@code True} if reencryption was cancelled.
+ */
+ public synchronized boolean excludePartition(int partId) {
+ return parts.remove(partId);
+ }
+
+ /**
+ * @return Cache group ID.
+ */
+ public int groupId() {
+ return grp.groupId();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try {
+ for (int partId : parts) {
+ long state = ctx.encryption().getEncryptionState(grp.groupId(), partId);
+
+ if (state == 0)
+ continue;
+
+ scanPartition(partId, ReencryptStateUtils.pageIndex(state), ReencryptStateUtils.pageCount(state));
+
+ if (isDone())
+ return;
+ }
+
+ boolean added = cpWaitGrps.add(this);
+
+ assert added;
+ }
+ catch (Throwable t) {
+ if (X.hasCause(t, NodeStoppingException.class))
+ onCancelled();
+ else
+ onDone(t);
+ }
+ }
+
+ /**
+ * @param partId Partition ID.
+ * @param off Start page offset.
+ * @param cnt Count of pages to scan.
+ */
+ private void scanPartition(int partId, int off, int cnt) throws IgniteCheckedException {
+ if (log.isDebugEnabled()) {
+ log.debug("Partition reencryption is started [grpId=" + grp.groupId() +
+ ", p=" + partId + ", remain=" + (cnt - off) + ", total=" + cnt + "]");
+ }
+
+ while (off < cnt) {
+ int pagesCnt = Math.min(batchSize, cnt - off);
+
+ if (limiter != null)
+ limiter.acquire(pagesCnt);
+
+ synchronized (this) {
+ if (isDone() || !parts.contains(partId))
+ break;
+
+ ctx.cache().context().database().checkpointReadLock();
+
+ try {
+ off += scanPages(partId, off, pagesCnt);
+ }
+ finally {
+ ctx.cache().context().database().checkpointReadUnlock();
+ }
+ }
+
+ ctx.encryption().setEncryptionState(grp, partId, off, cnt);
+ }
+
+ if (log.isDebugEnabled()) {
+ log.debug("Partition reencryption is finished " +
+ "[grpId=" + grp.groupId() +
+ ", p=" + partId +
+ ", remain=" + (cnt - off) +
+ ", total=" + cnt + "]");
+ }
+ }
+
+ /**
+ * @param off Start page offset.
+ * @param cnt Count of pages to scan.
+ * @return Count of scanned pages.
+ * @throws IgniteCheckedException If failed.
+ */
+ private int scanPages(int partId, int off, int cnt) throws IgniteCheckedException {
+ int grpId = grp.groupId();
+ byte flag = GroupPartitionId.getFlagByPartId(partId);
+
+ for (int pageIdx = off; pageIdx < off + cnt; pageIdx++) {
+ long pageId = PageIdUtils.pageId(partId, flag, pageIdx);
+ long page = pageMem.acquirePage(grpId, pageId);
+
+ try {
+ if (pageMem.isDirty(grpId, pageId, page))
+ continue;
+
+ pageMem.writeLock(grpId, pageId, page, true);
+ pageMem.writeUnlock(grpId, pageId, page, null, true);
+ }
+ finally {
+ pageMem.releasePage(grpId, pageId, page);
+ }
+ }
+
+ return cnt;
+ }
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/ChangeCacheEncryptionRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/ChangeCacheEncryptionRequest.java
new file mode 100644
index 0000000..54d7405
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/ChangeCacheEncryptionRequest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.encryption;
+
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.UUID;
+
+/**
+ * Change cache group encryption key request.
+ */
+@SuppressWarnings("AssignmentOrReturnOfFieldWithMutableType")
+public class ChangeCacheEncryptionRequest implements Serializable {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 0L;
+
+ /** Request ID. */
+ private final UUID reqId = UUID.randomUUID();
+
+ /** Cache group IDs. */
+ private final int[] grpIds;
+
+ /** Encryption keys. */
+ private final byte[][] keys;
+
+ /** Key identifiers. */
+ private final byte[] keyIds;
+
+ /** Master key digest. */
+ private final byte[] masterKeyDigest;
+
+ /**
+ * @param grpIds Cache group IDs.
+ * @param keys Encryption keys.
+ * @param keyIds Key identifiers.
+ * @param masterKeyDigest Master key digest.
+ */
+ public ChangeCacheEncryptionRequest(int[] grpIds, byte[][] keys, byte[] keyIds, byte[] masterKeyDigest) {
+ this.grpIds = grpIds;
+ this.keys = keys;
+ this.keyIds = keyIds;
+ this.masterKeyDigest = masterKeyDigest;
+ }
+
+ /**
+ * @return Request ID.
+ */
+ public UUID requestId() {
+ return reqId;
+ }
+
+ /**
+ * @return Cache group IDs.
+ */
+ public int[] groupIds() {
+ return grpIds;
+ }
+
+ /**
+ * @return Encryption keys.
+ */
+ public byte[][] keys() {
+ return keys;
+ }
+
+ /**
+ * @return Key identifiers.
+ */
+ public byte[] keyIds() { return keyIds; }
+
+ /**
+ * @return Master key digest.
+ */
+ public byte[] masterKeyDigest() {
+ return masterKeyDigest;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ return Objects.equals(reqId, ((ChangeCacheEncryptionRequest)o).reqId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hash(reqId);
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/EncryptionMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/EncryptionMXBeanImpl.java
index 0270222..f2e31a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/EncryptionMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/EncryptionMXBeanImpl.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.managers.encryption;
+import java.util.Collections;
import org.apache.ignite.internal.GridKernalContextImpl;
import org.apache.ignite.mxbean.EncryptionMXBean;
@@ -41,4 +42,9 @@ public class EncryptionMXBeanImpl implements EncryptionMXBean {
@Override public void changeMasterKey(String masterKeyName) {
encryptionMgr.changeMasterKey(masterKeyName).get();
}
+
+ /** {@inheritDoc} */
+ @Override public void changeCacheGroupKey(String cacheOrGrpName) {
+ encryptionMgr.changeCacheGroupKey(Collections.singleton(cacheOrGrpName)).get();
+ }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GridEncryptionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GridEncryptionManager.java
index 78590d3..3683313 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GridEncryptionManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GridEncryptionManager.java
@@ -40,6 +40,7 @@ import org.apache.ignite.IgniteEncryption;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteFeatures;
@@ -47,7 +48,9 @@ import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.managers.GridManagerAdapter;
import org.apache.ignite.internal.managers.communication.GridMessageListener;
import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
-import org.apache.ignite.internal.pagemem.wal.record.MasterKeyChangeRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MasterKeyChangeRecordV2;
+import org.apache.ignite.internal.pagemem.wal.record.ReencryptionStartRecord;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage;
@@ -61,6 +64,7 @@ import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
import org.apache.ignite.internal.util.future.IgniteFutureImpl;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteFuture;
@@ -72,8 +76,6 @@ import org.apache.ignite.spi.IgniteSpiException;
import org.apache.ignite.spi.discovery.DiscoveryDataBag;
import org.apache.ignite.spi.discovery.DiscoveryDataBag.GridDiscoveryData;
import org.apache.ignite.spi.discovery.DiscoveryDataBag.JoiningNodeDiscoveryData;
-import org.apache.ignite.spi.discovery.DiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.encryption.EncryptionSpi;
import org.jetbrains.annotations.Nullable;
@@ -83,6 +85,7 @@ import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
import static org.apache.ignite.internal.GridComponent.DiscoveryDataExchangeType.ENCRYPTION_MGR;
import static org.apache.ignite.internal.GridTopic.TOPIC_GEN_ENC_KEY;
+import static org.apache.ignite.internal.IgniteFeatures.CACHE_GROUP_KEY_CHANGE;
import static org.apache.ignite.internal.IgniteFeatures.MASTER_KEY_CHANGE;
import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.MASTER_KEY_CHANGE_FINISH;
@@ -115,6 +118,8 @@ import static org.apache.ignite.internal.util.distributed.DistributedProcess.Dis
* <ul>
* <li>1. If new key for group doesn't exists locally it added to local store.</li>
* <li>2. If new key for group exists locally, then received key skipped.</li>
+ * <li>3. If a cache group is encrypted with a different (previous) encryption key, then background
+ * re-encryption of this group with a new key is started.</li>
* </ul>
* </li>
* </ul>
@@ -136,6 +141,22 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
*/
private static final IgniteProductVersion CACHE_ENCRYPTION_SINCE = IgniteProductVersion.fromString("2.7.0");
+ /** Prefix for a master key name. */
+ public static final String MASTER_KEY_NAME_PREFIX = "encryption-master-key-name";
+
+ /** Prefix for a encryption group key in meta store, which contains encryption keys with identifiers. */
+ public static final String ENCRYPTION_KEYS_PREFIX = "grp-encryption-keys-";
+
+ /** Initial identifier for cache group encryption key. */
+ public static final int INITIAL_KEY_ID = 0;
+
+ /** The name on the meta store key, that contains wal segments encrypted using previous encryption keys. */
+ private static final String REENCRYPTED_WAL_SEGMENTS = "reencrypted-wal-segments";
+
+ /** Prefix for a encryption group key in meta store. */
+ @Deprecated
+ private static final String ENCRYPTION_KEY_PREFIX = "grp-encryption-key-";
+
/** Synchronization mutex. */
private final Object metaStorageMux = new Object();
@@ -154,14 +175,8 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
/** Flag to enable/disable write to metastore on cluster state change. */
private volatile boolean writeToMetaStoreEnabled;
- /** Prefix for a encryption group key in meta store. */
- public static final String ENCRYPTION_KEY_PREFIX = "grp-encryption-key-";
-
- /** Prefix for a master key name. */
- public static final String MASTER_KEY_NAME_PREFIX = "encryption-master-key-name";
-
- /** Group encryption keys. */
- private final ConcurrentHashMap<Integer, Serializable> grpEncKeys = new ConcurrentHashMap<>();
+ /** Cache group encryption keys. */
+ private CacheGroupEncryptionKeys grpKeys;
/** Pending generate encryption key futures. */
private ConcurrentMap<IgniteUuid, GenerateEncryptionKeyFuture> genEncKeyFuts = new ConcurrentHashMap<>();
@@ -182,7 +197,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
private volatile boolean recoveryMasterKeyName;
/** Master key change future. Not {@code null} on request initiator. */
- private MasterKeyChangeFuture masterKeyChangeFut;
+ private KeyChangeFuture masterKeyChangeFut;
/** Pending master key request or {@code null} if there is no ongoing master key change process. */
private volatile MasterKeyChangeRequest masterKeyChangeRequest;
@@ -194,10 +209,25 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
* Master key change prepare process. Checks that all server nodes have the same new master key and then starts
* finish process.
*/
- private DistributedProcess<MasterKeyChangeRequest, MasterKeyChangeResult> prepareMKChangeProc;
+ private DistributedProcess<MasterKeyChangeRequest, EmptyResult> prepareMKChangeProc;
/** Process to perform the master key change. Changes master key and reencrypt group keys. */
- private DistributedProcess<MasterKeyChangeRequest, MasterKeyChangeResult> performMKChangeProc;
+ private DistributedProcess<MasterKeyChangeRequest, EmptyResult> performMKChangeProc;
+
+ /**
+ * A two-phase distributed process that rotates the encryption keys of specified cache groups and initiates
+ * re-encryption of those cache groups.
+ */
+ private GroupKeyChangeProcess grpKeyChangeProc;
+
+ /** Cache groups for which encryption key was changed, and they must be re-encrypted. */
+ private final Map<Integer, long[]> reencryptGroups = new ConcurrentHashMap<>();
+
+ /** Cache groups for which encryption key was changed on node join. */
+ private final Map<Integer, Integer> reencryptGroupsForced = new ConcurrentHashMap<>();
+
+ /** Cache group page stores scanner. */
+ private CacheGroupPageScanner pageScanner;
/**
* @param ctx Kernel context.
@@ -283,11 +313,17 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
performMKChangeProc = new DistributedProcess<>(ctx, MASTER_KEY_CHANGE_FINISH, this::performMasterKeyChange,
this::finishPerformMasterKeyChange);
+
+ grpKeys = new CacheGroupEncryptionKeys(getSpi());
+ pageScanner = new CacheGroupPageScanner(ctx);
+ grpKeyChangeProc = new GroupKeyChangeProcess(ctx, grpKeys);
}
/** {@inheritDoc} */
@Override public void stop(boolean cancel) throws IgniteCheckedException {
stopSpi();
+
+ pageScanner.stop();
}
/** {@inheritDoc} */
@@ -340,7 +376,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
* Callback for local join.
*/
public void onLocalJoin() {
- if (!isCoordinator())
+ if (!U.isLocalNodeCoordinator(ctx.discovery()))
return;
//We can't store keys before node join to cluster(on statically configured cache registration).
@@ -350,7 +386,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
//And sends that keys to every joining node.
synchronized (metaStorageMux) {
//Keys read from meta storage.
- HashMap<Integer, byte[]> knownEncKeys = knownEncryptionKeys();
+ HashMap<Integer, GroupKeyEncrypted> knownEncKeys = grpKeys.getAll();
//Generated(not saved!) keys for a new caches.
//Configured statically in config, but doesn't stored on the disk.
@@ -362,7 +398,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
//We can store keys to the disk, because we are on a coordinator.
for (Map.Entry<Integer, byte[]> entry : newEncKeys.entrySet()) {
- groupKey(entry.getKey(), entry.getValue());
+ addGroupKey(entry.getKey(), new GroupKeyEncrypted(INITIAL_KEY_ID, entry.getValue()));
U.quietAndInfo(log, "Added encryption key on local join [grpId=" + entry.getKey() + "]");
}
@@ -392,6 +428,12 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
if (res != null)
return res;
+ if (grpKeyChangeProc.inProgress()) {
+ return new IgniteNodeValidationResult(ctx.localNodeId(),
+ "Cache group key change is in progress! Node join is rejected. [node=" + node.id() + "]",
+ "Cache group key change is in progress! Node join is rejected.");
+ }
+
NodeEncryptionKeys nodeEncKeys = (NodeEncryptionKeys)discoData.joiningNodeData();
if (!discoData.hasJoiningNodeData() || nodeEncKeys == null) {
@@ -406,23 +448,49 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
"Master key digest differs! Node join is rejected.");
}
+ if (!IgniteFeatures.nodeSupports(node, CACHE_GROUP_KEY_CHANGE)) {
+ return new IgniteNodeValidationResult(ctx.localNodeId(),
+ "Joining node doesn't support multiple encryption keys for single group [node=" + node.id() + "]",
+ "Joining node doesn't support multiple encryption keys for single group.");
+ }
+
if (F.isEmpty(nodeEncKeys.knownKeys)) {
U.quietAndInfo(log, "Joining node doesn't have stored group keys [node=" + node.id() + "]");
return null;
}
- for (Map.Entry<Integer, byte[]> entry : nodeEncKeys.knownKeys.entrySet()) {
- Serializable locEncKey = grpEncKeys.get(entry.getKey());
+ assert !F.isEmpty(nodeEncKeys.knownKeysWithIds);
+
+ for (Map.Entry<Integer, List<GroupKeyEncrypted>> entry : nodeEncKeys.knownKeysWithIds.entrySet()) {
+ int grpId = entry.getKey();
+
+ GroupKey locEncKey = groupKey(grpId);
if (locEncKey == null)
continue;
- Serializable rmtKey = getSpi().decryptKey(entry.getValue());
+ List<GroupKeyEncrypted> rmtKeys = entry.getValue();
- if (F.eq(locEncKey, rmtKey))
+ if (rmtKeys == null)
continue;
+ GroupKeyEncrypted rmtKeyEncrypted = null;
+
+ for (GroupKeyEncrypted rmtKey0 : rmtKeys) {
+ if (rmtKey0.id() != locEncKey.unsignedId())
+ continue;
+
+ rmtKeyEncrypted = rmtKey0;
+
+ break;
+ }
+
+ if (rmtKeyEncrypted == null || F.eq(locEncKey.key(), getSpi().decryptKey(rmtKeyEncrypted.key())))
+ continue;
+
+ // The remote node should not rotate the cache key to the current one
+ // until the old key (with an identifier that is currently active in the cluster) is removed.
return new IgniteNodeValidationResult(ctx.localNodeId(),
"Cache key differs! Node join is rejected. [node=" + node.id() + ", grp=" + entry.getKey() + "]",
"Cache key differs! Node join is rejected.");
@@ -436,10 +504,14 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
if (dataBag.isJoiningNodeClient())
return;
- HashMap<Integer, byte[]> knownEncKeys = knownEncryptionKeys();
+ Set<Integer> grpIds = grpKeys.groupIds();
- HashMap<Integer, byte[]> newKeys =
- newEncryptionKeys(knownEncKeys == null ? Collections.EMPTY_SET : knownEncKeys.keySet());
+ HashMap<Integer, List<GroupKeyEncrypted>> knownEncKeys = U.newHashMap(grpIds.size());
+
+ for (int grpId : grpIds)
+ knownEncKeys.put(grpId, grpKeys.getAll(grpId));
+
+ HashMap<Integer, byte[]> newKeys = newEncryptionKeys(grpIds);
if (log.isInfoEnabled()) {
String knownGrps = F.isEmpty(knownEncKeys) ? null : F.concat(knownEncKeys.keySet(), ",");
@@ -467,9 +539,9 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
for (Map.Entry<Integer, byte[]> entry : nodeEncryptionKeys.newKeys.entrySet()) {
if (groupKey(entry.getKey()) == null) {
U.quietAndInfo(log, "Store group key received from joining node [node=" +
- data.joiningNodeId() + ", grp=" + entry.getKey() + "]");
+ data.joiningNodeId() + ", grp=" + entry.getKey() + "]");
- groupKey(entry.getKey(), entry.getValue());
+ addGroupKey(entry.getKey(), new GroupKeyEncrypted(INITIAL_KEY_ID, entry.getValue()));
}
else {
U.quietAndInfo(log, "Skip group key received from joining node. Already exists. [node=" +
@@ -483,16 +555,18 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
if (dataBag.isJoiningNodeClient() || dataBag.commonDataCollectedFor(ENCRYPTION_MGR.ordinal()))
return;
- HashMap<Integer, byte[]> knownEncKeys = knownEncryptionKeys();
+ HashMap<Integer, GroupKeyEncrypted> knownEncKeys = grpKeys.getAll();
HashMap<Integer, byte[]> newKeys =
newEncryptionKeys(knownEncKeys == null ? Collections.EMPTY_SET : knownEncKeys.keySet());
- if (knownEncKeys == null)
- knownEncKeys = newKeys;
- else if (newKeys != null) {
+ if (!F.isEmpty(newKeys)) {
+ if (knownEncKeys == null)
+ knownEncKeys = new HashMap<>();
+
for (Map.Entry<Integer, byte[]> entry : newKeys.entrySet()) {
- byte[] old = knownEncKeys.putIfAbsent(entry.getKey(), entry.getValue());
+ GroupKeyEncrypted old =
+ knownEncKeys.putIfAbsent(entry.getKey(), new GroupKeyEncrypted(INITIAL_KEY_ID, entry.getValue()));
assert old == null;
}
@@ -506,55 +580,93 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
if (ctx.clientNode())
return;
- Map<Integer, byte[]> encKeysFromCluster = (Map<Integer, byte[]>)data.commonData();
+ Map<Integer, Object> encKeysFromCluster = (Map<Integer, Object>)data.commonData();
if (F.isEmpty(encKeysFromCluster))
return;
- for (Map.Entry<Integer, byte[]> entry : encKeysFromCluster.entrySet()) {
- if (groupKey(entry.getKey()) == null) {
- U.quietAndInfo(log, "Store group key received from coordinator [grp=" + entry.getKey() + "]");
+ for (Map.Entry<Integer, Object> entry : encKeysFromCluster.entrySet()) {
+ int grpId = entry.getKey();
- groupKey(entry.getKey(), entry.getValue());
- }
- else {
+ GroupKeyEncrypted rmtKey;
+
+ if (entry.getValue() instanceof GroupKeyEncrypted)
+ rmtKey = (GroupKeyEncrypted)entry.getValue();
+ else
+ rmtKey = new GroupKeyEncrypted(INITIAL_KEY_ID, (byte[])entry.getValue());
+
+ GroupKey locGrpKey = groupKey(grpId);
+
+ if (locGrpKey != null && locGrpKey.unsignedId() == rmtKey.id()) {
U.quietAndInfo(log, "Skip group key received from coordinator. Already exists. [grp=" +
- entry.getKey() + "]");
+ grpId + ", keyId=" + rmtKey.id() + "]");
+
+ continue;
}
+
+ U.quietAndInfo(log, "Store group key received from coordinator [grp=" + grpId +
+ ", keyId=" + rmtKey.id() + "]");
+
+ grpKeys.addKey(grpId, rmtKey);
+
+ if (locGrpKey == null)
+ continue;
+
+ GroupKey prevKey = grpKeys.changeActiveKey(grpId, rmtKey.id());
+
+ if (ctx.config().getDataStorageConfiguration().getWalMode() != WALMode.NONE)
+ grpKeys.reserveWalKey(grpId, prevKey.unsignedId(), ctx.cache().context().wal().currentSegment());
+
+ reencryptGroupsForced.put(grpId, rmtKey.id());
}
}
/**
* Returns group encryption key.
*
- * @param grpId Group id.
- * @return Group encryption key.
+ * @param grpId Cache group ID.
+ * @return Group encryption key with identifier, that was set for writing.
*/
- @Nullable public Serializable groupKey(int grpId) {
- if (grpEncKeys.isEmpty())
- return null;
-
- return grpEncKeys.get(grpId);
+ @Nullable public GroupKey groupKey(int grpId) {
+ return grpKeys.getActiveKey(grpId);
}
/**
- * Store group encryption key.
+ * Returns group encryption key with specified identifier.
*
- * @param grpId Group id.
- * @param encGrpKey Encrypted group key.
+ * @param grpId Cache group ID.
+ * @param keyId Encryption key ID.
+ * @return Group encryption key.
*/
- public void groupKey(int grpId, byte[] encGrpKey) {
- assert !grpEncKeys.containsKey(grpId);
+ @Nullable public GroupKey groupKey(int grpId, int keyId) {
+ return grpKeys.getKey(grpId, keyId);
+ }
- Serializable encKey = withMasterKeyChangeReadLock(() -> getSpi().decryptKey(encGrpKey));
+ /**
+ * Gets the existing encryption key IDs for the specified cache group.
+ *
+ * @param grpId Cache group ID.
+ * @return List of the key identifiers.
+ */
+ @Nullable public List<Integer> groupKeyIds(int grpId) {
+ return grpKeys.keyIds(grpId);
+ }
+ /**
+ * Adds new cache group encryption key.
+ *
+ * @param grpId Cache group ID.
+ * @param key Encryption key.
+ */
+ void addGroupKey(int grpId, GroupKeyEncrypted key) {
synchronized (metaStorageMux) {
- if (log.isDebugEnabled())
- log.debug("Key added. [grp=" + grpId + "]");
-
- grpEncKeys.put(grpId, encKey);
+ try {
+ grpKeys.addKey(grpId, key);
- writeToMetaStore(grpId, encGrpKey);
+ writeGroupKeysToMetaStore(grpId, grpKeys.getAll(grpId));
+ } catch (IgniteCheckedException e) {
+ throw new IgniteException("Failed to write cache group encryption key [grpId=" + grpId + ']', e);
+ }
}
}
@@ -588,7 +700,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
digest = masterKeyDigest(masterKeyName);
} catch (Exception e) {
return new IgniteFinishedFutureImpl<>(new IgniteException("Master key change was rejected. " +
- "Unable to get the master key digest."));
+ "Unable to get the master key digest.", e));
}
MasterKeyChangeRequest request = new MasterKeyChangeRequest(UUID.randomUUID(), encryptKeyName(masterKeyName),
@@ -611,7 +723,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
"The previous change was not completed."));
}
- masterKeyChangeFut = new MasterKeyChangeFuture(request.requestId());
+ masterKeyChangeFut = new KeyChangeFuture(request.requestId());
prepareMKChangeProc.start(request.requestId(), request);
@@ -627,22 +739,107 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
return withMasterKeyChangeReadLock(() -> getSpi().getMasterKeyName());
}
+ /** {@inheritDoc} */
+ @Override public IgniteFuture<Void> changeCacheGroupKey(Collection<String> cacheOrGrpNames) {
+ A.notEmpty(cacheOrGrpNames, "cacheOrGrpNames");
+
+ synchronized (opsMux) {
+ if (stopped) {
+ return new IgniteFinishedFutureImpl<>(new IgniteException("Cache group key change was rejected. " +
+ "Node is stopping."));
+ }
+
+ return grpKeyChangeProc.start(cacheOrGrpNames);
+ }
+ }
+
+ /**
+ * @param grpIds Cache group IDs.
+ * @param keyIds Encryption key IDs.
+ * @param keys Encryption keys.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected void changeCacheGroupKeyLocal(int[] grpIds, byte[] keyIds, byte[][] keys) throws IgniteCheckedException {
+ Map<Integer, Byte> encryptionStatus = U.newHashMap(grpIds.length);
+
+ for (int i = 0; i < grpIds.length; i++)
+ encryptionStatus.put(grpIds[i], keyIds[i]);
+
+ WALPointer ptr = ctx.cache().context().wal().log(new ReencryptionStartRecord(encryptionStatus));
+
+ if (ptr != null)
+ ctx.cache().context().wal().flush(ptr, false);
+
+ for (int i = 0; i < grpIds.length; i++) {
+ int grpId = grpIds[i];
+ int newKeyId = keyIds[i] & 0xff;
+
+ withMasterKeyChangeReadLock(() -> {
+ synchronized (metaStorageMux) {
+ // Set new key as key for writing. Note that we cannot pass the encrypted key here because the master
+ // key may have changed in which case we will not be able to decrypt the cache encryption key.
+ GroupKey prevGrpKey = grpKeys.changeActiveKey(grpId, newKeyId);
+
+ writeGroupKeysToMetaStore(grpId, grpKeys.getAll(grpId));
+
+ if (ptr == null)
+ return null;
+
+ grpKeys.reserveWalKey(grpId, prevGrpKey.unsignedId(), ctx.cache().context().wal().currentSegment());
+
+ writeTrackedWalIdxsToMetaStore();
+ }
+
+ return null;
+ });
+
+ CacheGroupContext grp = ctx.cache().cacheGroup(grpId);
+
+ if (grp != null && grp.affinityNode())
+ reencryptGroups.put(grpId, pageScanner.pagesCount(grp));
+
+ if (log.isInfoEnabled())
+ log.info("New encryption key for group was added [grpId=" + grpId + ", keyId=" + newKeyId + "]");
+ }
+
+ startReencryption(encryptionStatus.keySet());
+ }
+
+ /**
+ * @param grpId Cache group ID.
+ * @return Future that will be completed when reencryption of the specified group is finished.
+ */
+ public IgniteInternalFuture<Void> reencryptionFuture(int grpId) {
+ return pageScanner.statusFuture(grpId);
+ }
+
+ /**
+ * @param grpId Cache group ID.
+ * @return {@code True} If the specified cache group is currently being re-encrypted.
+ */
+ public boolean reencryptionInProgress(int grpId) {
+ // The method guarantees not only the completion of the re-encryption, but also that the clearing of
+ // unused keys is complete.
+ return reencryptGroups.containsKey(grpId);
+ }
+
/**
- * Removes encryption key.
+ * Removes encryption key(s).
*
- * @param grpId Group id.
+ * @param grpId Cache group ID.
*/
private void removeGroupKey(int grpId) {
synchronized (metaStorageMux) {
ctx.cache().context().database().checkpointReadLock();
try {
- grpEncKeys.remove(grpId);
+ if (grpKeys.remove(grpId) == null)
+ return;
- metaStorage.remove(ENCRYPTION_KEY_PREFIX + grpId);
+ metaStorage.remove(ENCRYPTION_KEYS_PREFIX + grpId);
if (log.isDebugEnabled())
- log.debug("Key removed. [grp=" + grpId + "]");
+ log.debug("Key(s) removed. [grp=" + grpId + "]");
}
catch (IgniteCheckedException e) {
U.error(log, "Failed to clear meta storage", e);
@@ -655,19 +852,41 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
/**
* Callback for cache group start event.
- * @param grpId Group id.
+ *
+ * @param grpId Cache group ID.
* @param encKey Encryption key
*/
public void beforeCacheGroupStart(int grpId, @Nullable byte[] encKey) {
if (encKey == null || ctx.clientNode())
return;
- groupKey(grpId, encKey);
+ withMasterKeyChangeReadLock(() -> {
+ addGroupKey(grpId, new GroupKeyEncrypted(INITIAL_KEY_ID, encKey));
+
+ return null;
+ });
+ }
+
+ /**
+ * Callback is called before invalidate page memory.
+ *
+ * @param grpId Cache group ID.
+ */
+ public void onCacheGroupStop(int grpId) {
+ try {
+ reencryptionFuture(grpId).cancel();
+ }
+ catch (IgniteCheckedException e) {
+ log.warning("Unable to cancel reencryption [grpId=" + grpId + "]", e);
+ }
+
+ reencryptGroups.remove(grpId);
}
/**
* Callback for cache group destroy event.
- * @param grpId Group id.
+ *
+ * @param grpId Cache group ID.
*/
public void onCacheGroupDestroyed(int grpId) {
if (groupKey(grpId) == null)
@@ -676,6 +895,59 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
removeGroupKey(grpId);
}
+ /**
+ * @param grp Cache group.
+ * @param partId Partition ID.
+ */
+ public void onDestroyPartitionStore(CacheGroupContext grp, int partId) {
+ if (pageScanner.excludePartition(grp.groupId(), partId))
+ setEncryptionState(grp, partId, 0, 0);
+ }
+
+ /**
+ * Callback when WAL segment is removed.
+ *
+ * @param segmentIdx WAL segment index.
+ */
+ public void onWalSegmentRemoved(long segmentIdx) {
+ withMasterKeyChangeReadLock(() -> {
+ synchronized (metaStorageMux) {
+ Map<Integer, Set<Integer>> rmvKeys = grpKeys.releaseWalKeys(segmentIdx);
+
+ if (F.isEmpty(rmvKeys))
+ return null;
+
+ try {
+ writeTrackedWalIdxsToMetaStore();
+
+ for (Map.Entry<Integer, Set<Integer>> entry : rmvKeys.entrySet()) {
+ Integer grpId = entry.getKey();
+
+ if (reencryptGroups.containsKey(grpId))
+ continue;
+
+ Set<Integer> keyIds = entry.getValue();
+
+ if (!grpKeys.removeKeysById(grpId, keyIds))
+ continue;
+
+ writeGroupKeysToMetaStore(grpId, grpKeys.getAll(grpId));
+
+ if (log.isInfoEnabled()) {
+ log.info("Previous encryption keys have been removed [grpId=" + grpId +
+ ", keyIds=" + keyIds + "]");
+ }
+ }
+ }
+ catch (IgniteCheckedException e) {
+ log.error("Unable to remove encryption keys from metastore.", e);
+ }
+ }
+
+ return null;
+ });
+ }
+
/** {@inheritDoc} */
@Override public void onReadyForRead(ReadOnlyMetastorage metastorage) {
try {
@@ -690,17 +962,35 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
}
}
- metastorage.iterate(ENCRYPTION_KEY_PREFIX, (key, val) -> {
- Integer grpId = Integer.valueOf(key.replace(ENCRYPTION_KEY_PREFIX, ""));
+ metastorage.iterate(ENCRYPTION_KEYS_PREFIX, (key, val) -> {
+ int grpId = Integer.parseInt(key.replace(ENCRYPTION_KEYS_PREFIX, ""));
- byte[] encGrpKey = (byte[])val;
+ if (grpKeys.groupIds().contains(grpId))
+ return;
- grpEncKeys.computeIfAbsent(grpId, k -> getSpi().decryptKey(encGrpKey));
+ grpKeys.setGroupKeys(grpId, (List<GroupKeyEncrypted>)val);
}, true);
- if (!grpEncKeys.isEmpty()) {
- U.quietAndInfo(log, "Encryption keys loaded from metastore. [grps=" +
- F.concat(grpEncKeys.keySet(), ",") + ", masterKeyName=" + getSpi().getMasterKeyName() + ']');
+ // Try to read keys in previous format.
+ if (grpKeys.groupIds().isEmpty()) {
+ metastorage.iterate(ENCRYPTION_KEY_PREFIX, (key, val) -> {
+ int grpId = Integer.parseInt(key.replace(ENCRYPTION_KEY_PREFIX, ""));
+
+ GroupKeyEncrypted grpKey = new GroupKeyEncrypted(INITIAL_KEY_ID, (byte[])val);
+
+ grpKeys.setGroupKeys(grpId, Collections.singletonList(grpKey));
+ }, true);
+ }
+
+ Serializable savedSegments = metastorage.read(REENCRYPTED_WAL_SEGMENTS);
+
+ if (savedSegments != null)
+ grpKeys.trackedWalSegments((Collection<CacheGroupEncryptionKeys.TrackedWalSegment>)savedSegments);
+
+ if (grpKeys.groupIds().isEmpty()) {
+ U.quietAndInfo(log, "Encryption keys loaded from metastore. " +
+ "[grps=" + F.concat(grpKeys.groupIds(), ",") +
+ ", masterKeyName=" + getSpi().getMasterKeyName() + ']');
}
}
catch (IgniteCheckedException e) {
@@ -730,20 +1020,47 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
/** {@inheritDoc} */
@Override public void onReadyForReadWrite(ReadWriteMetastorage metaStorage) throws IgniteCheckedException {
- synchronized (metaStorageMux) {
- this.metaStorage = metaStorage;
+ withMasterKeyChangeReadLock(() -> {
+ synchronized (metaStorageMux) {
+ this.metaStorage = metaStorage;
- writeToMetaStoreEnabled = true;
+ writeToMetaStoreEnabled = true;
- if (recoveryMasterKeyName)
- writeKeysToWal();
+ if (recoveryMasterKeyName)
+ writeKeysToWal();
- writeKeysToMetaStore(restoredFromWAL || recoveryMasterKeyName);
+ writeKeysToMetaStore(restoredFromWAL || recoveryMasterKeyName);
- restoredFromWAL = false;
+ restoredFromWAL = false;
+
+ recoveryMasterKeyName = false;
+ }
- recoveryMasterKeyName = false;
+ return null;
+ });
+
+ for (Map.Entry<Integer, Integer> entry : reencryptGroupsForced.entrySet()) {
+ int grpId = entry.getKey();
+
+ if (reencryptGroups.containsKey(grpId))
+ continue;
+
+ if (entry.getValue() != groupKey(grpId).unsignedId())
+ continue;
+
+ CacheGroupContext grp = ctx.cache().cacheGroup(grpId);
+
+ if (grp == null || !grp.affinityNode())
+ continue;
+
+ long[] offsets = pageScanner.pagesCount(grp);
+
+ reencryptGroups.put(grpId, offsets);
}
+
+ reencryptGroupsForced.clear();
+
+ startReencryption(reencryptGroups.keySet());
}
/** {@inheritDoc} */
@@ -768,6 +1085,37 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
}
/**
+ * Set reencryption status for partition.
+ *
+ * @param grp Cache group.
+ * @param partId Partition ID.
+ * @param idx Index of the last reencrypted page.
+ * @param total Total pages to be reencrypted.
+ */
+ public void setEncryptionState(CacheGroupContext grp, int partId, int idx, int total) {
+ // The last element of the array is used to store the status of the index partition.
+ long[] states = reencryptGroups.computeIfAbsent(grp.groupId(), v -> new long[grp.affinity().partitions() + 1]);
+
+ states[Math.min(partId, states.length - 1)] = ReencryptStateUtils.state(idx, total);
+ }
+
+ /**
+ * Get reencryption status for partition.
+ *
+ * @param grpId Cache group ID.
+ * @param partId Parttiion ID.
+ * @return Index and count of pages to be reencrypted.
+ */
+ public long getEncryptionState(int grpId, int partId) {
+ long[] states = reencryptGroups.get(grpId);
+
+ if (states == null)
+ return 0;
+
+ return states[Math.min(partId, states.length - 1)];
+ }
+
+ /**
* @param keyCnt Count of keys to generate.
* @return Future that will contain results of generation.
*/
@@ -812,6 +1160,51 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
}
/**
+ * @param grpIds Cache group IDs.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void startReencryption(Collection<Integer> grpIds) throws IgniteCheckedException {
+ for (int grpId : grpIds) {
+ IgniteInternalFuture<?> fut = pageScanner.schedule(grpId);
+
+ fut.listen(f -> {
+ if (f.isCancelled() || f.error() != null) {
+ log.warning("Reencryption " +
+ (f.isCancelled() ? "cancelled" : "failed") + " [grp=" + grpId + "]", f.error());
+
+ return;
+ }
+
+ withMasterKeyChangeReadLock(() -> {
+ synchronized (metaStorageMux) {
+ cleanupKeys(grpId);
+
+ reencryptGroups.remove(grpId);
+ }
+
+ return null;
+ });
+ });
+ }
+ }
+
+ /**
+ * @param grpId Cache group ID.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void cleanupKeys(int grpId) throws IgniteCheckedException {
+ Set<Integer> rmvKeyIds = grpKeys.removeUnusedKeys(grpId);
+
+ if (rmvKeyIds.isEmpty())
+ return;
+
+ writeGroupKeysToMetaStore(grpId, grpKeys.getAll(grpId));
+
+ if (log.isInfoEnabled())
+ log.info("Previous encryption keys were removed [grpId=" + grpId + ", keyIds=" + rmvKeyIds + "]");
+ }
+
+ /**
* Writes all unsaved grpEncKeys to metaStorage.
*
* @param writeAll {@code True} if force rewrite all keys.
@@ -821,11 +1214,55 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
if (writeAll)
metaStorage.write(MASTER_KEY_NAME_PREFIX, getSpi().getMasterKeyName());
- for (Map.Entry<Integer, Serializable> entry : grpEncKeys.entrySet()) {
- if (!writeAll && metaStorage.read(ENCRYPTION_KEY_PREFIX + entry.getKey()) != null)
+ if (!reencryptGroupsForced.isEmpty())
+ writeTrackedWalIdxsToMetaStore();
+
+ for (Integer grpId : grpKeys.groupIds()) {
+ if (!writeAll && !reencryptGroupsForced.containsKey(grpId) &&
+ metaStorage.read(ENCRYPTION_KEYS_PREFIX + grpId) != null)
continue;
- writeToMetaStore(entry.getKey(), getSpi().encryptKey(entry.getValue()));
+ writeGroupKeysToMetaStore(grpId, grpKeys.getAll(grpId));
+ }
+ }
+
+ /**
+ * Writes cache group encryption keys to metastore.
+ *
+ * @param grpId Cache group ID.
+ */
+ private void writeGroupKeysToMetaStore(int grpId, List<GroupKeyEncrypted> keys) throws IgniteCheckedException {
+ assert Thread.holdsLock(metaStorageMux);
+
+ if (metaStorage == null || !writeToMetaStoreEnabled || stopped)
+ return;
+
+ ctx.cache().context().database().checkpointReadLock();
+
+ try {
+ metaStorage.write(ENCRYPTION_KEYS_PREFIX + grpId, (Serializable)keys);
+ }
+ finally {
+ ctx.cache().context().database().checkpointReadUnlock();
+ }
+ }
+
+ /**
+ * Writes tracked (encrypted with previous encryption keys) WAL segments to metastore.
+ */
+ private void writeTrackedWalIdxsToMetaStore() throws IgniteCheckedException {
+ assert Thread.holdsLock(metaStorageMux);
+
+ if (metaStorage == null || !writeToMetaStoreEnabled || stopped)
+ return;
+
+ ctx.cache().context().database().checkpointReadLock();
+
+ try {
+ metaStorage.write(REENCRYPTED_WAL_SEGMENTS, (Serializable)grpKeys.trackedWalSegments());
+ }
+ finally {
+ ctx.cache().context().database().checkpointReadUnlock();
}
}
@@ -851,29 +1288,6 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
}
/**
- * Writes encryption key to metastore.
- *
- * @param grpId Group id.
- * @param encGrpKey Group encryption key.
- */
- private void writeToMetaStore(int grpId, byte[] encGrpKey) {
- if (metaStorage == null || !writeToMetaStoreEnabled)
- return;
-
- ctx.cache().context().database().checkpointReadLock();
-
- try {
- metaStorage.write(ENCRYPTION_KEY_PREFIX + grpId, encGrpKey);
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException("Failed to write cache group encryption key [grpId=" + grpId + ']', e);
- }
- finally {
- ctx.cache().context().database().checkpointReadUnlock();
- }
- }
-
- /**
* @param knownKeys Saved keys set.
* @return New keys for local cache groups.
*/
@@ -898,27 +1312,12 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
}
/**
- * @return Local encryption keys.
- */
- @Nullable private HashMap<Integer, byte[]> knownEncryptionKeys() {
- if (F.isEmpty(grpEncKeys))
- return null;
-
- HashMap<Integer, byte[]> knownKeys = new HashMap<>();
-
- for (Map.Entry<Integer, Serializable> entry : grpEncKeys.entrySet())
- knownKeys.put(entry.getKey(), getSpi().encryptKey(entry.getValue()));
-
- return knownKeys;
- }
-
- /**
* Generates required count of encryption keys.
*
* @param keyCnt Keys count.
* @return Tuple of collection with newly generated encryption keys and master key digest.
*/
- private T2<Collection<byte[]>, byte[]> createKeys(int keyCnt) {
+ T2<Collection<byte[]>, byte[]> createKeys(int keyCnt) {
return withMasterKeyChangeReadLock(() -> {
if (keyCnt == 0)
return new T2<>(Collections.emptyList(), getSpi().masterKeyDigest());
@@ -974,12 +1373,14 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
/** Writes the record with the master key name and all keys to WAL. */
private void writeKeysToWal() throws IgniteCheckedException {
- Map<Integer, byte[]> reencryptedKeys = new HashMap<>();
+ List<T2<Integer, GroupKeyEncrypted>> reencryptedKeys = new ArrayList<>();
- for (Map.Entry<Integer, Serializable> entry : grpEncKeys.entrySet())
- reencryptedKeys.put(entry.getKey(), getSpi().encryptKey(entry.getValue()));
+ for (int grpId : grpKeys.groupIds()) {
+ for (GroupKeyEncrypted grpKey : grpKeys.getAll(grpId))
+ reencryptedKeys.add(new T2<>(grpId, grpKey));
+ }
- MasterKeyChangeRecord rec = new MasterKeyChangeRecord(getSpi().getMasterKeyName(), reencryptedKeys);
+ MasterKeyChangeRecordV2 rec = new MasterKeyChangeRecordV2(getSpi().getMasterKeyName(), reencryptedKeys);
WALPointer ptr = ctx.cache().context().wal().log(rec);
@@ -991,7 +1392,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
*
* @param rec Record.
*/
- public void applyKeys(MasterKeyChangeRecord rec) {
+ public void applyKeys(MasterKeyChangeRecordV2 rec) {
assert !writeToMetaStoreEnabled && !ctx.state().clusterState().active();
log.info("Master key name loaded from WAL [masterKeyName=" + rec.getMasterKeyName() + ']');
@@ -999,8 +1400,13 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
try {
getSpi().setMasterKeyName(rec.getMasterKeyName());
- for (Map.Entry<Integer, byte[]> entry : rec.getGrpKeys().entrySet())
- grpEncKeys.put(entry.getKey(), getSpi().decryptKey(entry.getValue()));
+ Map<Integer, List<GroupKeyEncrypted>> keysMap = new HashMap<>();
+
+ for (T2<Integer, GroupKeyEncrypted> entry : rec.getGrpKeys())
+ keysMap.computeIfAbsent(entry.getKey(), v -> new ArrayList<>()).add(entry.getValue());
+
+ for (Map.Entry<Integer, List<GroupKeyEncrypted>> entry : keysMap.entrySet())
+ grpKeys.setGroupKeys(entry.getKey(), entry.getValue());
restoredFromWAL = true;
} catch (IgniteSpiException e) {
@@ -1009,12 +1415,24 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
}
/**
+ * Start reencryption using logical WAL record.
+ *
+ * @param rec Reencryption start logical record.
+ */
+ public void applyReencryptionStartRecord(ReencryptionStartRecord rec) {
+ assert !writeToMetaStoreEnabled;
+
+ for (Map.Entry<Integer, Byte> e : rec.groups().entrySet())
+ reencryptGroupsForced.put(e.getKey(), e.getValue() & 0xff);
+ }
+
+ /**
* Prepares master key change. Checks master key consistency.
*
* @param req Request.
* @return Result future.
*/
- private IgniteInternalFuture<MasterKeyChangeResult> prepareMasterKeyChange(MasterKeyChangeRequest req) {
+ private IgniteInternalFuture<EmptyResult> prepareMasterKeyChange(MasterKeyChangeRequest req) {
if (masterKeyChangeRequest != null) {
return new GridFinishedFuture<>(new IgniteException("Master key change was rejected. " +
"The previous change was not completed."));
@@ -1044,7 +1462,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
ctx.localNodeId() + ']', e));
}
- return new GridFinishedFuture<>(new MasterKeyChangeResult());
+ return new GridFinishedFuture<>(new EmptyResult());
}
/**
@@ -1054,14 +1472,14 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
* @param res Results.
* @param err Errors.
*/
- private void finishPrepareMasterKeyChange(UUID id, Map<UUID, MasterKeyChangeResult> res, Map<UUID, Exception> err) {
+ private void finishPrepareMasterKeyChange(UUID id, Map<UUID, EmptyResult> res, Map<UUID, Exception> err) {
if (!err.isEmpty()) {
if (masterKeyChangeRequest != null && masterKeyChangeRequest.requestId().equals(id))
masterKeyChangeRequest = null;
completeMasterKeyChangeFuture(id, err);
}
- else if (isCoordinator())
+ else if (U.isLocalNodeCoordinator(ctx.discovery()))
performMKChangeProc.start(id, masterKeyChangeRequest);
}
@@ -1071,7 +1489,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
* @param req Request.
* @return Result future.
*/
- private IgniteInternalFuture<MasterKeyChangeResult> performMasterKeyChange(MasterKeyChangeRequest req) {
+ private IgniteInternalFuture<EmptyResult> performMasterKeyChange(MasterKeyChangeRequest req) {
if (masterKeyChangeRequest == null || !masterKeyChangeRequest.equals(req))
return new GridFinishedFuture<>(new IgniteException("Unknown master key change was rejected."));
@@ -1089,7 +1507,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
masterKeyDigest = req.digest();
- return new GridFinishedFuture<>(new MasterKeyChangeResult());
+ return new GridFinishedFuture<>(new EmptyResult());
}
/**
@@ -1099,7 +1517,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
* @param res Results.
* @param err Errors.
*/
- private void finishPerformMasterKeyChange(UUID id, Map<UUID, MasterKeyChangeResult> res, Map<UUID, Exception> err) {
+ private void finishPerformMasterKeyChange(UUID id, Map<UUID, EmptyResult> res, Map<UUID, Exception> err) {
completeMasterKeyChangeFuture(id, err);
}
@@ -1130,29 +1548,16 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
* @param msg Error message.
*/
private void cancelFutures(String msg) {
+ assert Thread.holdsLock(opsMux);
+
for (GenerateEncryptionKeyFuture fut : genEncKeyFuts.values())
fut.onDone(new IgniteFutureCancelledException(msg));
if (masterKeyChangeFut != null && !masterKeyChangeFut.isDone())
masterKeyChangeFut.onDone(new IgniteFutureCancelledException(msg));
- }
-
- /**
- * Checks whether local node is coordinator. Nodes that are leaving or failed
- * (but are still in topology) are removed from search.
- *
- * @return {@code true} if local node is coordinator.
- */
- private boolean isCoordinator() {
- DiscoverySpi spi = ctx.discovery().getInjectedDiscoverySpi();
-
- if (spi instanceof TcpDiscoverySpi)
- return ((TcpDiscoverySpi)spi).isLocalNodeCoordinator();
- else {
- ClusterNode crd = U.oldest(ctx.discovery().aliveServerNodes(), null);
- return crd != null && F.eq(ctx.localNodeId(), crd.id());
- }
+ if (grpKeyChangeProc != null)
+ grpKeyChangeProc.cancel(msg);
}
/** @return {@code True} if the master key change process in progress. */
@@ -1167,7 +1572,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
*
* @return Digest of last changed master key or {@code null} if master key was not changed.
*/
- public byte[] masterKeyDigest() {
+ @Nullable public byte[] masterKeyDigest() {
return masterKeyDigest;
}
@@ -1175,7 +1580,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
* @param c Callable to run with master key change read lock.
* @return Computed result.
*/
- private <T> T withMasterKeyChangeReadLock(Callable<T> c) {
+ <T> T withMasterKeyChangeReadLock(Callable<T> c) {
masterKeyChangeLock.readLock().lock();
try {
@@ -1339,24 +1744,38 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
}
/** */
- private static class MasterKeyChangeResult implements Serializable {
+ protected static class EmptyResult implements Serializable {
/** Serial version uid. */
private static final long serialVersionUID = 0L;
}
/** */
- public static class NodeEncryptionKeys implements Serializable {
+ protected static class NodeEncryptionKeys implements Serializable {
/** */
private static final long serialVersionUID = 0L;
/** */
- NodeEncryptionKeys(Map<Integer, byte[]> knownKeys, Map<Integer, byte[]> newKeys, byte[] masterKeyDigest) {
- this.knownKeys = knownKeys;
+ NodeEncryptionKeys(
+ HashMap<Integer, List<GroupKeyEncrypted>> knownKeysWithIds,
+ Map<Integer, byte[]> newKeys,
+ byte[] masterKeyDigest
+ ) {
this.newKeys = newKeys;
this.masterKeyDigest = masterKeyDigest;
+
+ if (F.isEmpty(knownKeysWithIds))
+ return;
+
+ // To be able to join the old cluster.
+ knownKeys = U.newHashMap(knownKeysWithIds.size());
+
+ for (Map.Entry<Integer, List<GroupKeyEncrypted>> entry : knownKeysWithIds.entrySet())
+ knownKeys.put(entry.getKey(), entry.getValue().get(0).key());
+
+ this.knownKeysWithIds = knownKeysWithIds;
}
- /** Known i.e. stored in {@code ReadWriteMetastorage} keys from node. */
+ /** Known i.e. stored in {@code ReadWriteMetastorage} keys from node (in compatible format). */
Map<Integer, byte[]> knownKeys;
/** New keys i.e. keys for a local statically configured caches. */
@@ -1364,6 +1783,9 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
/** Master key digest. */
byte[] masterKeyDigest;
+
+ /** Known i.e. stored in {@code ReadWriteMetastorage} keys from node. */
+ Map<Integer, List<GroupKeyEncrypted>> knownKeysWithIds;
}
/** */
@@ -1423,13 +1845,13 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
}
}
- /** Master key change future. */
- private static class MasterKeyChangeFuture extends GridFutureAdapter<Void> {
+ /** Key change future. */
+ protected static class KeyChangeFuture extends GridFutureAdapter<Void> {
/** Request ID. */
private final UUID id;
/** @param id Request ID. */
- private MasterKeyChangeFuture(UUID id) {
+ KeyChangeFuture(UUID id) {
this.id = id;
}
@@ -1440,7 +1862,7 @@ public class GridEncryptionManager extends GridManagerAdapter<EncryptionSpi> imp
/** {@inheritDoc} */
@Override public String toString() {
- return S.toString(MasterKeyChangeFuture.class, this);
+ return S.toString(KeyChangeFuture.class, this);
}
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKey.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKey.java
new file mode 100644
index 0000000..5182a18
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKey.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.encryption;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Cache group encryption key with identifier.
+ */
+public class GroupKey {
+ /** Encryption key ID. */
+ private final int id;
+
+ /** Encryption key. */
+ private final Serializable key;
+
+ /**
+ * @param id Encryption key ID.
+ * @param key Encryption key.
+ */
+ public GroupKey(int id, Serializable key) {
+ this.id = id;
+ this.key = key;
+ }
+
+ /**
+ * @return Encryption key ID.
+ */
+ public byte id() {
+ return (byte)id;
+ }
+
+ /**
+ * @return Unsigned encryption key ID.
+ */
+ public int unsignedId() {
+ return id & 0xff;
+ }
+
+ /**
+ * @return Encryption key.
+ */
+ public Serializable key() {
+ return key;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ GroupKey grpKey = (GroupKey)o;
+
+ return id == grpKey.id && Objects.equals(key, grpKey.key);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hash(id, key);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "GroupKey [id=" + id + ']';
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyChangeProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyChangeProcess.java
new file mode 100644
index 0000000..3e12351
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyChangeProcess.java
@@ -0,0 +1,356 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.encryption;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteFeatures;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.encryption.GridEncryptionManager.EmptyResult;
+import org.apache.ignite.internal.managers.encryption.GridEncryptionManager.KeyChangeFuture;
+import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.util.distributed.DistributedProcess;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
+import org.apache.ignite.internal.util.future.IgniteFutureImpl;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteFutureCancelledException;
+
+import static org.apache.ignite.internal.IgniteFeatures.CACHE_GROUP_KEY_CHANGE;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CACHE_GROUP_KEY_CHANGE_FINISH;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CACHE_GROUP_KEY_CHANGE_PREPARE;
+
+/**
+ * A two-phase distributed process that rotates the encryption keys of specified cache groups and initiates
+ * re-encryption of those cache groups.
+ */
+class GroupKeyChangeProcess {
+ /** Grid kernal context. */
+ private final GridKernalContext ctx;
+
+ /** Cache group encyption key change prepare phase. */
+ private final DistributedProcess<ChangeCacheEncryptionRequest, EmptyResult> prepareGKChangeProc;
+
+ /** Cache group encyption key change perform phase. */
+ private final DistributedProcess<ChangeCacheEncryptionRequest, EmptyResult> performGKChangeProc;
+
+ /** Group encryption keys. */
+ private final CacheGroupEncryptionKeys keys;
+
+ /** Cache group key change future. */
+ private volatile GroupKeyChangeFuture fut;
+
+ /** Cache group key change request. */
+ private volatile ChangeCacheEncryptionRequest req;
+
+ /**
+ * @param ctx Grid kernal context.
+ * @param keys Cache group encryption keys.
+ */
+ GroupKeyChangeProcess(GridKernalContext ctx, CacheGroupEncryptionKeys keys) {
+ this.ctx = ctx;
+ this.keys = keys;
+
+ prepareGKChangeProc =
+ new DistributedProcess<>(ctx, CACHE_GROUP_KEY_CHANGE_PREPARE, this::prepare, this::finishPrepare);
+ performGKChangeProc =
+ new DistributedProcess<>(ctx, CACHE_GROUP_KEY_CHANGE_FINISH, this::perform, this::finishPerform);
+ }
+
+ /**
+ * @return {@code True} if operation is still in progress.
+ */
+ public boolean inProgress() {
+ return req != null;
+ }
+
+ /**
+ * @param msg Error message.
+ */
+ public void cancel(String msg) {
+ GridFutureAdapter<Void> keyChangeFut = fut;
+
+ if (keyChangeFut != null && !keyChangeFut.isDone())
+ keyChangeFut.onDone(new IgniteFutureCancelledException(msg));
+ }
+
+ /**
+ * Starts cache group encryption key change process.
+ *
+ * @param cacheOrGrpNames Cache or group names.
+ */
+ public IgniteFuture<Void> start(Collection<String> cacheOrGrpNames) {
+ if (ctx.clientNode())
+ throw new UnsupportedOperationException("Client and daemon nodes can not perform this operation.");
+
+ if (!IgniteFeatures.allNodesSupports(ctx.grid().cluster().nodes(), CACHE_GROUP_KEY_CHANGE))
+ throw new IllegalStateException("Not all nodes in the cluster support this operation.");
+
+ if (!ctx.state().clusterState().state().active())
+ throw new IgniteException("Operation was rejected. The cluster is inactive.");
+
+ IgniteInternalFuture<Void> fut0 = fut;
+
+ if (fut0 != null && !fut0.isDone()) {
+ return new IgniteFinishedFutureImpl<>(new IgniteException("Cache group key change was rejected. " +
+ "The previous change was not completed."));
+ }
+
+ int[] grpIds = new int[cacheOrGrpNames.size()];
+ byte[] keyIds = new byte[grpIds.length];
+
+ int n = 0;
+
+ for (String cacheOrGroupName : cacheOrGrpNames) {
+ CacheGroupDescriptor grpDesc = ctx.cache().cacheGroupDescriptor(CU.cacheId(cacheOrGroupName));
+
+ if (grpDesc == null) {
+ DynamicCacheDescriptor cacheDesc = ctx.cache().cacheDescriptor(cacheOrGroupName);
+
+ if (cacheDesc == null) {
+ throw new IgniteException("Cache group key change was rejected. " +
+ "Cache or group \"" + cacheOrGroupName + "\" doesn't exists");
+ }
+
+ int grpId = cacheDesc.groupId();
+
+ grpDesc = ctx.cache().cacheGroupDescriptor(grpId);
+
+ if (grpDesc.sharedGroup()) {
+ throw new IgniteException("Cache group key change was rejected. " +
+ "Cache or group \"" + cacheOrGroupName + "\" is a part of group \"" +
+ grpDesc.groupName() + "\". Provide group name instead of cache name for shared groups.");
+ }
+ }
+
+ if (!grpDesc.config().isEncryptionEnabled()) {
+ throw new IgniteException("Cache group key change was rejected. " +
+ "Cache or group \"" + cacheOrGroupName + "\" is not encrypted.");
+ }
+
+ if (ctx.encryption().reencryptionInProgress(grpDesc.groupId())) {
+ throw new IgniteException("Cache group key change was rejected. " +
+ "Cache group reencryption is in progress [grp=" + cacheOrGroupName + "]");
+ }
+
+ grpIds[n] = grpDesc.groupId();
+ keyIds[n] = (byte)(ctx.encryption().groupKey(grpDesc.groupId()).unsignedId() + 1);
+
+ n += 1;
+ }
+
+ T2<Collection<byte[]>, byte[]> keysAndDigest = ctx.encryption().createKeys(grpIds.length);
+
+ ChangeCacheEncryptionRequest req = new ChangeCacheEncryptionRequest(
+ grpIds,
+ keysAndDigest.get1().toArray(new byte[grpIds.length][]),
+ keyIds,
+ keysAndDigest.get2()
+ );
+
+ fut = new GroupKeyChangeFuture(req);
+
+ prepareGKChangeProc.start(req.requestId(), req);
+
+ return new IgniteFutureImpl<>(fut);
+ }
+
+ /**
+ * Validates existing keys.
+ *
+ * @param req Request.
+ * @return Result future.
+ */
+ private IgniteInternalFuture<EmptyResult> prepare(ChangeCacheEncryptionRequest req) {
+ if (ctx.clientNode())
+ return new GridFinishedFuture<>();
+
+ if (inProgress()) {
+ return new GridFinishedFuture<>(new IgniteException("Cache group key change was rejected. " +
+ "The previous change was not completed."));
+ }
+
+ this.req = req;
+
+ try {
+ for (int i = 0; i < req.groupIds().length; i++) {
+ int grpId = req.groupIds()[i];
+ int keyId = req.keyIds()[i] & 0xff;
+
+ if (ctx.encryption().reencryptionInProgress(grpId)) {
+ return new GridFinishedFuture<>(new IgniteException("Cache group key change was rejected. " +
+ "Cache group reencryption is in progress [grpId=" + grpId + "]"));
+ }
+
+ List<Integer> keyIds = ctx.encryption().groupKeyIds(grpId);
+
+ if (keyIds == null) {
+ return new GridFinishedFuture<>(new IgniteException("Cache group key change was rejected." +
+ "Encrypted cache group not found [grpId=" + grpId + "]"));
+ }
+
+ GroupKey currKey = ctx.encryption().groupKey(grpId);
+
+ for (int locKeyId : keyIds) {
+ if (locKeyId != keyId)
+ continue;
+
+ Long walSegment = keys.reservedSegment(grpId, keyId);
+
+ // Can overwrite inactive key if it was added during prepare phase.
+ if (walSegment == null && currKey.id() != (byte)keyId)
+ continue;
+
+ return new GridFinishedFuture<>(
+ new IgniteException("Cache group key change was rejected. Cannot add new key identifier, " +
+ "it's already present. There existing WAL segments that encrypted with this key [" +
+ "grpId=" + grpId + ", newId=" + keyId + ", currId=" + currKey.unsignedId() +
+ ", walSegment=" + walSegment + "]."));
+ }
+ }
+
+ return ctx.encryption().withMasterKeyChangeReadLock(() -> {
+ if (!Arrays.equals(ctx.config().getEncryptionSpi().masterKeyDigest(), req.masterKeyDigest())) {
+ return new GridFinishedFuture<>(new IgniteException("Cache group key change was rejected. " +
+ "Master key has been changed."));
+ }
+
+ for (int i = 0; i < req.groupIds().length; i++) {
+ // Save the new key as inactive, because the master key may change later
+ // and there will be no way to decrypt the received keys.
+ GroupKeyEncrypted grpKey = new GroupKeyEncrypted(req.keyIds()[i] & 0xff, req.keys()[i]);
+
+ ctx.encryption().addGroupKey(req.groupIds()[i], grpKey);
+ }
+
+ return new GridFinishedFuture<>(new EmptyResult());
+ });
+
+ }
+ catch (Exception e) {
+ return new GridFinishedFuture<>(new IgniteException("Cache group key change was rejected [nodeId=" +
+ ctx.localNodeId() + ']', e));
+ }
+ }
+
+ /**
+ * Starts group key change if there are no errors.
+ *
+ * @param id Request id.
+ * @param res Results.
+ * @param err Errors.
+ */
+ private void finishPrepare(UUID id, Map<UUID, EmptyResult> res, Map<UUID, Exception> err) {
+ if (!err.isEmpty()) {
+ if (req != null && req.requestId().equals(id))
+ req = null;
+
+ completeFuture(id, err, fut);
+ }
+ else if (U.isLocalNodeCoordinator(ctx.discovery()))
+ performGKChangeProc.start(id, req);
+ }
+
+ /**
+ * Sets new encrpytion key as active (for writing) and starts background reencryption.
+ *
+ * @param req Request.
+ * @return Result future.
+ */
+ private IgniteInternalFuture<EmptyResult> perform(ChangeCacheEncryptionRequest req) {
+ if (this.req == null || !this.req.equals(req))
+ return new GridFinishedFuture<>(new IgniteException("Unknown cache group key change was rejected."));
+
+ try {
+ if (!ctx.state().clusterState().state().active())
+ throw new IgniteException("Cache group key change was rejected. The cluster is inactive.");
+
+ if (!ctx.clientNode())
+ ctx.encryption().changeCacheGroupKeyLocal(req.groupIds(), req.keyIds(), req.keys());
+ } catch (Exception e) {
+ return new GridFinishedFuture<>(e);
+ } finally {
+ this.req = null;
+ }
+
+ return new GridFinishedFuture<>(new EmptyResult());
+ }
+
+ /**
+ * Finishes cache encryption key rotation.
+ *
+ * @param id Request id.
+ * @param res Results.
+ * @param err Errors.
+ */
+ private void finishPerform(UUID id, Map<UUID, EmptyResult> res, Map<UUID, Exception> err) {
+ completeFuture(id, err, fut);
+ }
+
+ /**
+ * @param reqId Request id.
+ * @param err Exception.
+ * @param fut Key change future.
+ * @return {@code True} if future was completed by this call.
+ */
+ private boolean completeFuture(UUID reqId, Map<UUID, Exception> err, GroupKeyChangeFuture fut) {
+ boolean isInitiator = fut != null && fut.id().equals(reqId);
+
+ if (!isInitiator || fut.isDone())
+ return false;
+
+ return !F.isEmpty(err) ? fut.onDone(F.firstValue(err)) : fut.onDone();
+ }
+
+ /** Cache group key change future. */
+ private static class GroupKeyChangeFuture extends KeyChangeFuture {
+ /** Request. */
+ private final ChangeCacheEncryptionRequest req;
+
+ /**
+ * @param req Request.
+ */
+ GroupKeyChangeFuture(ChangeCacheEncryptionRequest req) {
+ super(req.requestId());
+
+ this.req = req;
+ }
+
+ /** @return Topology version. */
+ public ChangeCacheEncryptionRequest request() {
+ return req;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(GroupKeyChangeFuture.class, this);
+ }
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/EncryptionMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyEncrypted.java
similarity index 55%
copy from modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/EncryptionMXBeanImpl.java
copy to modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyEncrypted.java
index 0270222..6b2ed05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/EncryptionMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyEncrypted.java
@@ -17,28 +17,41 @@
package org.apache.ignite.internal.managers.encryption;
-import org.apache.ignite.internal.GridKernalContextImpl;
-import org.apache.ignite.mxbean.EncryptionMXBean;
+import java.io.Serializable;
/**
- * Encryption features MBean.
+ * Cache group encryption key with identifier. Key is encrypted.
*/
-public class EncryptionMXBeanImpl implements EncryptionMXBean {
- /** Encryption manager. */
- private final GridEncryptionManager encryptionMgr;
+public class GroupKeyEncrypted implements Serializable {
+ /** Serial version UID. */
+ private static final long serialVersionUID = 0L;
- /** @param ctx Context. */
- public EncryptionMXBeanImpl(GridKernalContextImpl ctx) {
- encryptionMgr = ctx.encryption();
+ /** Encryption key ID. */
+ private final int id;
+
+ /** Encryption key. */
+ private final byte[] key;
+
+ /**
+ * @param id Encryption key ID.
+ * @param key Encryption key.
+ */
+ public GroupKeyEncrypted(int id, byte[] key) {
+ this.id = id;
+ this.key = key;
}
- /** {@inheritDoc} */
- @Override public String getMasterKeyName() {
- return encryptionMgr.getMasterKeyName();
+ /**
+ * @return Encryption key ID.
+ */
+ public int id() {
+ return id;
}
- /** {@inheritDoc} */
- @Override public void changeMasterKey(String masterKeyName) {
- encryptionMgr.changeMasterKey(masterKeyName).get();
+ /**
+ * @return Encryption key.
+ */
+ public byte[] key() {
+ return key;
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/EncryptionMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/ReencryptStateUtils.java
similarity index 55%
copy from modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/EncryptionMXBeanImpl.java
copy to modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/ReencryptStateUtils.java
index 0270222..37292d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/EncryptionMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/ReencryptStateUtils.java
@@ -17,28 +17,30 @@
package org.apache.ignite.internal.managers.encryption;
-import org.apache.ignite.internal.GridKernalContextImpl;
-import org.apache.ignite.mxbean.EncryptionMXBean;
-
-/**
- * Encryption features MBean.
- */
-public class EncryptionMXBeanImpl implements EncryptionMXBean {
- /** Encryption manager. */
- private final GridEncryptionManager encryptionMgr;
-
- /** @param ctx Context. */
- public EncryptionMXBeanImpl(GridKernalContextImpl ctx) {
- encryptionMgr = ctx.encryption();
+/** */
+public class ReencryptStateUtils {
+ /**
+ * @param idx Index of the last reencrypted page.
+ * @param total Total pages to be reencrypted.
+ * @return Reencryption status.
+ */
+ public static long state(int idx, int total) {
+ return ((long)idx) << Integer.SIZE | (total & 0xffffffffL);
}
- /** {@inheritDoc} */
- @Override public String getMasterKeyName() {
- return encryptionMgr.getMasterKeyName();
+ /**
+ * @param state Reencryption status.
+ * @return Index of the last reencrypted page.
+ */
+ public static int pageIndex(long state) {
+ return (int)(state >> Integer.SIZE);
}
- /** {@inheritDoc} */
- @Override public void changeMasterKey(String masterKeyName) {
- encryptionMgr.changeMasterKey(masterKeyName).get();
+ /**
+ * @param state Reencryption status.
+ * @return Total pages to be reencrypted.
+ */
+ public static int pageCount(long state) {
+ return (int)state;
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index 40e70b3..cc183bf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -167,6 +167,11 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
public void notchLastCheckpointPtr(WALPointer ptr);
/**
+ * @return Current segment index.
+ */
+ public long currentSegment();
+
+ /**
* @return Total number of segments in the WAL archive.
*/
public int walArchiveSegments();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MasterKeyChangeRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MasterKeyChangeRecord.java
index 583ff73..dba07b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MasterKeyChangeRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MasterKeyChangeRecord.java
@@ -24,7 +24,10 @@ import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType
/**
* Logical record that stores encryption keys. Written to the WAL on the master key change.
+ *
+ * @deprecated Replaced by MasterKeyChangeRecordV2.
*/
+@Deprecated
public class MasterKeyChangeRecord extends WALRecord {
/** Master key name. */
private final String masterKeyName;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MasterKeyChangeRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MasterKeyChangeRecordV2.java
similarity index 70%
copy from modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MasterKeyChangeRecord.java
copy to modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MasterKeyChangeRecordV2.java
index 583ff73..10145d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MasterKeyChangeRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MasterKeyChangeRecordV2.java
@@ -17,26 +17,27 @@
package org.apache.ignite.internal.pagemem.wal.record;
-import java.util.Map;
-import java.util.Map.Entry;
+import java.util.List;
+import org.apache.ignite.internal.managers.encryption.GroupKeyEncrypted;
+import org.apache.ignite.internal.util.typedef.T2;
-import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MASTER_KEY_CHANGE_RECORD;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MASTER_KEY_CHANGE_RECORD_V2;
/**
* Logical record that stores encryption keys. Written to the WAL on the master key change.
*/
-public class MasterKeyChangeRecord extends WALRecord {
+public class MasterKeyChangeRecordV2 extends WALRecord {
/** Master key name. */
private final String masterKeyName;
/** Group keys encrypted by the master key. */
- private final Map<Integer, byte[]> grpKeys;
+ private final List<T2<Integer, GroupKeyEncrypted>> grpKeys;
/**
* @param masterKeyName Master key name.
* @param grpKeys Encrypted group keys.
*/
- public MasterKeyChangeRecord(String masterKeyName, Map<Integer, byte[]> grpKeys) {
+ public MasterKeyChangeRecordV2(String masterKeyName, List<T2<Integer, GroupKeyEncrypted>> grpKeys) {
this.masterKeyName = masterKeyName;
this.grpKeys = grpKeys;
}
@@ -47,22 +48,23 @@ public class MasterKeyChangeRecord extends WALRecord {
}
/** @return Encrypted group keys. */
- public Map<Integer, byte[]> getGrpKeys() {
+ public List<T2<Integer, GroupKeyEncrypted>> getGrpKeys() {
return grpKeys;
}
/** {@inheritDoc} */
@Override public RecordType type() {
- return MASTER_KEY_CHANGE_RECORD;
+ return MASTER_KEY_CHANGE_RECORD_V2;
}
/** @return Record data size. */
public int dataSize() {
- int size = /*Master key name length*/4 + masterKeyName.getBytes().length + /*Group keys map size*/4;
+ int size = /*Master key name length*/4 + masterKeyName.getBytes().length + /*list size*/4;
- for (Entry<Integer, byte[]> entry : grpKeys.entrySet())
- size += /*grpId*/4 + /*grp key size*/4 + entry.getValue().length;
+ for (T2<Integer, GroupKeyEncrypted> entry : grpKeys)
+ size += /*grpId*/4 + /*grp key size*/4 + /*grp key id size*/1 + entry.get2().key().length;
return size;
}
}
+
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/ReencryptionStartRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/ReencryptionStartRecord.java
new file mode 100644
index 0000000..c8b08d2
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/ReencryptionStartRecord.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.pagemem.wal.record;
+
+import java.util.Map;
+
+/**
+ * Logical record to restart reencryption with the latest encryption key.
+ */
+public class ReencryptionStartRecord extends WALRecord {
+ /** Map of reencrypted cache groups with encryption key identifiers. */
+ private final Map<Integer, Byte> grps;
+
+ /**
+ * @param grps Map of reencrypted cache groups with encryption key identifiers.
+ */
+ public ReencryptionStartRecord(Map<Integer, Byte> grps) {
+ this.grps = grps;
+ }
+
+ /**
+ * @return Map of reencrypted cache groups with encryption key identifiers.
+ */
+ public Map<Integer, Byte> groups() {
+ return grps;
+ }
+
+ /** {@inheritDoc} */
+ @Override public RecordType type() {
+ return RecordType.REENCRYPTION_START_RECORD;
+ }
+
+ /** @return Record data size. */
+ public int dataSize() {
+ return 4 + ((/*grpId*/4 + /*keyId*/1) * grps.size());
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
index 8a01467..f07b71a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
@@ -234,7 +234,25 @@ public abstract class WALRecord {
TRACKING_PAGE_REPAIR_DELTA(61, PHYSICAL),
/** Atomic out-of-order update. */
- OUT_OF_ORDER_UPDATE(62, LOGICAL);
+ OUT_OF_ORDER_UPDATE(62, LOGICAL),
+
+ /** Encrypted WAL-record. */
+ ENCRYPTED_RECORD_V2(63, PHYSICAL),
+
+ /** Ecnrypted data record. */
+ ENCRYPTED_DATA_RECORD_V2(64, LOGICAL),
+
+ /** Master key change record containing multiple keys for single cache group. */
+ MASTER_KEY_CHANGE_RECORD_V2(65, LOGICAL),
+
+ /** Logical record to restart reencryption with the latest encryption key. */
+ REENCRYPTION_START_RECORD(66, LOGICAL),
+
+ /** Partition meta page delta record includes encryption status data. */
+ PARTITION_META_PAGE_DELTA_RECORD_V3(67, PHYSICAL),
+
+ /** Index meta page delta record includes encryption status data. */
+ INDEX_META_PAGE_DELTA_RECORD(68, PHYSICAL);
/** Index for serialization. Should be consistent throughout all versions. */
private final int idx;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateIndexDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateIndexDataRecord.java
new file mode 100644
index 0000000..e2f83c0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateIndexDataRecord.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.pagemem.wal.record.delta;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIOV2;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Meta page delta record, includes encryption status data.
+ */
+public class MetaPageUpdateIndexDataRecord extends PageDeltaRecord {
+ /** Index of the last reencrypted page. */
+ private int encryptPageIdx;
+
+ /** Total pages to be reencrypted. */
+ private int encryptPageCnt;
+
+ /**
+ * @param grpId Cache group ID.
+ * @param pageId Page ID.
+ * @param encryptPageIdx Index of the last reencrypted page.
+ * @param encryptPageCnt Total pages to be reencrypted.
+ */
+ public MetaPageUpdateIndexDataRecord(int grpId, long pageId, int encryptPageIdx, int encryptPageCnt) {
+ super(grpId, pageId);
+
+ this.encryptPageIdx = encryptPageIdx;
+ this.encryptPageCnt = encryptPageCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
+ if (PageIO.getVersion(pageAddr) < 2)
+ ((PageMetaIOV2)PageMetaIOV2.VERSIONS.latest()).upgradePage(pageAddr);
+
+ PageMetaIOV2 io = (PageMetaIOV2)PageMetaIOV2.VERSIONS.forPage(pageAddr);
+
+ io.setEncryptedPageIndex(pageAddr, encryptPageIdx);
+ io.setEncryptedPageCount(pageAddr, encryptPageCnt);
+ }
+
+ /** {@inheritDoc} */
+ @Override public RecordType type() {
+ return RecordType.INDEX_META_PAGE_DELTA_RECORD;
+ }
+
+ /**
+ * @param in Input.
+ */
+ public MetaPageUpdateIndexDataRecord(DataInput in) throws IOException {
+ super(in.readInt(), in.readLong());
+
+ encryptPageIdx = in.readInt();
+ encryptPageCnt = in.readInt();
+ }
+
+ /**
+ * @param buf Buffer.
+ */
+ public void toBytes(ByteBuffer buf) {
+ buf.putInt(groupId());
+ buf.putLong(pageId());
+
+ buf.putInt(encryptionPagesIndex());
+ buf.putInt(encryptionPagesCount());
+ }
+
+ /**
+ * @return Index of the last reencrypted page.
+ */
+ private int encryptionPagesIndex() {
+ return encryptPageIdx;
+ }
+
+ /**
+ * @return Total pages to be reencrypted.
+ */
+ private int encryptionPagesCount() {
+ return encryptPageCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(MetaPageUpdateIndexDataRecord.class, this, "partId",
+ PageIdUtils.partId(pageId()), "super", super.toString());
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecordV3.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecordV3.java
new file mode 100644
index 0000000..c7a7157
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecordV3.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.pagemem.wal.record.delta;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIOV3;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Partition meta page delta record includes encryption status data.
+ */
+public class MetaPageUpdatePartitionDataRecordV3 extends MetaPageUpdatePartitionDataRecordV2 {
+ /** Index of the last reencrypted page. */
+ private int encryptedPageIdx;
+
+ /** Total pages to be reencrypted. */
+ private int encryptedPageCnt;
+
+ /**
+ * @param grpId Group id.
+ * @param pageId Page id.
+ * @param updateCntr Update counter.
+ * @param globalRmvId Global remove id.
+ * @param partSize Partition size.
+ * @param cntrsPageId Cntrs page id.
+ * @param state State.
+ * @param allocatedIdxCandidate Allocated index candidate.
+ * @param link Link.
+ * @param encryptedPageIdx Index of the last reencrypted page.
+ * @param encryptedPageCnt Total pages to be reencrypted.
+ */
+ public MetaPageUpdatePartitionDataRecordV3(
+ int grpId,
+ long pageId,
+ long updateCntr,
+ long globalRmvId,
+ int partSize,
+ long cntrsPageId,
+ byte state,
+ int allocatedIdxCandidate,
+ long link,
+ int encryptedPageIdx,
+ int encryptedPageCnt) {
+ super(grpId, pageId, updateCntr, globalRmvId, partSize, cntrsPageId, state, allocatedIdxCandidate, link);
+
+ this.encryptedPageIdx = encryptedPageIdx;
+ this.encryptedPageCnt = encryptedPageCnt;
+ }
+
+ /**
+ * @param in Input.
+ */
+ public MetaPageUpdatePartitionDataRecordV3(DataInput in) throws IOException {
+ super(in);
+
+ encryptedPageIdx = in.readInt();
+ encryptedPageCnt = in.readInt();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
+ super.applyDelta(pageMem, pageAddr);
+
+ PagePartitionMetaIOV3 io = (PagePartitionMetaIOV3)PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
+
+ io.setEncryptedPageIndex(pageAddr, encryptedPageIdx);
+ io.setEncryptedPageCount(pageAddr, encryptedPageCnt);
+ }
+
+ /**
+ * @return Index of the last reencrypted page.
+ */
+ public int encryptedPageIndex() {
+ return encryptedPageIdx;
+ }
+
+ /**
+ * @return Total pages to be reencrypted.
+ */
+ public int encryptedPageCount() {
+ return encryptedPageCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void toBytes(ByteBuffer buf) {
+ super.toBytes(buf);
+
+ buf.putInt(encryptedPageIndex());
+ buf.putInt(encryptedPageCount());
+ }
+
+ /** {@inheritDoc} */
+ @Override public RecordType type() {
+ return RecordType.PARTITION_META_PAGE_DELTA_RECORD_V3;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(MetaPageUpdatePartitionDataRecordV3.class, this, "partId",
+ PageIdUtils.partId(pageId()), "super", super.toString());
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsImpl.java
index 15a989d..75fdd15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsImpl.java
@@ -25,10 +25,15 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.internal.managers.encryption.GridEncryptionManager;
+import org.apache.ignite.internal.managers.encryption.ReencryptStateUtils;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.store.PageStore;
import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -39,6 +44,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.Grid
import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.metric.MetricRegistry;
import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric;
import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric;
@@ -175,6 +181,16 @@ public class CacheGroupMetricsImpl {
mreg.register("TotalAllocatedSize",
this::getTotalAllocatedSize,
"Total size of memory allocated for group, in bytes.");
+
+ if (ctx.config().isEncryptionEnabled()) {
+ mreg.register("ReencryptionFinished",
+ () -> !ctx.shared().kernalContext().encryption().reencryptionInProgress(ctx.groupId()),
+ "The flag indicates whether reencryption is finished or not.");
+
+ mreg.register("ReencryptionPagesLeft",
+ this::getPagesLeftForReencryption,
+ "Number of pages left for reencryption.");
+ }
}
/** */
@@ -486,6 +502,40 @@ public class CacheGroupMetricsImpl {
return sparseStorageSize == null ? 0 : sparseStorageSize.value();
}
+ /** */
+ public long getPagesLeftForReencryption() {
+ if (!ctx.shared().kernalContext().encryption().reencryptionInProgress(ctx.groupId()))
+ return 0;
+
+ long pagesLeft = 0;
+
+ FilePageStoreManager mgr = (FilePageStoreManager)ctx.shared().pageStore();
+
+ GridEncryptionManager encMgr = ctx.shared().kernalContext().encryption();
+
+ try {
+ for (int p = 0; p < ctx.affinity().partitions(); p++) {
+ PageStore pageStore = mgr.getStore(ctx.groupId(), p);
+
+ if (!pageStore.exists())
+ continue;
+
+ long state = encMgr.getEncryptionState(ctx.groupId(), p);
+
+ pagesLeft += ReencryptStateUtils.pageCount(state) - ReencryptStateUtils.pageIndex(state);
+ }
+
+ long state = encMgr.getEncryptionState(ctx.groupId(), PageIdAllocator.INDEX_PARTITION);
+
+ pagesLeft += ReencryptStateUtils.pageCount(state) - ReencryptStateUtils.pageIndex(state);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+
+ return pagesLeft;
+ }
+
/** Removes all metric for cache group. */
public void remove() {
if (ctx.shared().kernalContext().isStopping())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index cb0daed..e260983 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -2021,9 +2021,13 @@ public class GridCacheUtils {
* @return Page size without encryption overhead.
*/
public static int encryptedPageSize(int pageSize, EncryptionSpi encSpi) {
+ // If encryption is enabled, a space of one encryption block is reserved to store CRC and encryption key ID.
+ // If encryption is disabled, NoopEncryptionSPI with a zero encryption block size is used.
+ assert encSpi.blockSize() >= /* CRC */ 4 + /* Key ID */ 1 || encSpi.blockSize() == 0;
+
return pageSize
- (encSpi.encryptedSizeNoPadding(pageSize) - pageSize)
- - encSpi.blockSize(); /* For CRC. */
+ - encSpi.blockSize(); /* For CRC and encryption key ID. */
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/txlog/TxLog.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/txlog/TxLog.java
index a41790f..8adb6b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/txlog/TxLog.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/txlog/TxLog.java
@@ -41,6 +41,7 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIOV2;
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseListImpl;
import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
@@ -124,7 +125,7 @@ public class TxLog implements CheckpointListener {
try {
if (PageIO.getType(pageAddr) != PageIO.T_META) {
// Initialize new page.
- PageMetaIO io = PageMetaIO.VERSIONS.latest();
+ PageMetaIO io = PageMetaIOV2.VERSIONS.latest();
io.initNewPage(pageAddr, metaId, pageMemory.pageSize());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 50224f3..0789a0c 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -85,12 +85,13 @@ import org.apache.ignite.internal.pagemem.wal.record.CacheState;
import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
-import org.apache.ignite.internal.pagemem.wal.record.MasterKeyChangeRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MasterKeyChangeRecordV2;
import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.MvccDataEntry;
import org.apache.ignite.internal.pagemem.wal.record.MvccTxRecord;
import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.pagemem.wal.record.ReencryptionStartRecord;
import org.apache.ignite.internal.pagemem.wal.record.RollbackRecord;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.pagemem.wal.record.WalRecordCacheGroupAware;
@@ -168,6 +169,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT;
import static org.apache.ignite.internal.pagemem.PageIdUtils.partId;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MASTER_KEY_CHANGE_RECORD;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MASTER_KEY_CHANGE_RECORD_V2;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.METASTORE_DATA_RECORD;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.fromOrdinal;
@@ -1255,6 +1257,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
grpIds.add(tup.get1().groupId());
+ if (gctx.config().isEncryptionEnabled())
+ cctx.kernalContext().encryption().onCacheGroupStop(gctx.groupId());
+
pageMem.onCacheGroupDestroyed(tup.get1().groupId());
if (tup.get2())
@@ -2347,6 +2352,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
case MVCC_DATA_RECORD:
case DATA_RECORD:
case ENCRYPTED_DATA_RECORD:
+ case ENCRYPTED_DATA_RECORD_V2:
DataRecord dataRec = (DataRecord)rec;
for (DataEntry dataEntry : dataRec.writeEntries()) {
@@ -2427,8 +2433,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
break;
- case MASTER_KEY_CHANGE_RECORD:
- cctx.kernalContext().encryption().applyKeys((MasterKeyChangeRecord)rec);
+ case MASTER_KEY_CHANGE_RECORD_V2:
+ cctx.kernalContext().encryption().applyKeys((MasterKeyChangeRecordV2)rec);
+
+ break;
+
+ case REENCRYPTION_START_RECORD:
+ cctx.kernalContext().encryption().applyReencryptionStartRecord((ReencryptionStartRecord)rec);
break;
@@ -3125,7 +3136,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
* @return WAL records predicate that passes only Metastorage and encryption data records.
*/
private IgniteBiPredicate<WALRecord.RecordType, WALPointer> onlyMetastorageAndEncryptionRecords() {
- return (type, ptr) -> type == METASTORE_DATA_RECORD || type == MASTER_KEY_CHANGE_RECORD;
+ return (type, ptr) -> type == METASTORE_DATA_RECORD ||
+ type == MASTER_KEY_CHANGE_RECORD || type == MASTER_KEY_CHANGE_RECORD_V2;
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 1fba65d..fa4fabe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -40,6 +40,8 @@ import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.SystemProperty;
import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.failure.FailureType;
+import org.apache.ignite.internal.managers.encryption.GridEncryptionManager;
+import org.apache.ignite.internal.managers.encryption.ReencryptStateUtils;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -55,7 +57,8 @@ import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
import org.apache.ignite.internal.pagemem.wal.record.RollbackRecord;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecordV2;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateIndexDataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecordV3;
import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheDiagnosticManager;
@@ -90,9 +93,10 @@ import org.apache.ignite.internal.processors.cache.persistence.partstorage.Parti
import org.apache.ignite.internal.processors.cache.persistence.partstorage.PartitionMetaStorageImpl;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIOV2;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionCountersIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIOV2;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIOV3;
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseListImpl;
import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
@@ -362,6 +366,9 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
}
});
}
+
+ if (grp.config().isEncryptionEnabled())
+ saveIndexReencryptionStatus(grp.groupId());
}
/**
@@ -389,8 +396,10 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
IgniteWriteAheadLogManager wal = this.ctx.wal();
+ GridEncryptionManager encMgr = this.ctx.kernalContext().encryption();
- if (size > 0 || updCntr > 0 || !store.partUpdateCounter().sequential()) {
+ if (size > 0 || updCntr > 0 || !store.partUpdateCounter().sequential() ||
+ (grp.config().isEncryptionEnabled() && encMgr.getEncryptionState(grp.groupId(), store.partId()) > 0)) {
GridDhtPartitionState state = null;
// localPartition will not acquire writeLock here because create=false.
@@ -429,7 +438,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
boolean changed = false;
try {
- PagePartitionMetaIOV2 io = PageIO.getPageIO(partMetaPageAddr);
+ PagePartitionMetaIOV3 io = PageIO.getPageIO(partMetaPageAddr);
long link = io.getGapsLink(partMetaPageAddr);
@@ -474,6 +483,27 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
changed |= io.setGlobalRemoveId(partMetaPageAddr, rmvId);
changed |= io.setSize(partMetaPageAddr, size);
+ int encryptIdx = 0;
+ int encryptCnt = 0;
+
+ if (grp.config().isEncryptionEnabled()) {
+ long reencryptState = encMgr.getEncryptionState(grpId, store.partId());
+
+ if (reencryptState != 0) {
+ encryptIdx = ReencryptStateUtils.pageIndex(reencryptState);
+ encryptCnt = ReencryptStateUtils.pageCount(reencryptState);
+
+ if (encryptIdx == encryptCnt) {
+ encMgr.setEncryptionState(grp, store.partId(), 0, 0);
+
+ encryptIdx = encryptCnt = 0;
+ }
+
+ changed |= io.setEncryptedPageIndex(partMetaPageAddr, encryptIdx);
+ changed |= io.setEncryptedPageCount(partMetaPageAddr, encryptCnt);
+ }
+ }
+
if (state != null)
changed |= io.setPartitionState(partMetaPageAddr, (byte)state.ordinal());
else
@@ -541,7 +571,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
pageCnt = io.getCandidatePageCount(partMetaPageAddr);
if (changed && PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, partMetaId, partMetaPage, wal, null))
- wal.log(new MetaPageUpdatePartitionDataRecordV2(
+ wal.log(new MetaPageUpdatePartitionDataRecordV3(
grpId,
partMetaId,
updCntr,
@@ -550,7 +580,9 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
cntrsPageId,
state == null ? -1 : (byte)state.ordinal(),
pageCnt,
- link
+ link,
+ encryptIdx,
+ encryptCnt
));
}
finally {
@@ -949,6 +981,9 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
public void destroyPartitionStore(int grpId, int partId) throws IgniteCheckedException {
PageMemoryEx pageMemory = (PageMemoryEx)grp.dataRegion().pageMemory();
+ if (grp.config().isEncryptionEnabled())
+ ctx.kernalContext().encryption().onDestroyPartitionStore(grp, partId);
+
int tag = pageMemory.invalidate(grp.groupId(), partId);
if (grp.walEnabled())
@@ -1019,13 +1054,13 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
try {
final long pageAddr = pageMem.writeLock(grpId, metaId, metaPage);
- boolean allocated = false;
+ boolean markDirty = false;
try {
long metastoreRoot, reuseListRoot;
if (PageIO.getType(pageAddr) != PageIO.T_META) {
- PageMetaIO pageIO = PageMetaIO.VERSIONS.latest();
+ PageMetaIOV2 pageIO = (PageMetaIOV2)PageMetaIOV2.VERSIONS.latest();
pageIO.initNewPage(pageAddr, metaId, pageMem.realPageSize(grpId));
@@ -1048,25 +1083,38 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
));
}
- allocated = true;
+ markDirty = true;
}
else {
- PageMetaIO pageIO = PageIO.getPageIO(pageAddr);
+ if (PageMetaIO.getVersion(pageAddr) < 2) {
+ ((PageMetaIOV2)PageMetaIOV2.VERSIONS.latest()).upgradePage(pageAddr);
+
+ markDirty = true;
+ }
+
+ PageMetaIOV2 pageIO = PageIO.getPageIO(pageAddr);
metastoreRoot = pageIO.getTreeRoot(pageAddr);
reuseListRoot = pageIO.getReuseListRoot(pageAddr);
+ int encrPageCnt = pageIO.getEncryptedPageCount(pageAddr);
+
+ if (encrPageCnt > 0) {
+ ctx.kernalContext().encryption().setEncryptionState(grp, PageIdAllocator.INDEX_PARTITION,
+ pageIO.getEncryptedPageIndex(pageAddr), encrPageCnt);
+ }
+
assert reuseListRoot != 0L;
}
return new Metas(
- new RootPage(new FullPageId(metastoreRoot, grpId), allocated),
- new RootPage(new FullPageId(reuseListRoot, grpId), allocated),
+ new RootPage(new FullPageId(metastoreRoot, grpId), markDirty),
+ new RootPage(new FullPageId(reuseListRoot, grpId), markDirty),
null,
null);
}
finally {
- pageMem.writeUnlock(grpId, metaId, metaPage, null, allocated);
+ pageMem.writeUnlock(grpId, metaId, metaPage, null, markDirty);
}
}
finally {
@@ -1256,6 +1304,55 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
}
/**
+ * @param grpId Cache group ID.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void saveIndexReencryptionStatus(int grpId) throws IgniteCheckedException {
+ long state = ctx.kernalContext().encryption().getEncryptionState(grpId, PageIdAllocator.INDEX_PARTITION);
+
+ if (state == 0)
+ return;
+
+ PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
+
+ long metaPageId = pageMem.metaPageId(grpId);
+ long metaPage = pageMem.acquirePage(grpId, metaPageId);
+
+ try {
+ boolean changed = false;
+
+ long metaPageAddr = pageMem.writeLock(grpId, metaPageId, metaPage);
+
+ try {
+ PageMetaIOV2 metaIo = PageMetaIO.getPageIO(metaPageAddr);
+
+ int encryptIdx = ReencryptStateUtils.pageIndex(state);
+ int encryptCnt = ReencryptStateUtils.pageCount(state);
+
+ if (encryptIdx == encryptCnt) {
+ ctx.kernalContext().encryption().setEncryptionState(grp, PageIdAllocator.INDEX_PARTITION, 0, 0);
+
+ encryptIdx = encryptCnt = 0;
+ }
+
+ changed |= metaIo.setEncryptedPageIndex(metaPageAddr, encryptIdx);
+ changed |= metaIo.setEncryptedPageCount(metaPageAddr, encryptCnt);
+
+ IgniteWriteAheadLogManager wal = ctx.cache().context().wal();
+
+ if (changed && PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, metaPageId, metaPage, wal, null))
+ wal.log(new MetaPageUpdateIndexDataRecord(grpId, metaPageId, encryptIdx, encryptCnt));
+ }
+ finally {
+ pageMem.writeUnlock(grpId, metaPageId, metaPage, null, changed);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, metaPageId, metaPage);
+ }
+ }
+
+ /**
*
*/
private static class WALHistoricalIterator implements IgniteHistoricalIterator {
@@ -1955,7 +2052,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
try {
if (PageIO.getType(pageAddr) != 0) {
- PagePartitionMetaIOV2 io = (PagePartitionMetaIOV2)PagePartitionMetaIO.VERSIONS.latest();
+ PagePartitionMetaIOV3 io = (PagePartitionMetaIOV3)PagePartitionMetaIO.VERSIONS.latest();
Map<Integer, Long> cacheSizes = null;
@@ -1968,6 +2065,13 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
delegate0.restoreState(io.getSize(pageAddr), io.getUpdateCounter(pageAddr), cacheSizes, data);
+ int encrPageCnt = io.getEncryptedPageCount(pageAddr);
+
+ if (encrPageCnt > 0) {
+ ctx.kernalContext().encryption().setEncryptionState(
+ grp, partId, io.getEncryptedPageIndex(pageAddr), encrPageCnt);
+ }
+
globalRemoveId().setIfGreater(io.getGlobalRemoveId(pageAddr));
}
}
@@ -2035,7 +2139,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
// Initialize new page.
if (PageIO.getType(pageAddr) != PageIO.T_PART_META) {
- PagePartitionMetaIOV2 io = (PagePartitionMetaIOV2)PagePartitionMetaIO.VERSIONS.latest();
+ PagePartitionMetaIOV3 io = (PagePartitionMetaIOV3)PagePartitionMetaIO.VERSIONS.latest();
io.initNewPage(pageAddr, partMetaId, pageMem.realPageSize(grpId));
@@ -2069,8 +2173,8 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
int pageVer = PagePartitionMetaIO.getVersion(pageAddr);
- if (pageVer < 2) {
- assert pageVer == 1;
+ if (pageVer < 3) {
+ assert pageVer == 1 || pageVer == 2;
if (log.isDebugEnabled())
log.info("Upgrade partition meta page version: [part=" + partId +
@@ -2080,7 +2184,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
io = PagePartitionMetaIO.VERSIONS.latest();
- ((PagePartitionMetaIOV2)io).upgradePage(pageAddr);
+ ((PagePartitionMetaIOV3)io).upgradePage(pageAddr);
pendingTreeRoot = pageMem.allocatePage(grpId, partId, PageMemory.FLAG_DATA);
partMetaStoreReuseListRoot = pageMem.allocatePage(grpId, partId, PageMemory.FLAG_DATA);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java
index 60f5017..88b619e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java
@@ -18,10 +18,10 @@
package org.apache.ignite.internal.processors.cache.persistence.file;
import java.io.IOException;
-import java.io.Serializable;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import org.apache.ignite.internal.managers.encryption.GridEncryptionManager;
+import org.apache.ignite.internal.managers.encryption.GroupKey;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -64,11 +64,6 @@ public class EncryptedFileIO implements FileIO {
private final EncryptionSpi encSpi;
/**
- * Encryption key.
- */
- private Serializable encKey;
-
- /**
* Extra bytes added by encryption.
*/
private final int encryptionOverhead;
@@ -242,12 +237,16 @@ public class EncryptedFileIO implements FileIO {
srcBuf.limit(srcBuf.position() + plainDataSize());
- encSpi.encryptNoPadding(srcBuf, key(), res);
+ GroupKey grpKey = encMgr.groupKey(groupId);
+
+ encSpi.encryptNoPadding(srcBuf, grpKey.key(), res);
res.rewind();
storeCRC(res);
+ res.put(grpKey.id());
+
srcBuf.limit(srcLimit);
srcBuf.position(srcBuf.position() + encryptionOverhead);
}
@@ -260,11 +259,31 @@ public class EncryptedFileIO implements FileIO {
assert encrypted.remaining() >= pageSize;
assert encrypted.limit() >= pageSize;
- checkCRC(encrypted);
+ int crc = FastCrc.calcCrc(encrypted, encryptedDataSize());
+
+ int storedCrc = 0;
+
+ storedCrc |= (int)encrypted.get() << 24;
+ storedCrc |= ((int)encrypted.get() & 0xff) << 16;
+ storedCrc |= ((int)encrypted.get() & 0xff) << 8;
+ storedCrc |= encrypted.get() & 0xff;
+
+ if (crc != storedCrc) {
+ throw new IOException("Content of encrypted page is broken. [StoredCrc=" + storedCrc +
+ ", calculatedCrc=" + crc + "]");
+ }
+
+ int keyId = encrypted.get() & 0xff;
+
+ encrypted.position(encrypted.position() - (encryptedDataSize() + 4 /* CRC size. */ + 1 /* key identifier. */));
encrypted.limit(encryptedDataSize());
- encSpi.decryptNoPadding(encrypted, key(), destBuf);
+ GroupKey grpKey = encMgr.groupKey(groupId, keyId);
+
+ assert grpKey != null : keyId;
+
+ encSpi.decryptNoPadding(encrypted, grpKey.key(), destBuf);
destBuf.put(zeroes); //Forcibly purge page buffer tail.
}
@@ -284,29 +303,6 @@ public class EncryptedFileIO implements FileIO {
}
/**
- * Checks encrypted data integrity.
- *
- * @param encrypted Encrypted data buffer.
- */
- private void checkCRC(ByteBuffer encrypted) throws IOException {
- int crc = FastCrc.calcCrc(encrypted, encryptedDataSize());
-
- int storedCrc = 0;
-
- storedCrc |= (int)encrypted.get() << 24;
- storedCrc |= ((int)encrypted.get() & 0xff) << 16;
- storedCrc |= ((int)encrypted.get() & 0xff) << 8;
- storedCrc |= encrypted.get() & 0xff;
-
- if (crc != storedCrc) {
- throw new IOException("Content of encrypted page is broken. [StoredCrc=" + storedCrc +
- ", calculatedCrd=" + crc + "]");
- }
-
- encrypted.position(encrypted.position() - (encryptedDataSize() + 4 /* CRC size. */));
- }
-
- /**
* @return Encrypted data size.
*/
private int encryptedDataSize() {
@@ -334,16 +330,6 @@ public class EncryptedFileIO implements FileIO {
return true;
}
- /**
- * @return Encryption key.
- */
- private Serializable key() {
- if (encKey == null)
- return encKey = encMgr.groupKey(groupId);
-
- return encKey;
- }
-
/** {@inheritDoc} */
@Override public int write(byte[] buf, int off, int len) throws IOException {
throw new UnsupportedOperationException("Encrypted File doesn't support this operation");
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
index b25bae4..84735c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
@@ -56,7 +56,8 @@ public class PageMetaIO extends PageIO {
/** */
public static final IOVersions<PageMetaIO> VERSIONS = new IOVersions<>(
- new PageMetaIO(1)
+ new PageMetaIO(1),
+ new PageMetaIOV2(2)
);
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIOV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIOV2.java
new file mode 100644
index 0000000..f9f956d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIOV2.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import org.apache.ignite.internal.pagemem.PageUtils;
+
+/**
+ * IO for index partition metadata page.
+ */
+public class PageMetaIOV2 extends PageMetaIO {
+ /** Total pages for reencryption offset. */
+ private static final int ENCRYPT_PAGE_IDX_OFF = END_OF_PAGE_META;
+
+ /** Last reencrypted page index offset. */
+ private static final int ENCRYPT_PAGE_MAX_OFF = ENCRYPT_PAGE_IDX_OFF + 4;
+
+ /**
+ * @param ver Version.
+ */
+ public PageMetaIOV2(int ver) {
+ super(ver);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Index of the last reencrypted page.
+ */
+ public int getEncryptedPageIndex(long pageAddr) {
+ return PageUtils.getInt(pageAddr, ENCRYPT_PAGE_IDX_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageIdx Index of the last reencrypted page.
+ *
+ * @return {@code true} if value has changed as a result of this method's invocation.
+ */
+ public boolean setEncryptedPageIndex(long pageAddr, int pageIdx) {
+ if (getEncryptedPageIndex(pageAddr) == pageIdx)
+ return false;
+
+ PageUtils.putLong(pageAddr, ENCRYPT_PAGE_IDX_OFF, pageIdx);
+
+ return true;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Total pages to be reencrypted.
+ */
+ public int getEncryptedPageCount(long pageAddr) {
+ return PageUtils.getInt(pageAddr, ENCRYPT_PAGE_MAX_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pagesCnt Total pages to be reencrypted.
+ *
+ * @return {@code true} if value has changed as a result of this method's invocation.
+ */
+ public boolean setEncryptedPageCount(long pageAddr, int pagesCnt) {
+ if (getEncryptedPageCount(pageAddr) == pagesCnt)
+ return false;
+
+ PageUtils.putInt(pageAddr, ENCRYPT_PAGE_MAX_OFF, pagesCnt);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
+ super.initNewPage(pageAddr, pageId, pageSize);
+
+ setEncryptedPageCount(pageAddr, 0);
+ setEncryptedPageIndex(pageAddr, 0);
+ }
+
+ /**
+ * Upgrade page to PageMetaIOV2.
+ *
+ * @param pageAddr Page address.
+ */
+ public void upgradePage(long pageAddr) {
+ assert PageIO.getType(pageAddr) == getType();
+
+ PageIO.setVersion(pageAddr, getVersion());
+
+ setEncryptedPageIndex(pageAddr, 0);
+ setEncryptedPageCount(pageAddr, 0);
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
index b58b5c0..bb06340 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
@@ -48,7 +48,8 @@ public class PagePartitionMetaIO extends PageMetaIO {
/** */
public static final IOVersions<PagePartitionMetaIO> VERSIONS = new IOVersions<>(
new PagePartitionMetaIO(1),
- new PagePartitionMetaIOV2(2)
+ new PagePartitionMetaIOV2(2),
+ new PagePartitionMetaIOV3(3)
);
/** {@inheritDoc} */
@@ -242,13 +243,24 @@ public class PagePartitionMetaIO extends PageMetaIO {
@Override protected void printPage(long pageAddr, int pageSize, GridStringBuilder sb) throws IgniteCheckedException {
super.printPage(pageAddr, pageSize, sb);
+ sb.a(",\nPagePartitionMeta[\n");
+
+ printFields(pageAddr, sb);
+
+ sb.a("\n]");
+ }
+
+ /**
+ * @param pageAddr Address.
+ * @param sb String builder.
+ */
+ protected void printFields(long pageAddr, GridStringBuilder sb) {
byte state = getPartitionState(pageAddr);
- sb.a(",\nPagePartitionMeta[\n\tsize=").a(getSize(pageAddr))
+ sb.a("\tsize=").a(getSize(pageAddr))
.a(",\n\tupdateCounter=").a(getUpdateCounter(pageAddr))
.a(",\n\tglobalRemoveId=").a(getGlobalRemoveId(pageAddr))
.a(",\n\tpartitionState=").a(state).a("(").a(GridDhtPartitionState.fromOrdinal(state)).a(")")
- .a(",\n\tcountersPageId=").a(getCountersPageId(pageAddr))
- .a("\n]");
+ .a(",\n\tcountersPageId=").a(getCountersPageId(pageAddr)).toString();
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV2.java
index efdfecd..6c30775 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV2.java
@@ -18,9 +18,7 @@
package org.apache.ignite.internal.processors.cache.persistence.tree.io;
-import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
import org.apache.ignite.internal.util.GridStringBuilder;
/**
@@ -37,6 +35,9 @@ public class PagePartitionMetaIOV2 extends PagePartitionMetaIO {
/** */
private static final int GAPS_LINK = PART_META_REUSE_LIST_ROOT_OFF + 8;
+ /** */
+ public static final int END_OF_PARTITION_PAGE_META_V2 = GAPS_LINK + 8;
+
/**
* @param ver Version.
*/
@@ -102,10 +103,10 @@ public class PagePartitionMetaIOV2 extends PagePartitionMetaIO {
}
/** {@inheritDoc} */
- @Override protected void printPage(long pageAddr, int pageSize, GridStringBuilder sb) throws IgniteCheckedException {
- byte state = getPartitionState(pageAddr);
+ @Override protected void printFields(long pageAddr, GridStringBuilder sb) {
+ super.printFields(pageAddr, sb);
- sb.a("PagePartitionMeta[\n\ttreeRoot=").a(getReuseListRoot(pageAddr));
+ sb.a("\ttreeRoot=").a(getReuseListRoot(pageAddr));
sb.a(",\n\tpendingTreeRoot=").a(getLastSuccessfulFullSnapshotId(pageAddr));
sb.a(",\n\tlastSuccessfulFullSnapshotId=").a(getLastSuccessfulFullSnapshotId(pageAddr));
sb.a(",\n\tlastSuccessfulSnapshotId=").a(getLastSuccessfulSnapshotId(pageAddr));
@@ -113,13 +114,7 @@ public class PagePartitionMetaIOV2 extends PagePartitionMetaIO {
sb.a(",\n\tlastSuccessfulSnapshotTag=").a(getLastSuccessfulSnapshotTag(pageAddr));
sb.a(",\n\tlastAllocatedPageCount=").a(getLastAllocatedPageCount(pageAddr));
sb.a(",\n\tcandidatePageCount=").a(getCandidatePageCount(pageAddr));
- sb.a(",\n\tsize=").a(getSize(pageAddr));
- sb.a(",\n\tupdateCounter=").a(getUpdateCounter(pageAddr));
- sb.a(",\n\tglobalRemoveId=").a(getGlobalRemoveId(pageAddr));
- sb.a(",\n\tpartitionState=").a(state).a("(").a(GridDhtPartitionState.fromOrdinal(state)).a(")");
- sb.a(",\n\tcountersPageId=").a(getCountersPageId(pageAddr));
sb.a(",\n\tcntrUpdDataPageId=").a(getGapsLink(pageAddr));
- sb.a("\n]");
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV3.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV3.java
new file mode 100644
index 0000000..1c81d2e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV3.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.util.GridStringBuilder;
+
+/**
+ * IO for partition metadata pages.
+ */
+public class PagePartitionMetaIOV3 extends PagePartitionMetaIOV2 {
+ /** Last reencrypted page index offset. */
+ private static final int ENCRYPT_PAGE_IDX_OFF = END_OF_PARTITION_PAGE_META_V2;
+
+ /** Total pages to be reencrypted offset. */
+ private static final int ENCRYPT_PAGE_MAX_OFF = ENCRYPT_PAGE_IDX_OFF + 4;
+
+ /**
+ * @param ver Version.
+ */
+ public PagePartitionMetaIOV3(int ver) {
+ super(ver);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
+ super.initNewPage(pageAddr, pageId, pageSize);
+
+ setEncryptedPageIndex(pageAddr, 0);
+ setEncryptedPageCount(pageAddr, 0);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Index of the last reencrypted page.
+ */
+ public int getEncryptedPageIndex(long pageAddr) {
+ return PageUtils.getInt(pageAddr, ENCRYPT_PAGE_IDX_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageIdx Index of the last reencrypted page.
+ *
+ * @return {@code true} if value has changed as a result of this method's invocation.
+ */
+ public boolean setEncryptedPageIndex(long pageAddr, int pageIdx) {
+ if (getEncryptedPageIndex(pageAddr) == pageIdx)
+ return false;
+
+ PageUtils.putLong(pageAddr, ENCRYPT_PAGE_IDX_OFF, pageIdx);
+
+ return true;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Total pages to be reencrypted.
+ */
+ public int getEncryptedPageCount(long pageAddr) {
+ return PageUtils.getInt(pageAddr, ENCRYPT_PAGE_MAX_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pagesCnt Total pages to be reencrypted.
+ *
+ * @return {@code true} if value has changed as a result of this method's invocation.
+ */
+ public boolean setEncryptedPageCount(long pageAddr, int pagesCnt) {
+ if (getEncryptedPageCount(pageAddr) == pagesCnt)
+ return false;
+
+ PageUtils.putInt(pageAddr, ENCRYPT_PAGE_MAX_OFF, pagesCnt);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void printFields(long pageAddr, GridStringBuilder sb) {
+ super.printFields(pageAddr, sb);
+
+ sb.a(",\n\tencryptedPageIndex=").a(getEncryptedPageIndex(pageAddr));
+ sb.a(",\n\tencryptedPageCount=").a(getEncryptedPageCount(pageAddr));
+ }
+
+ /**
+ * Upgrade page to PagePartitionMetaIOV3.
+ *
+ * @param pageAddr Page address.
+ */
+ @Override public void upgradePage(long pageAddr) {
+ assert PageIO.getType(pageAddr) == getType();
+
+ int ver = PageIO.getVersion(pageAddr);
+
+ assert ver < getVersion();
+
+ if (ver < 2)
+ super.upgradePage(pageAddr);
+
+ PageIO.setVersion(pageAddr, getVersion());
+
+ setEncryptedPageIndex(pageAddr, 0);
+ setEncryptedPageCount(pageAddr, 0);
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index b79d637..70da8e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -1062,6 +1062,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
// Bump up the oldest archive segment index.
if (segmentAware.lastTruncatedArchiveIdx() < desc.idx)
segmentAware.lastTruncatedArchiveIdx(desc.idx);
+
+ cctx.kernalContext().encryption().onWalSegmentRemoved(desc.idx);
}
}
@@ -1088,6 +1090,11 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
/** {@inheritDoc} */
+ @Override public long currentSegment() {
+ return segmentAware.curAbsWalIdx();
+ }
+
+ /** {@inheritDoc} */
@Override public int walArchiveSegments() {
long lastTruncated = segmentAware.lastTruncatedArchiveIdx();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
index 46fbd77..2da9128 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
@@ -20,7 +20,6 @@ package org.apache.ignite.internal.processors.cache.persistence.wal.serializer;
import java.io.DataInput;
import java.io.EOFException;
import java.io.IOException;
-import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
@@ -32,6 +31,8 @@ import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.managers.encryption.GridEncryptionManager;
+import org.apache.ignite.internal.managers.encryption.GroupKey;
+import org.apache.ignite.internal.managers.encryption.GroupKeyEncrypted;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.wal.record.CacheState;
import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
@@ -39,10 +40,11 @@ import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
import org.apache.ignite.internal.pagemem.wal.record.EncryptedRecord;
import org.apache.ignite.internal.pagemem.wal.record.LazyDataEntry;
-import org.apache.ignite.internal.pagemem.wal.record.MasterKeyChangeRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MasterKeyChangeRecordV2;
import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.pagemem.wal.record.ReencryptionStartRecord;
import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType;
@@ -68,12 +70,14 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootInlineFlagsCreatedVersionRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootInlineRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateIndexDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastAllocatedIndex;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastSuccessfulFullSnapshotId;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastSuccessfulSnapshotId;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateNextSnapshotId;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecordV2;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecordV3;
import org.apache.ignite.internal.pagemem.wal.record.delta.NewRootInitRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.PageListMetaResetCountRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListAddPageRecord;
@@ -117,8 +121,10 @@ import org.apache.ignite.spi.encryption.noop.NoopEncryptionSpi;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_RECORD;
-import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.ENCRYPTED_DATA_RECORD;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.ENCRYPTED_DATA_RECORD_V2;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.ENCRYPTED_RECORD;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.ENCRYPTED_RECORD_V2;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MASTER_KEY_CHANGE_RECORD_V2;
import static org.apache.ignite.internal.processors.cache.GridCacheOperation.READ;
import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.REC_TYPE_SIZE;
import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.putRecordType;
@@ -190,7 +196,7 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
int clSz = plainSize(record);
if (needEncryption(record))
- return encSpi.encryptedSize(clSz) + 4 /* groupId */ + 4 /* data size */ + REC_TYPE_SIZE;
+ return encSpi.encryptedSize(clSz) + 4 /* groupId */ + 4 /* data size */ + 1 /* key ID */ + REC_TYPE_SIZE;
return clSz;
}
@@ -198,7 +204,7 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
/** {@inheritDoc} */
@Override public WALRecord readRecord(RecordType type, ByteBufferBackedDataInput in, int size)
throws IOException, IgniteCheckedException {
- if (type == ENCRYPTED_RECORD) {
+ if (type == ENCRYPTED_RECORD || type == ENCRYPTED_RECORD_V2) {
if (encSpi == null) {
T2<Integer, RecordType> knownData = skipEncryptedRecord(in, true);
@@ -206,7 +212,8 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
return new EncryptedRecord(knownData.get1(), knownData.get2());
}
- T3<ByteBufferBackedDataInput, Integer, RecordType> clData = readEncryptedData(in, true);
+ T3<ByteBufferBackedDataInput, Integer, RecordType> clData =
+ readEncryptedData(in, true, type == ENCRYPTED_RECORD_V2);
//This happen during startup. On first WAL iteration we restore only metastore.
//So, no encryption keys available. See GridCacheDatabaseSharedManager#readMetastore
@@ -270,30 +277,36 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
*
* @param in Input stream.
* @param readType If {@code true} plain record type will be read from {@code in}.
+ * @param readKeyId If {@code true} encryption key identifier will be read from {@code in}.
* @return Plain data stream, group id, plain record type,
* @throws IOException If failed.
* @throws IgniteCheckedException If failed.
*/
- private T3<ByteBufferBackedDataInput, Integer, RecordType> readEncryptedData(ByteBufferBackedDataInput in,
- boolean readType)
- throws IOException, IgniteCheckedException {
+ private T3<ByteBufferBackedDataInput, Integer, RecordType> readEncryptedData(
+ ByteBufferBackedDataInput in,
+ boolean readType,
+ boolean readKeyId
+ ) throws IOException, IgniteCheckedException {
int grpId = in.readInt();
int encRecSz = in.readInt();
+
RecordType plainRecType = null;
if (readType)
plainRecType = RecordV1Serializer.readRecordType(in);
+ int keyId = readKeyId ? in.readUnsignedByte() : GridEncryptionManager.INITIAL_KEY_ID;
+
byte[] encData = new byte[encRecSz];
in.readFully(encData);
- Serializable key = encMgr.groupKey(grpId);
+ GroupKey grpKey = encMgr.groupKey(grpId, keyId);
- if (key == null)
+ if (grpKey == null)
return new T3<>(null, grpId, plainRecType);
- byte[] clData = encSpi.decrypt(encData, key);
+ byte[] clData = encSpi.decrypt(encData, grpKey.key());
return new T3<>(new ByteBufferBackedDataInputImpl().buffer(ByteBuffer.wrap(clData)), grpId, plainRecType);
}
@@ -339,11 +352,11 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
if (plainRecType != null)
putRecordType(dst, plainRecType);
- Serializable key = encMgr.groupKey(grpId);
+ GroupKey grpKey = encMgr.groupKey(grpId);
- assert key != null;
+ dst.put(grpKey.id());
- encSpi.encrypt(clData, key, dst);
+ encSpi.encrypt(clData, grpKey.key(), dst);
}
/**
@@ -372,6 +385,9 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
case META_PAGE_INIT:
return /*cache ID*/4 + /*page ID*/8 + /*ioType*/2 + /*ioVer*/2 + /*tree root*/8 + /*reuse root*/8;
+ case INDEX_META_PAGE_DELTA_RECORD:
+ return /*cache ID*/4 + /*page ID*/8 + /*encrypt page index*/ 4 + /*encrypt pages count*/4;
+
case PARTITION_META_PAGE_UPDATE_COUNTERS:
return /*cache ID*/4 + /*page ID*/8 + /*upd cntr*/8 + /*rmv id*/8 + /*part size*/4 + /*counters page id*/8 + /*state*/ 1
+ /*allocatedIdxCandidate*/ 4;
@@ -380,6 +396,10 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
return /*cache ID*/4 + /*page ID*/8 + /*upd cntr*/8 + /*rmv id*/8 + /*part size*/4 + /*counters page id*/8 + /*state*/ 1
+ /*allocatedIdxCandidate*/ 4 + /*link*/ 8;
+ case PARTITION_META_PAGE_DELTA_RECORD_V3:
+ return /*cache ID*/4 + /*page ID*/8 + /*upd cntr*/8 + /*rmv id*/8 + /*part size*/4 + /*counters page id*/8 + /*state*/ 1
+ + /*allocatedIdxCandidate*/ 4 + /*link*/ 8 + /*encrypt page index*/ 4 + /*encrypt pages count*/4;
+
case MEMORY_RECOVERY:
return 8;
@@ -536,10 +556,12 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
case TX_RECORD:
return txRecordSerializer.size((TxRecord)record);
- case MASTER_KEY_CHANGE_RECORD:
- MasterKeyChangeRecord rec = (MasterKeyChangeRecord)record;
+ case MASTER_KEY_CHANGE_RECORD_V2:
+ return ((MasterKeyChangeRecordV2)record).dataSize();
+
+ case REENCRYPTION_START_RECORD:
+ return ((ReencryptionStartRecord)record).dataSize();
- return rec.dataSize();
default:
throw new UnsupportedOperationException("Type: " + record.type());
}
@@ -609,6 +631,11 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
break;
+ case INDEX_META_PAGE_DELTA_RECORD:
+ res = new MetaPageUpdateIndexDataRecord(in);
+
+ break;
+
case PARTITION_META_PAGE_UPDATE_COUNTERS:
res = new MetaPageUpdatePartitionDataRecord(in);
@@ -619,6 +646,11 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
break;
+ case PARTITION_META_PAGE_DELTA_RECORD_V3:
+ res = new MetaPageUpdatePartitionDataRecordV3(in);
+
+ break;
+
case MEMORY_RECOVERY:
long ts = in.readLong();
@@ -647,12 +679,13 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
break;
case ENCRYPTED_DATA_RECORD:
+ case ENCRYPTED_DATA_RECORD_V2:
entryCnt = in.readInt();
entries = new ArrayList<>(entryCnt);
for (int i = 0; i < entryCnt; i++)
- entries.add(readEncryptedDataEntry(in));
+ entries.add(readEncryptedDataEntry(in, type == ENCRYPTED_DATA_RECORD_V2));
res = new DataRecord(entries, 0L);
@@ -1184,6 +1217,7 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
break;
case MASTER_KEY_CHANGE_RECORD:
+ case MASTER_KEY_CHANGE_RECORD_V2:
int keyNameLen = in.readInt();
byte[] keyNameBytes = new byte[keyNameLen];
@@ -1194,21 +1228,39 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
int keysCnt = in.readInt();
- HashMap<Integer, byte[]> grpKeys = new HashMap<>(keysCnt);
+ List<T2<Integer, GroupKeyEncrypted>> grpKeys = new ArrayList<>(keysCnt);
+
+ boolean readKeyId = type == MASTER_KEY_CHANGE_RECORD_V2;
for (int i = 0; i < keysCnt; i++) {
int grpId = in.readInt();
+ int keyId = readKeyId ? in.readByte() & 0xff : 0;
int grpKeySize = in.readInt();
-
byte[] grpKey = new byte[grpKeySize];
in.readFully(grpKey);
- grpKeys.put(grpId, grpKey);
+ grpKeys.add(new T2<>(grpId, new GroupKeyEncrypted(keyId, grpKey)));
}
- res = new MasterKeyChangeRecord(masterKeyName, grpKeys);
+ res = new MasterKeyChangeRecordV2(masterKeyName, grpKeys);
+
+ break;
+
+ case REENCRYPTION_START_RECORD:
+ int grpsCnt = in.readInt();
+
+ Map<Integer, Byte> map = U.newHashMap(grpsCnt);
+
+ for (int i = 0; i < grpsCnt; i++) {
+ int grpId = in.readInt();
+ byte keyId = in.readByte();
+
+ map.put(grpId, keyId);
+ }
+
+ res = new ReencryptionStartRecord(map);
break;
@@ -1265,8 +1317,14 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
break;
+ case INDEX_META_PAGE_DELTA_RECORD:
+ ((MetaPageUpdateIndexDataRecord)rec).toBytes(buf);
+
+ break;
+
case PARTITION_META_PAGE_UPDATE_COUNTERS:
case PARTITION_META_PAGE_UPDATE_COUNTERS_V2:
+ case PARTITION_META_PAGE_DELTA_RECORD_V3:
((MetaPageUpdatePartitionDataRecord)rec).toBytes(buf);
break;
@@ -1795,23 +1853,40 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
case SWITCH_SEGMENT_RECORD:
break;
- case MASTER_KEY_CHANGE_RECORD:
- MasterKeyChangeRecord mkChangeRec = (MasterKeyChangeRecord)rec;
+ case MASTER_KEY_CHANGE_RECORD_V2:
+ MasterKeyChangeRecordV2 mkChangeRec = (MasterKeyChangeRecordV2)rec;
byte[] keyIdBytes = mkChangeRec.getMasterKeyName().getBytes();
buf.putInt(keyIdBytes.length);
buf.put(keyIdBytes);
- Map<Integer, byte[]> grpKeys = mkChangeRec.getGrpKeys();
+ List<T2<Integer, GroupKeyEncrypted>> grpKeys = mkChangeRec.getGrpKeys();
buf.putInt(grpKeys.size());
- for (Entry<Integer, byte[]> entry : grpKeys.entrySet()) {
- buf.putInt(entry.getKey());
+ for (T2<Integer, GroupKeyEncrypted> entry : grpKeys) {
+ GroupKeyEncrypted grpKey = entry.get2();
+
+ buf.putInt(entry.get1());
+ buf.put((byte)grpKey.id());
+
+ buf.putInt(grpKey.key().length);
+ buf.put(grpKey.key());
+ }
+
+ break;
+
+ case REENCRYPTION_START_RECORD:
+ ReencryptionStartRecord statusRecord = (ReencryptionStartRecord)rec;
+
+ Map<Integer, Byte> grps = statusRecord.groups();
+
+ buf.putInt(grps.size());
- buf.putInt(entry.getValue().length);
- buf.put(entry.getValue());
+ for (Map.Entry<Integer, Byte> e : grps.entrySet()) {
+ buf.putInt(e.getKey());
+ buf.put(e.getValue());
}
break;
@@ -1927,11 +2002,12 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
/**
* @param in Input to read from.
+ * @param readKeyId If {@code true} encryption key identifier will be read from {@code in}.
* @return Read entry.
* @throws IOException If failed.
* @throws IgniteCheckedException If failed.
*/
- DataEntry readEncryptedDataEntry(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
+ DataEntry readEncryptedDataEntry(ByteBufferBackedDataInput in, boolean readKeyId) throws IOException, IgniteCheckedException {
boolean needDecryption = in.readByte() == ENCRYPTED;
if (needDecryption) {
@@ -1941,7 +2017,7 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
return new EncryptedDataEntry();
}
- T3<ByteBufferBackedDataInput, Integer, RecordType> clData = readEncryptedData(in, false);
+ T3<ByteBufferBackedDataInput, Integer, RecordType> clData = readEncryptedData(in, false, readKeyId);
if (clData.get1() == null)
return null;
@@ -2035,12 +2111,12 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
return rec.type();
if (needEncryption(rec))
- return ENCRYPTED_RECORD;
+ return ENCRYPTED_RECORD_V2;
if (rec.type() != DATA_RECORD)
return rec.type();
- return isDataRecordEncrypted((DataRecord)rec) ? ENCRYPTED_DATA_RECORD : DATA_RECORD;
+ return isDataRecordEncrypted((DataRecord)rec) ? ENCRYPTED_DATA_RECORD_V2 : DATA_RECORD;
}
/**
@@ -2129,7 +2205,7 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
int clSz = entrySize(entry);
if (!encryptionDisabled && needEncryption(cctx.cacheContext(entry.cacheId()).groupId()))
- sz += encSpi.encryptedSize(clSz) + 1 /* encrypted flag */ + 4 /* groupId */ + 4 /* data size */;
+ sz += encSpi.encryptedSize(clSz) + 1 /*encrypted flag*/ + 4 /*groupId*/ + 4 /*data size*/ + 1 /*key ID*/;
else {
sz += clSz;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
index 6e70ee6..8622629 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
@@ -56,6 +56,8 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.ENCRYPTED_DATA_RECORD_V2;
+
/**
* Record data V2 serializer.
*/
@@ -180,13 +182,14 @@ public class RecordDataV2Serializer extends RecordDataV1Serializer {
return new MvccDataRecord(entries, timeStamp);
case ENCRYPTED_DATA_RECORD:
+ case ENCRYPTED_DATA_RECORD_V2:
entryCnt = in.readInt();
timeStamp = in.readLong();
entries = new ArrayList<>(entryCnt);
for (int i = 0; i < entryCnt; i++)
- entries.add(readEncryptedDataEntry(in));
+ entries.add(readEncryptedDataEntry(in, type == ENCRYPTED_DATA_RECORD_V2));
return new DataRecord(entries, timeStamp);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/BasicRateLimiter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/BasicRateLimiter.java
new file mode 100644
index 0000000..8429ccb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/BasicRateLimiter.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.typedef.internal.A;
+
+import static java.lang.Math.max;
+import static java.util.concurrent.TimeUnit.MICROSECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+/**
+ * The simplified version of Google Guava smooth rate limiter.<br><br>
+ *
+ * The primary feature of a rate limiter is its "stable rate", the maximum rate that is should
+ * allow at normal conditions. This is enforced by "throttling" incoming requests as needed, i.e.
+ * compute, for an incoming request, the appropriate throttle time, and make the calling thread
+ * wait as much.<br><br>
+ *
+ * The simplest way to maintain a rate of QPS is to keep the timestamp of the last granted
+ * request, and ensure that (1/QPS) seconds have elapsed since then. For example, for a rate of
+ * QPS=5 (5 tokens per second), if we ensure that a request isn't granted earlier than 200ms after
+ * the last one, then we achieve the intended rate. If a request comes and the last request was
+ * granted only 100ms ago, then we wait for another 100ms. At this rate, serving 15 fresh permits
+ * (i.e. for an acquire(15) request) naturally takes 3 seconds.<br><br>
+ *
+ * It is important to realize that such a limiter has a very superficial memory of the past:
+ * it only remembers the last request. if the limiter was unused for a long period of
+ * time, then a request arrived and was immediately granted? This limiter would immediately
+ * forget about that past underutilization.
+ */
+public class BasicRateLimiter {
+ /** Start timestamp. */
+ private final long startTime = System.nanoTime();
+
+ /** Synchronization mutex. */
+ private final Object mux = new Object();
+
+ /**
+ * The interval between two unit requests, at our stable rate. E.g., a stable rate of 5 permits
+ * per second has a stable interval of 200ms.
+ */
+ private double stableIntervalMicros;
+
+ /**
+ * The time when the next request (no matter its size) will be granted. After granting a request,
+ * this is pushed further in the future. Large requests push this further than small requests.
+ */
+ private long nextFreeTicketMicros;
+
+ /**
+ * @param permitsPerSecond Estimated number of permits per second.
+ */
+ public BasicRateLimiter(double permitsPerSecond) {
+ setRate(permitsPerSecond);
+ }
+
+ /**
+ * Updates the stable rate.
+ *
+ * @param permitsPerSecond The new stable rate of this {@code RateLimiter}.
+ * @throws IllegalArgumentException If {@code permitsPerSecond} is negative or zero.
+ */
+ public void setRate(double permitsPerSecond) {
+ A.ensure(permitsPerSecond > 0, "Requested permits (" + permitsPerSecond + ") must be positive");
+
+ synchronized (mux) {
+ resync();
+
+ stableIntervalMicros = SECONDS.toMicros(1L) / permitsPerSecond;
+ }
+ }
+
+ /**
+ * @return The stable rate (as {@code permits per seconds}).
+ */
+ public double getRate() {
+ synchronized (mux) {
+ return SECONDS.toMicros(1L) / stableIntervalMicros;
+ }
+ }
+
+ /**
+ * Acquires the given number of permits from this {@code RateLimiter}, blocking until the request
+ * can be granted. Tells the amount of time slept, if any.
+ *
+ * @param permits The number of permits to acquire.
+ * @throws IllegalArgumentException If the requested number of permits is negative or zero.
+ */
+ public void acquire(int permits) throws IgniteInterruptedCheckedException {
+ long microsToWait = reserve(permits);
+
+ try {
+ MICROSECONDS.sleep(microsToWait);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteInterruptedCheckedException(e);
+ }
+ }
+
+ /**
+ * Reserves the given number of permits for future use.
+ *
+ * @param permits The number of permits.
+ * @return Time in microseconds to wait until the resource can be acquired, never negative.
+ */
+ private long reserve(int permits) {
+ A.ensure(permits > 0, "Requested permits (" + permits + ") must be positive");
+
+ synchronized (mux) {
+ long nowMicros = resync();
+
+ long momentAvailable = nextFreeTicketMicros;
+
+ nextFreeTicketMicros = momentAvailable + (long)(permits * stableIntervalMicros);
+
+ return max(momentAvailable - nowMicros, 0);
+ }
+ }
+
+ /**
+ * Updates {@code nextFreeTicketMicros} based on the current time.
+ *
+ * @return Time passed (since start) in microseconds.
+ */
+ private long resync() {
+ long passed = MICROSECONDS.convert(System.nanoTime() - startTime, NANOSECONDS);
+
+ // if nextFreeTicket is in the past, resync to now
+ if (passed > nextFreeTicketMicros)
+ nextFreeTicketMicros = passed;
+
+ return passed;
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
index fe0fee5..6539453 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
@@ -430,6 +430,16 @@ public class DistributedProcess<I extends Serializable, R extends Serializable>
*
* @see IgniteSnapshotManager
*/
- END_SNAPSHOT
+ END_SNAPSHOT,
+
+ /**
+ * Cache group encyption key change prepare phase.
+ */
+ CACHE_GROUP_KEY_CHANGE_PREPARE,
+
+ /**
+ * Cache group encyption key change perform phase.
+ */
+ CACHE_GROUP_KEY_CHANGE_FINISH
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/EncryptionMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/EncryptionMXBean.java
index 9db1049..5aaca05 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/EncryptionMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/EncryptionMXBean.java
@@ -17,6 +17,7 @@
package org.apache.ignite.mxbean;
+import java.util.Collection;
import org.apache.ignite.IgniteEncryption;
/**
@@ -43,4 +44,15 @@ public interface EncryptionMXBean {
public void changeMasterKey(
@MXBeanParameter(name = "masterKeyName", description = "Master key name.") String masterKeyName
);
+
+ /**
+ * Starts cache group encryption key change process.
+ *
+ * @param cacheOrGrpName Cache or group name.
+ * @see IgniteEncryption#changeCacheGroupKey(Collection)
+ */
+ @MXBeanDescription("Change cache group key.")
+ public void changeCacheGroupKey(
+ @MXBeanParameter(name = "cacheOrGroupName", description = "Cache or group name.") String cacheOrGrpName
+ );
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/encryption/AbstractEncryptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/encryption/AbstractEncryptionTest.java
index 68b4888..8c66afe 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/encryption/AbstractEncryptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/encryption/AbstractEncryptionTest.java
@@ -20,21 +20,40 @@ package org.apache.ignite.internal.encryption;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
import java.security.KeyStore;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.encryption.GridEncryptionManager;
+import org.apache.ignite.internal.managers.encryption.GroupKey;
+import org.apache.ignite.internal.managers.encryption.ReencryptStateUtils;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -47,6 +66,7 @@ import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.configuration.WALMode.FSYNC;
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
import static org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionSpi.CIPHER_ALGO;
import static org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionSpi.DEFAULT_MASTER_KEY_NAME;
@@ -115,6 +135,18 @@ public abstract class AbstractEncryptionTest extends GridCommonAbstractTest {
return KEYSTORE_PATH;
}
+ /**
+ * @param name Cache name.
+ * @param grp Cache group name.
+ */
+ protected <K, V> CacheConfiguration<K, V> cacheConfiguration(String name, String grp) {
+ CacheConfiguration<K, V> cfg = new CacheConfiguration<>(name);
+
+ return cfg.setWriteSynchronizationMode(FULL_SYNC)
+ .setGroupName(grp)
+ .setEncryptionEnabled(true);
+ }
+
/** */
void checkEncryptedCaches(IgniteEx grid0, IgniteEx grid1) {
Set<String> cacheNames = new HashSet<>(grid0.cacheNames());
@@ -139,13 +171,21 @@ public abstract class AbstractEncryptionTest extends GridCommonAbstractTest {
assertTrue(encrypted1.configuration().isEncryptionEnabled());
- KeystoreEncryptionKey encKey0 = (KeystoreEncryptionKey)grid0.context().encryption().groupKey(grpId);
+ GroupKey grpKey0 = grid0.context().encryption().groupKey(grpId);
+
+ assertNotNull(grpKey0);
+
+ KeystoreEncryptionKey encKey0 = (KeystoreEncryptionKey)grpKey0.key();
assertNotNull(encKey0);
assertNotNull(encKey0.key());
if (!grid1.configuration().isClientMode()) {
- KeystoreEncryptionKey encKey1 = (KeystoreEncryptionKey)grid1.context().encryption().groupKey(grpId);
+ GroupKey grpKey1 = grid1.context().encryption().groupKey(grpId);
+
+ assertNotNull(grpKey1);
+
+ KeystoreEncryptionKey encKey1 = (KeystoreEncryptionKey)grpKey1.key();
assertNotNull(encKey1);
assertNotNull(encKey1.key());
@@ -161,12 +201,21 @@ public abstract class AbstractEncryptionTest extends GridCommonAbstractTest {
/** */
protected void checkData(IgniteEx grid0) {
- IgniteCache<Long, String> cache = grid0.cache(cacheName());
+ IgniteCache<Long, Object> cache = grid0.cache(cacheName());
assertNotNull(cache);
- for (long i = 0; i < 100; i++)
- assertEquals("" + i, cache.get(i));
+ int size = cache.size();
+
+ assertTrue("Cache cannot be empty", size > 0);
+
+ for (long i = 0; i < size; i++)
+ assertEquals(generateValue(i), cache.get(i));
+ }
+
+ /** */
+ protected Object generateValue(long id) {
+ return String.valueOf(id);
}
/** */
@@ -178,22 +227,17 @@ public abstract class AbstractEncryptionTest extends GridCommonAbstractTest {
/** */
protected void createEncryptedCache(IgniteEx grid0, @Nullable IgniteEx grid1, String cacheName, String cacheGroup,
boolean putData) throws IgniteInterruptedCheckedException {
- CacheConfiguration<Long, String> ccfg = new CacheConfiguration<Long, String>(cacheName)
- .setWriteSynchronizationMode(FULL_SYNC)
- .setGroupName(cacheGroup)
- .setEncryptionEnabled(true);
-
- IgniteCache<Long, String> cache = grid0.createCache(ccfg);
+ IgniteCache<Long, Object> cache = grid0.createCache(cacheConfiguration(cacheName, cacheGroup));
if (grid1 != null)
GridTestUtils.waitForCondition(() -> grid1.cachex(cacheName()) != null, 2_000L);
if (putData) {
for (long i = 0; i < 100; i++)
- cache.put(i, "" + i);
+ cache.put(i, generateValue(i));
for (long i = 0; i < 100; i++)
- assertEquals("" + i, cache.get(i));
+ assertEquals(generateValue(i), cache.get(i));
}
}
@@ -271,4 +315,206 @@ public abstract class AbstractEncryptionTest extends GridCommonAbstractTest {
return true;
}
+
+ /**
+ * Load data into cache "{@link #cacheName()}" using node "{@link #GRID_0}".
+ *
+ * @param cnt Count of entries.
+ */
+ protected void loadData(int cnt) {
+ loadData(cacheName(), cnt);
+ }
+
+ /**
+ * Load data into cache using node "{@link #GRID_0}".
+ *
+ * @param cnt Count of entries.
+ * @param cacheName Cache name.
+ */
+ protected void loadData(String cacheName, int cnt) {
+ info("Loading " + cnt + " entries into " + cacheName);
+
+ int start = grid(GRID_0).cache(cacheName).size();
+
+ try (IgniteDataStreamer<Long, Object> streamer = grid(GRID_0).dataStreamer(cacheName)) {
+ for (long i = start; i < (cnt + start); i++)
+ streamer.addData(i, generateValue(i));
+ }
+
+ info("Load data finished");
+ }
+
+ /**
+ * Ensures that all pages of page store have expected encryption key identifier.
+ *
+ * @param grpId Cache group ID.
+ * @param expKeyId Encryption key ID.
+ * @param timeout Timeout to wait for encryption to complete.
+ * @throws Exception If failed.
+ */
+ protected void checkGroupKey(int grpId, int expKeyId, long timeout) throws Exception {
+ awaitEncryption(G.allGrids(), grpId, timeout);
+
+ for (Ignite g : G.allGrids()) {
+ IgniteEx grid = (IgniteEx)g;
+
+ if (grid.context().clientNode())
+ continue;
+
+ info("Validating encryption key [node=" + g.cluster().localNode().id() + ", grp=" + grpId + "]");
+
+ CacheGroupContext grp = grid.context().cache().cacheGroup(grpId);
+
+ if (grp == null || !grp.affinityNode()) {
+ info("Context doesn't exits on " + grid.localNode().id());
+
+ continue;
+ }
+
+ GridEncryptionManager encryption = grid.context().encryption();
+
+ assertEquals(grid.localNode().id().toString(), (byte)expKeyId, encryption.groupKey(grpId).id());
+
+ IgniteInternalFuture<Void> fut = encryption.reencryptionFuture(grpId);
+
+ // The future will be completed after the checkpoint, forcecheckpoint does nothing
+ // if the checkpoint has already been scheduled.
+ GridTestUtils.waitForCondition(() -> {
+ if (fut.isDone())
+ return true;
+
+ try {
+ forceCheckpoint(g);
+ }
+ catch (IgniteCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ return fut.isDone();
+ }, timeout);
+
+ assertTrue(fut.isDone());
+
+ List<Integer> parts = IntStream.range(0, grp.shared().affinity().affinity(grpId).partitions())
+ .boxed().collect(Collectors.toList());
+
+ parts.add(INDEX_PARTITION);
+
+ int realPageSize = grp.dataRegion().pageMemory().realPageSize(grpId);
+ int encryptionBlockSize = grp.shared().kernalContext().config().getEncryptionSpi().blockSize();
+
+ for (int p : parts) {
+ FilePageStore pageStore =
+ (FilePageStore)((FilePageStoreManager)grp.shared().pageStore()).getStore(grpId, p);
+
+ if (!pageStore.exists())
+ continue;
+
+ long state = grid.context().encryption().getEncryptionState(grpId, p);
+
+ String msg = String.format("p=%d, off=%d, total=%d",
+ p, ReencryptStateUtils.pageIndex(state), ReencryptStateUtils.pageCount(state));
+
+ assertEquals(msg, 0, ReencryptStateUtils.pageCount(state));
+ assertEquals(msg, 0, ReencryptStateUtils.pageIndex(state));
+
+ long startPageId = PageIdUtils.pageId(p, PageIdAllocator.FLAG_DATA, 0);
+
+ int pagesCnt = pageStore.pages();
+ int pageSize = pageStore.getPageSize();
+
+ ByteBuffer pageBuf = ByteBuffer.allocate(pageSize);
+
+ Path path = new File(pageStore.getFileAbsolutePath()).toPath();
+
+ try (FileChannel ch = FileChannel.open(path, StandardOpenOption.READ)) {
+ for (int n = 0; n < pagesCnt; n++) {
+ long pageId = startPageId + n;
+ long pageOff = pageStore.pageOffset(pageId);
+
+ pageBuf.position(0);
+
+ ch.position(pageOff);
+ ch.read(pageBuf);
+
+ pageBuf.position(realPageSize + encryptionBlockSize);
+
+ int pageCrc = pageBuf.getInt();
+ int pageKeyId = pageBuf.get() & 0xff;
+
+ // If this page is empty we can skip it.
+ if (pageCrc == 0 && pageKeyId == 0) {
+ pageBuf.position(0);
+
+ boolean emptyPage = false;
+
+ while (pageBuf.hasRemaining() && !emptyPage)
+ emptyPage = pageBuf.getLong() == 0;
+
+ if (emptyPage)
+ continue;
+ }
+
+ msg = String.format("File=%s, page=%d", pageStore.getFileAbsolutePath(), n);
+ assertEquals(msg, expKeyId, pageKeyId);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * @param grids Grids.
+ * @param grpId Cache group ID.
+ * @param timeout Timeout to wait for encryption to complete.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected void awaitEncryption(List<Ignite> grids, int grpId, long timeout) throws IgniteCheckedException {
+ GridCompoundFuture<Void, ?> fut = new GridCompoundFuture<>();
+
+ for (Ignite node : grids) {
+ IgniteEx grid = (IgniteEx)node;
+
+ if (grid.context().clientNode())
+ continue;
+
+ IgniteInternalFuture<Void> fut0 = GridTestUtils.runAsync(() -> {
+ boolean success =
+ GridTestUtils.waitForCondition(() -> !isReencryptionInProgress(grid, grpId), timeout);
+
+ assertTrue(success);
+
+ return null;
+ });
+
+ fut.add(fut0);
+ }
+
+ fut.markInitialized();
+
+ fut.get(timeout);
+ }
+
+ /**
+ * @param node Node.
+ * @param grpId Cache group ID.
+ * @return {@code True} If reencryption of the specified group is not yet complete.
+ */
+ protected boolean isReencryptionInProgress(IgniteEx node, int grpId) {
+ CacheGroupContext grp = node.context().cache().cacheGroup(grpId);
+
+ if (grp == null || !grp.affinityNode())
+ return false;
+
+ for (int p = 0; p < grp.affinity().partitions(); p++) {
+ long state = node.context().encryption().getEncryptionState(grpId, p);
+
+ if (ReencryptStateUtils.pageIndex(state) != ReencryptStateUtils.pageCount(state))
+ return true;
+ }
+
+ long state = node.context().encryption().getEncryptionState(grpId, INDEX_PARTITION);
+
+ return ReencryptStateUtils.pageIndex(state) != ReencryptStateUtils.pageCount(state);
+ }
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/encryption/CacheGroupKeyChangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/encryption/CacheGroupKeyChangeTest.java
new file mode 100644
index 0000000..810e05d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/encryption/CacheGroupKeyChangeTest.java
@@ -0,0 +1,1063 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.encryption;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.managers.encryption.GridEncryptionManager;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
+import org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType;
+import org.apache.ignite.internal.util.distributed.InitMessage;
+import org.apache.ignite.internal.util.distributed.SingleNodeMessage;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
+import org.apache.ignite.testframework.GridTestUtils.DiscoveryHook;
+import org.junit.Test;
+
+import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
+import static org.apache.ignite.configuration.WALMode.NONE;
+import static org.apache.ignite.internal.managers.encryption.GridEncryptionManager.INITIAL_KEY_ID;
+import static org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionSpi.DEFAULT_MASTER_KEY_NAME;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
+/**
+ * Cache group key change distributed process tests.
+ */
+public class CacheGroupKeyChangeTest extends AbstractEncryptionTest {
+ /** Timeout. */
+ private static final long MAX_AWAIT_MILLIS = 15_000;
+
+ /** 1 megabyte in bytes. */
+ private static final int MB = 1024 * 1024;
+
+ /** */
+ private static final String GRID_2 = "grid-2";
+
+ /** Discovery hook for distributed process. */
+ private InitMessageDiscoveryHook discoveryHook;
+
+ /** Count of cache backups. */
+ private int backups;
+
+ /** Number of WAL segments. */
+ private int walSegments = 10;
+
+ /** WAL mode. */
+ private WALMode walMode = LOG_ONLY;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(name);
+
+ cfg.setConsistentId(name);
+ cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+ if (discoveryHook != null)
+ ((TestTcpDiscoverySpi)cfg.getDiscoverySpi()).discoveryHook(discoveryHook);
+
+ DataStorageConfiguration memCfg = new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setMaxSize(100 * MB)
+ .setPersistenceEnabled(true))
+ .setPageSize(4 * 1024)
+ .setWalSegmentSize(MB)
+ .setWalSegments(walSegments)
+ .setMaxWalArchiveSize(2 * walSegments * MB)
+ .setCheckpointFrequency(30 * 1000L)
+ .setWalMode(walMode);
+
+ cfg.setDataStorageConfiguration(memCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected <K, V> CacheConfiguration<K, V> cacheConfiguration(String name, String grp) {
+ CacheConfiguration<K, V> cfg = super.cacheConfiguration(name, grp);
+
+ return cfg.setAffinity(new RendezvousAffinityFunction(false, 8)).setBackups(backups);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ @SuppressWarnings("ThrowableNotThrown")
+ public void testRejectNodeJoinDuringRotation() throws Exception {
+ T2<IgniteEx, IgniteEx> grids = startTestGrids(true);
+
+ createEncryptedCache(grids.get1(), grids.get2(), cacheName(), null);
+
+ int grpId = CU.cacheId(cacheName());
+
+ assertEquals(0, grids.get1().context().encryption().groupKey(grpId).id());
+
+ TestRecordingCommunicationSpi commSpi = TestRecordingCommunicationSpi.spi(grids.get2());
+
+ commSpi.blockMessages((node, msg) -> msg instanceof SingleNodeMessage);
+
+ IgniteFuture<Void> fut = grids.get1().encryption().changeCacheGroupKey(Collections.singleton(cacheName()));
+
+ commSpi.waitForBlocked();
+
+ assertThrowsWithCause(() -> startGrid(3), IgniteCheckedException.class);
+
+ commSpi.stopBlock();
+
+ fut.get();
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ checkEncryptedCaches(grids.get1(), grids.get2());
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testNotAllBltNodesPresent() throws Exception {
+ startTestGrids(true);
+
+ createEncryptedCache(grid(GRID_0), grid(GRID_1), cacheName(), null);
+
+ stopGrid(GRID_1);
+
+ grid(GRID_0).encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ startGrid(GRID_1);
+
+ checkGroupKey(CU.cacheId(cacheName()), INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testNodeFailsBeforePrepare() throws Exception {
+ checkNodeFailsDuringRotation(false, true, true);
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testNodeFailsBeforePerform() throws Exception {
+ checkNodeFailsDuringRotation(false, false, true);
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testNodeFailsAfterPrepare() throws Exception {
+ checkNodeFailsDuringRotation(false, true, false);
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testCrdFailsAfterPrepare() throws Exception {
+ checkNodeFailsDuringRotation(true, true, false);
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testNodeFailsAfterPerform() throws Exception {
+ checkNodeFailsDuringRotation(false, false, false);
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testCrdFailsAfterPerform() throws Exception {
+ checkNodeFailsDuringRotation(true, false, false);
+ }
+
+ /**
+ * @param stopCrd {@code True} to stop coordinator.
+ * @param prepare {@code True} to stop on the prepare phase. {@code False} to stop on the perform phase.
+ * @param discoBlock {@code True} to block discovery, {@code False} to block communication SPI.
+ */
+ private void checkNodeFailsDuringRotation(boolean stopCrd, boolean prepare, boolean discoBlock) throws Exception {
+ cleanPersistenceDir();
+
+ DistributedProcessType type = prepare ?
+ DistributedProcessType.CACHE_GROUP_KEY_CHANGE_PREPARE : DistributedProcessType.CACHE_GROUP_KEY_CHANGE_FINISH;
+
+ InitMessageDiscoveryHook locHook = new InitMessageDiscoveryHook(type);
+
+ if (discoBlock && stopCrd)
+ discoveryHook = locHook;
+
+ IgniteEx grid0 = startGrid(GRID_0);
+
+ if (discoBlock && !stopCrd)
+ discoveryHook = locHook;
+
+ IgniteEx grid1 = startGrid(GRID_1);
+
+ grid0.cluster().state(ClusterState.ACTIVE);
+
+ createEncryptedCache(grid0, grid1, cacheName(), null);
+
+ int grpId = CU.cacheId(cacheName());
+
+ checkGroupKey(grpId, INITIAL_KEY_ID, MAX_AWAIT_MILLIS);
+
+ TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(grid1);
+
+ if (!discoBlock) {
+ AtomicBoolean preparePhase = new AtomicBoolean(true);
+
+ spi.blockMessages((node, msg) -> {
+ if (msg instanceof SingleNodeMessage) {
+ boolean isPrepare = preparePhase.compareAndSet(true, false);
+
+ return prepare || !isPrepare;
+ }
+
+ return false;
+ });
+ }
+
+ String alive = stopCrd ? GRID_1 : GRID_0;
+ String stopped = stopCrd ? GRID_0 : GRID_1;
+
+ IgniteFuture<Void> changeFut = grid(alive).encryption().changeCacheGroupKey(Collections.singleton(cacheName()));
+
+ IgniteInternalFuture<?> stopFut = new GridFinishedFuture<>();
+
+ if (discoBlock) {
+ locHook.waitForBlocked(MAX_AWAIT_MILLIS);
+
+ stopGrid(stopped, true);
+
+ locHook.stopBlock();
+ }
+ else {
+ spi.waitForBlocked();
+
+ stopFut = runAsync(() -> stopGrid(stopped, true));
+ }
+
+ changeFut.get(MAX_AWAIT_MILLIS);
+ stopFut.get(MAX_AWAIT_MILLIS);
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ IgniteEx stoppedNode = startGrid(stopped);
+
+ stoppedNode.resetLostPartitions(Collections.singleton(ENCRYPTED_CACHE));
+
+ awaitPartitionMapExchange();
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ stoppedNode.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS);
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 2, MAX_AWAIT_MILLIS);
+ }
+
+ /**
+ * Ensures that we can rotate the key more than 255 times.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testKeyIdentifierOverflow() throws Exception {
+ IgniteEx node = startTestGrids(true).get1();
+
+ createEncryptedCache(node, null, cacheName(), null, false);
+
+ int grpId = CU.cacheId(cacheName());
+
+ byte keyId = INITIAL_KEY_ID;
+
+ do {
+ node.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ // Validates reencryption of index partition.
+ checkGroupKey(grpId, ++keyId & 0xff, MAX_AWAIT_MILLIS);
+ } while (keyId != INITIAL_KEY_ID);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testMasterAndCacheGroupKeySimultaneousChange() throws Exception {
+ startTestGrids(true);
+
+ IgniteEx node0 = grid(GRID_0);
+ IgniteEx node1 = grid(GRID_1);
+
+ createEncryptedCache(node0, node1, cacheName(), null);
+
+ int grpId = CU.cacheId(cacheName());
+
+ assertTrue(checkMasterKeyName(DEFAULT_MASTER_KEY_NAME));
+
+ Random rnd = ThreadLocalRandom.current();
+
+ for (byte keyId = 1; keyId < 50; keyId++) {
+ String currMkName = node0.context().config().getEncryptionSpi().getMasterKeyName();
+ String newMkName = currMkName.equals(MASTER_KEY_NAME_2) ? MASTER_KEY_NAME_3 : MASTER_KEY_NAME_2;
+
+ boolean changeGrpFirst = rnd.nextBoolean();
+
+ IgniteFuture<Void> grpKeyFut;
+ IgniteFuture<Void> masterKeyFut;
+
+ if (changeGrpFirst) {
+ grpKeyFut = node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName()));
+ masterKeyFut = node0.encryption().changeMasterKey(newMkName);
+ }
+ else {
+ masterKeyFut = node0.encryption().changeMasterKey(newMkName);
+ grpKeyFut = node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName()));
+ }
+
+ masterKeyFut.get(MAX_AWAIT_MILLIS);
+ assertTrue(checkMasterKeyName(newMkName));
+
+ try {
+ grpKeyFut.get(MAX_AWAIT_MILLIS);
+ checkGroupKey(grpId, keyId, MAX_AWAIT_MILLIS);
+ } catch (IgniteException e) {
+ assertTrue(e.getMessage().contains("Cache group key change was rejected. Master key has been changed."));
+
+ // Retry iteration.
+ keyId -= 1;
+ }
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testCacheStartDuringRotation() throws Exception {
+ T2<IgniteEx, IgniteEx> grids = startTestGrids(true);
+
+ createEncryptedCache(grids.get1(), grids.get2(), cacheName(), null);
+
+ TestRecordingCommunicationSpi commSpi = TestRecordingCommunicationSpi.spi(grids.get2());
+
+ commSpi.blockMessages((node, msg) -> msg instanceof SingleNodeMessage);
+
+ IgniteFuture<Void> fut = grids.get1().encryption().changeCacheGroupKey(Collections.singleton(cacheName()));
+
+ commSpi.waitForBlocked();
+
+ IgniteCache<Integer, Integer> cache = grids.get1().createCache(cacheConfiguration("cache1", null));
+
+ for (int i = 0; i < 100; i++)
+ cache.put(i, i);
+
+ commSpi.stopBlock();
+
+ fut.get();
+
+ checkGroupKey(CU.cacheId(cacheName()), INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ checkGroupKey(CU.cacheId("cache1"), INITIAL_KEY_ID, MAX_AWAIT_MILLIS);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testCacheStartSameGroupDuringRotation() throws Exception {
+ T2<IgniteEx, IgniteEx> grids = startTestGrids(true);
+
+ String grpName = "shared";
+
+ createEncryptedCache(grids.get1(), grids.get2(), cacheName(), grpName);
+
+ TestRecordingCommunicationSpi commSpi = TestRecordingCommunicationSpi.spi(grids.get2());
+
+ commSpi.blockMessages((node, msg) -> msg instanceof SingleNodeMessage);
+
+ IgniteFuture<Void> fut = grids.get1().encryption().changeCacheGroupKey(Collections.singleton(grpName));
+
+ commSpi.waitForBlocked();
+
+ IgniteCache<Integer, Integer> cache =
+ grids.get1().createCache(cacheConfiguration("cache1", grpName));
+
+ commSpi.stopBlock();
+
+ for (int i = 0; i < 100; i++)
+ cache.put(i, i);
+
+ fut.get();
+
+ checkGroupKey(CU.cacheId(grpName), INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testChangeKeyDuringRebalancing() throws Exception {
+ T2<IgniteEx, IgniteEx> grids = startTestGrids(true);
+
+ IgniteEx node0 = grids.get1();
+ IgniteEx node1 = grids.get2();
+
+ createEncryptedCache(node0, node1, cacheName(), null);
+
+ loadData(500_000);
+
+ IgniteEx node2 = startGrid(GRID_2);
+
+ resetBaselineTopology();
+
+ int grpId = CU.cacheId(cacheName());
+
+ node2.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS);
+
+ awaitPartitionMapExchange();
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testNodeWithOlderKeyBecameCoordinator() throws Exception {
+ backups = 1;
+
+ startTestGrids(true);
+
+ IgniteEx node0 = grid(GRID_0);
+ IgniteEx node1 = grid(GRID_1);
+
+ createEncryptedCache(node0, node1, cacheName(), null);
+
+ int grpId = CU.cacheId(cacheName());
+
+ stopGrid(GRID_0);
+
+ // Changing encryption key on one node.
+ node1.context().encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS);
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ stopGrid(GRID_1);
+
+ // The node with only the old key ID has become the coordinator.
+ node0 = startGrid(GRID_0);
+ assertTrue(Collections.singleton(INITIAL_KEY_ID).containsAll(node0.context().encryption().groupKeyIds(grpId)));
+
+ node1 = startGrid(GRID_1);
+ node1.cluster().state(ClusterState.ACTIVE);
+
+ // Wait until cache will be reencrypted with the old key.
+ checkGroupKey(grpId, INITIAL_KEY_ID, MAX_AWAIT_MILLIS);
+
+ GridEncryptionManager encrMgr0 = node0.context().encryption();
+ GridEncryptionManager encrMgr1 = node1.context().encryption();
+
+ // Changing the encryption key is not possible until the WAL segment,
+ // encrypted (probably) with the previous key, is deleted.
+ assertThrowsAnyCause(log,
+ () -> encrMgr1.changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS),
+ IgniteException.class,
+ "Cache group key change was rejected. Cannot add new key identifier, it's already present.");
+
+ long walIdx = node1.context().cache().context().wal().currentSegment();
+
+ // Simulate WAL segment deletion.
+ for (long n = 0; n <= walIdx; n++)
+ node1.context().encryption().onWalSegmentRemoved(walIdx);
+
+ encrMgr1.changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS);
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ checkEncryptedCaches(node0, node1);
+
+ walIdx = Math.max(node0.context().cache().context().wal().currentSegment(),
+ node1.context().cache().context().wal().currentSegment());
+
+ // Simulate WAL segment deletion.
+ for (long n = 0; n <= walIdx; n++) {
+ encrMgr0.onWalSegmentRemoved(walIdx);
+ encrMgr1.onWalSegmentRemoved(walIdx);
+ }
+
+ // Make sure the previous key has been removed.
+ assertEquals(1, encrMgr0.groupKeyIds(grpId).size());
+ assertEquals(encrMgr1.groupKeyIds(grpId), encrMgr0.groupKeyIds(grpId));
+ }
+
+ /**
+ * Ensures that a node cannot join the cluster if it cannot replace an existing encryption key.
+ * <p>
+ * If the joining node has a different encryption key than the coordinator, but with the same identifier, it should
+ * not perform key rotation to a new key (recevied from coordinator) until the previous key is deleted.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testNodeJoinRejectedIfKeyCannotBeReplaced() throws Exception {
+ backups = 2;
+
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ startGrid(GRID_2);
+
+ resetBaselineTopology();
+
+ createEncryptedCache(nodes.get1(), nodes.get2(), cacheName(), null);
+
+ forceCheckpoint();
+
+ stopGrid(GRID_0);
+ stopGrid(GRID_1);
+
+ grid(GRID_2).encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS);
+
+ int grpId = CU.cacheId(cacheName());
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ grid(GRID_2).encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS);
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 2, MAX_AWAIT_MILLIS);
+
+ stopGrid(GRID_2);
+
+ startTestGrids(false);
+
+ checkGroupKey(grpId, INITIAL_KEY_ID, MAX_AWAIT_MILLIS);
+
+ grid(GRID_0).encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS);
+
+ assertThrowsAnyCause(log,
+ () -> startGrid(GRID_2),
+ IgniteSpiException.class,
+ "Cache key differs! Node join is rejected.");
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testKeyChangeWithNodeFilter() throws Exception {
+ startTestGrids(true);
+
+ IgniteEx node0 = grid(GRID_0);
+ IgniteEx node1 = grid(GRID_1);
+
+ Object nodeId0 = node0.localNode().consistentId();
+ Object nodeId1 = node1.localNode().consistentId();
+
+ String cache1 = cacheName();
+ String cache2 = "cache2";
+
+ node0.createCache(cacheConfiguration(cache1, null)
+ .setNodeFilter(node -> !node.consistentId().equals(nodeId0)));
+
+ node0.createCache(cacheConfiguration(cache2, null)
+ .setNodeFilter(node -> !node.consistentId().equals(nodeId1)));
+
+ loadData(10_000);
+
+ forceCheckpoint();
+
+ int grpId1 = CU.cacheId(cache1);
+ int grpId2 = CU.cacheId(cache2);
+
+ node0.encryption().changeCacheGroupKey(Arrays.asList(cache1, cache2)).get();
+
+ List<Integer> keys0 = node0.context().encryption().groupKeyIds(grpId1);
+ List<Integer> keys1 = node1.context().encryption().groupKeyIds(grpId1);
+
+ assertEquals(2, keys0.size());
+ assertEquals(2, keys1.size());
+
+ assertTrue(keys0.containsAll(keys1));
+
+ keys0 = node0.context().encryption().groupKeyIds(grpId2);
+ keys1 = node1.context().encryption().groupKeyIds(grpId2);
+
+ assertEquals(2, keys0.size());
+ assertEquals(2, keys1.size());
+
+ assertTrue(keys0.containsAll(keys1));
+
+ checkGroupKey(grpId1, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ checkGroupKey(grpId2, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ stopAllGrids();
+
+ startTestGrids(false);
+
+ node0 = grid(GRID_0);
+ node1 = grid(GRID_1);
+
+ IgniteCache<Object, Object> allNodesCache = node0.createCache("cacheX");
+
+ // Previous keys must be deleted when the corresponding WAL segment is deleted, so we adding data on all nodes.
+ long endTime = U.currentTimeMillis() + 30_000;
+ int cntr = 0;
+
+ do {
+ allNodesCache.put(cntr, String.valueOf(cntr));
+
+ if (node0.context().encryption().groupKeyIds(grpId1).size() == 1 &&
+ node1.context().encryption().groupKeyIds(grpId1).size() == 1 &&
+ node0.context().encryption().groupKeyIds(grpId2).size() == 1 &&
+ node1.context().encryption().groupKeyIds(grpId2).size() == 1)
+ break;
+
+ ++cntr;
+ } while (U.currentTimeMillis() < endTime);
+
+ assertEquals(1, node0.context().encryption().groupKeyIds(grpId1).size());
+ assertEquals(1, node0.context().encryption().groupKeyIds(grpId2).size());
+
+ assertEquals(node0.context().encryption().groupKeyIds(grpId1), node1.context().encryption().groupKeyIds(grpId1));
+ assertEquals(node0.context().encryption().groupKeyIds(grpId2), node1.context().encryption().groupKeyIds(grpId2));
+
+ checkGroupKey(grpId1, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ checkGroupKey(grpId2, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ checkEncryptedCaches(node0, node1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testBasicChangeWithConstantLoad() throws Exception {
+ walSegments = 20;
+
+ startTestGrids(true);
+
+ IgniteEx node0 = grid(GRID_0);
+ IgniteEx node1 = grid(GRID_1);
+
+ GridEncryptionManager encrMgr0 = node0.context().encryption();
+ GridEncryptionManager encrMgr1 = node1.context().encryption();
+
+ createEncryptedCache(node0, node1, cacheName(), null);
+
+ forceCheckpoint();
+
+ int grpId = CU.cacheId(cacheName());
+
+ IgniteInternalFuture<?> loadFut = loadDataAsync(node0);
+
+ try {
+ IgniteCache<Object, Object> cache = node0.cache(cacheName());
+
+ boolean success = waitForCondition(() -> cache.size() > 2000, MAX_AWAIT_MILLIS);
+ assertTrue(success);
+
+ node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS);
+
+ awaitEncryption(G.allGrids(), grpId, MAX_AWAIT_MILLIS);
+
+ waitForCondition(() ->
+ encrMgr0.groupKeyIds(grpId).size() == 1 && encrMgr1.groupKeyIds(grpId).size() == 1, MAX_AWAIT_MILLIS);
+ } finally {
+ loadFut.cancel();
+ }
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ assertEquals(node0.cluster().localNode().id().toString(), 1, encrMgr0.groupKeyIds(grpId).size());
+ assertEquals(node1.cluster().localNode().id().toString(), 1, encrMgr1.groupKeyIds(grpId).size());
+ }
+
+ /**
+ * Ensures that unused key will be removed even if user cleaned wal archive folder manually.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testWalArchiveCleanup() throws Exception {
+ cleanPersistenceDir();
+
+ IgniteEx node = startGrid(GRID_0);
+
+ node.cluster().state(ClusterState.ACTIVE);
+
+ createEncryptedCache(node, null, cacheName(), null);
+
+ node.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ IgniteWriteAheadLogManager walMgr = node.context().cache().context().wal();
+
+ long reservedIdx = walMgr.currentSegment();
+
+ boolean reserved = walMgr.reserve(new WALPointer(reservedIdx, 0, 0));
+ assertTrue(reserved);
+
+ IgniteInternalFuture<?> loadFut = loadDataAsync(node);
+
+ // Wait until the reserved segment is moved to the archive.
+ try {
+ boolean success = waitForCondition(() -> walMgr.lastArchivedSegment() >= reservedIdx, MAX_AWAIT_MILLIS);
+ assertTrue(success);
+ } finally {
+ loadFut.cancel();
+ }
+
+ forceCheckpoint();
+
+ int grpId = CU.cacheId(cacheName());
+
+ assertEquals(2, node.context().encryption().groupKeyIds(grpId).size());
+
+ stopAllGrids();
+
+ // Cleanup WAL arcive folder.
+ File dbDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false);
+
+ boolean rmvd = U.delete(new File(dbDir, "wal/archive"));
+
+ assertTrue(rmvd);
+
+ node = startGrid(GRID_0);
+
+ node.cluster().state(ClusterState.ACTIVE);
+
+ loadFut = loadDataAsync(node);
+
+ // Make sure that unused encryption key has been deleted.
+ try {
+ GridEncryptionManager encryptMgr = node.context().encryption();
+
+ boolean success = waitForCondition(() -> encryptMgr.groupKeyIds(grpId).size() == 1, MAX_AWAIT_MILLIS);
+ assertTrue(success);
+ } finally {
+ loadFut.cancel();
+ }
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /**
+ * @param grid Grid.
+ * @return Future for this operation.
+ */
+ private IgniteInternalFuture<?> loadDataAsync(Ignite grid) {
+ return runAsync(() -> {
+ long cntr = grid.cache(cacheName()).size();
+
+ try (IgniteDataStreamer<Long, String> streamer = grid.dataStreamer(cacheName())) {
+ while (!Thread.currentThread().isInterrupted()) {
+ streamer.addData(cntr, String.valueOf(cntr));
+
+ ++cntr;
+ }
+ }
+ });
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testCacheStartOnClientDuringRotation() throws Exception {
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ IgniteEx node0 = nodes.get1();
+ IgniteEx node1 = nodes.get2();
+
+ IgniteEx client = startClientGrid(getConfiguration("client"));
+
+ node0.cluster().state(ClusterState.ACTIVE);
+
+ String grpName = "shared";
+
+ createEncryptedCache(client, null, cacheName(), grpName);
+
+ awaitPartitionMapExchange();
+
+ TestRecordingCommunicationSpi commSpi = TestRecordingCommunicationSpi.spi(node1);
+
+ commSpi.blockMessages((node, message) -> message instanceof SingleNodeMessage);
+
+ IgniteFuture<Void> changeKeyFut = node0.encryption().changeCacheGroupKey(Collections.singleton(grpName));
+
+ commSpi.waitForBlocked();
+
+ String cacheName = "userCache";
+
+ IgniteInternalFuture<?> cacheStartFut = runAsync(() -> {
+ client.getOrCreateCache(cacheConfiguration(cacheName, grpName));
+ });
+
+ commSpi.stopBlock();
+
+ changeKeyFut.get(MAX_AWAIT_MILLIS);
+ cacheStartFut.get(MAX_AWAIT_MILLIS);
+
+ IgniteCache<Integer, String> cache = client.cache(cacheName);
+
+ for (int i = 0; i < 200; i++)
+ cache.put(i, String.valueOf(i));
+
+ checkEncryptedCaches(node0, client);
+
+ checkGroupKey(CU.cacheId(grpName), INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ checkEncryptedCaches(node0, node1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testClientJoinDuringRotation() throws Exception {
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ IgniteEx node0 = nodes.get1();
+ IgniteEx node1 = nodes.get2();
+
+ node0.cluster().state(ClusterState.ACTIVE);
+
+ createEncryptedCache(node0, node1, cacheName(), null);
+
+ awaitPartitionMapExchange();
+
+ TestRecordingCommunicationSpi commSpi = TestRecordingCommunicationSpi.spi(node1);
+
+ commSpi.blockMessages((node, message) -> message instanceof SingleNodeMessage);
+
+ IgniteFuture<Void> changeKeyFut = node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName()));
+
+ commSpi.waitForBlocked();
+
+ IgniteEx client = startClientGrid(getConfiguration("client"));
+
+ assertTrue(!changeKeyFut.isDone());
+
+ commSpi.stopBlock();
+
+ changeKeyFut.get(MAX_AWAIT_MILLIS);
+
+ checkEncryptedCaches(node0, client);
+
+ checkGroupKey(CU.cacheId(cacheName()), INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /**
+ * Ensures that node can join after rotation of encryption key.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testNodeJoinAfterRotation() throws Exception {
+ backups = 1;
+
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ createEncryptedCache(nodes.get1(), nodes.get2(), cacheName(), null);
+
+ forceCheckpoint();
+
+ stopGrid(GRID_1);
+ resetBaselineTopology();
+
+ nodes.get1().encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ startGrid(GRID_1);
+ resetBaselineTopology();
+ awaitPartitionMapExchange();
+
+ int grpId = CU.cacheId(cacheName());
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ checkEncryptedCaches(grid(GRID_0), grid(GRID_1));
+
+ GridEncryptionManager encrMgr0 = grid(GRID_0).context().encryption();
+ GridEncryptionManager encrMgr1 = grid(GRID_1).context().encryption();
+
+ long maxWalIdx = Math.max(nodes.get1().context().cache().context().wal().currentSegment(),
+ nodes.get2().context().cache().context().wal().currentSegment());
+
+ for (long idx = 0; idx <= maxWalIdx; idx++) {
+ encrMgr0.onWalSegmentRemoved(maxWalIdx);
+ encrMgr1.onWalSegmentRemoved(maxWalIdx);
+ }
+
+ assertEquals(1, encrMgr1.groupKeyIds(grpId).size());
+ assertEquals(encrMgr0.groupKeyIds(grpId), encrMgr1.groupKeyIds(grpId));
+
+ startGrid(GRID_2);
+
+ resetBaselineTopology();
+ awaitPartitionMapExchange();
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ checkEncryptedCaches(grid(GRID_2), nodes.get1());
+
+ assertEquals(encrMgr0.groupKeyIds(grpId), grid(GRID_2).context().encryption().groupKeyIds(grpId));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testWrongCacheGroupSpecified() throws Exception {
+ T2<IgniteEx, IgniteEx> grids = startTestGrids(true);
+
+ IgniteEx node0 = grids.get1();
+ IgniteEx node1 = grids.get2();
+
+ assertThrowsAnyCause(log,
+ () -> node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS),
+ IgniteException.class,
+ "Cache group key change was rejected. Cache or group \"" + cacheName() + "\" doesn't exists");
+
+ node0.createCache(new CacheConfiguration<>(cacheName()).setNodeFilter(node -> node.equals(node0.localNode())));
+
+ assertThrowsAnyCause(log,
+ () -> node1.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS),
+ IgniteException.class,
+ "Cache group key change was rejected. Cache or group \"" + cacheName() + "\" is not encrypted.");
+
+ node0.destroyCache(cacheName());
+
+ awaitPartitionMapExchange();
+
+ String grpName = "cacheGroup1";
+
+ createEncryptedCache(node0, node1, cacheName(), grpName);
+
+ assertThrowsAnyCause(log,
+ () -> node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS),
+ IgniteException.class,
+ "Cache group key change was rejected. Cache or group \"" + cacheName() + "\" is a part of group \"" +
+ grpName + "\". Provide group name instead of cache name for shared groups.");
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testChangeCacheGroupKeyWithoutWAL() throws Exception {
+ walMode = NONE;
+ T2<IgniteEx, IgniteEx> grids = startTestGrids(true);
+
+ createEncryptedCache(grids.get1(), grids.get2(), cacheName(), null);
+
+ IgniteEx node0 = grids.get1();
+
+ node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ int grpId = CU.cacheId(cacheName());
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ assertEquals(1, node0.context().encryption().groupKeyIds(grpId).size());
+ assertEquals(1, grids.get2().context().encryption().groupKeyIds(grpId).size());
+ }
+
+ /**
+ * Custom discovery hook to block distributed process.
+ */
+ private static class InitMessageDiscoveryHook extends DiscoveryHook {
+ /**
+ * Latch to sync execution.
+ */
+ private final CountDownLatch unlockLatch = new CountDownLatch(1);
+
+ /**
+ * Latch to sync execution.
+ */
+ private final CountDownLatch blockedLatch = new CountDownLatch(1);
+
+ /**
+ * Distributed process type.
+ */
+ private final DistributedProcessType type;
+
+ /**
+ * @param type Distributed process type.
+ */
+ private InitMessageDiscoveryHook(DistributedProcessType type) {
+ this.type = type;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void beforeDiscovery(DiscoveryCustomMessage customMsg) {
+ if (!(customMsg instanceof InitMessage))
+ return;
+
+ InitMessage<Serializable> msg = (InitMessage<Serializable>)customMsg;
+
+ if (msg.type() != type.ordinal())
+ return;
+
+ try {
+ blockedLatch.countDown();
+
+ unlockLatch.await(MAX_AWAIT_MILLIS, TimeUnit.MILLISECONDS);
+ }
+ catch (InterruptedException ignore) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * @param timeout Timeout in milliseconds.
+ * @throws InterruptedException If interrupted.
+ */
+ public void waitForBlocked(long timeout) throws InterruptedException {
+ blockedLatch.await(timeout, TimeUnit.MILLISECONDS);
+ }
+
+ /** */
+ public void stopBlock() {
+ unlockLatch.countDown();
+ }
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/encryption/CacheGroupReencryptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/encryption/CacheGroupReencryptionTest.java
new file mode 100644
index 0000000..1f7cf36
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/encryption/CacheGroupReencryptionTest.java
@@ -0,0 +1,867 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.encryption;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.OpenOption;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.EncryptionConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteFutureCancelledCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.CacheGroupMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.metric.MetricRegistry;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.spi.metric.BooleanMetric;
+import org.apache.ignite.spi.metric.LongMetric;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.Test;
+
+import static org.apache.ignite.configuration.EncryptionConfiguration.DFLT_REENCRYPTION_RATE_MBPS;
+import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
+import static org.apache.ignite.internal.managers.encryption.GridEncryptionManager.INITIAL_KEY_ID;
+import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
+
+/**
+ * Cache re-encryption tests.
+ */
+public class CacheGroupReencryptionTest extends AbstractEncryptionTest {
+ /** */
+ private static final String GRID_2 = "grid-2";
+
+ /** */
+ private static final String GRID_3 = "grid-3";
+
+ /** Timeout. */
+ private static final long MAX_AWAIT_MILLIS = 15_000;
+
+ /** File IO fail flag. */
+ private final AtomicBoolean failFileIO = new AtomicBoolean();
+
+ /** Count of cache backups. */
+ private int backups;
+
+ /** Re-encryption rate limit. */
+ private double pageScanRate = DFLT_REENCRYPTION_RATE_MBPS;
+
+ /** The number of pages that is scanned during re-encryption under checkpoint lock. */
+ private int pageScanBatchSize = EncryptionConfiguration.DFLT_REENCRYPTION_BATCH_SIZE;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(name);
+
+ cfg.setConsistentId(name);
+
+ cfg.setIncludeEventTypes(EventType.EVT_CACHE_REBALANCE_STOPPED);
+
+ EncryptionConfiguration encCfg = new EncryptionConfiguration()
+ .setReencryptionBatchSize(pageScanBatchSize)
+ .setReencryptionRateLimit(pageScanRate);
+
+ DataStorageConfiguration memCfg = new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setMaxSize(1024 * 1024 * 1024L)
+ .setPersistenceEnabled(true))
+ .setPageSize(4 * 1024)
+ .setWalSegmentSize(10 * 1024 * 1024)
+ .setWalSegments(4)
+ .setMaxWalArchiveSize(100 * 1024 * 1024L)
+ .setCheckpointFrequency(30 * 1000L)
+ .setWalMode(LOG_ONLY)
+ .setFileIOFactory(new FailingFileIOFactory(new RandomAccessFileIOFactory(), failFileIO))
+ .setEncryptionConfiguration(encCfg);
+
+ cfg.setDataStorageConfiguration(memCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected <K, V> CacheConfiguration<K, V> cacheConfiguration(String name, String grp) {
+ CacheConfiguration<K, V> cfg = super.cacheConfiguration(name, grp);
+
+ cfg.setIndexedTypes(Long.class, IndexedObject.class);
+
+ return cfg.setAffinity(new RendezvousAffinityFunction(false, 16)).setBackups(backups);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Object generateValue(long id) {
+ return new IndexedObject(id, "string-" + id);
+ }
+
+ /**
+ * Check physical recovery after checkpoint failure during re-encryption.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPhysicalRecovery() throws Exception {
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ createEncryptedCache(nodes.get1(), nodes.get2(), cacheName(), null);
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runAsync(() -> loadData(50_000));
+
+ forceCheckpoint();
+
+ enableCheckpoints(nodes.get1(), false);
+ enableCheckpoints(nodes.get2(), false);
+
+ int grpId = CU.cacheId(cacheName());
+
+ failFileIO.set(true);
+
+ nodes.get1().encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ awaitEncryption(G.allGrids(), grpId, MAX_AWAIT_MILLIS);
+
+ fut.get();
+
+ assertThrowsAnyCause(log, () -> {
+ enableCheckpoints(grid(GRID_0), true);
+ enableCheckpoints(grid(GRID_1), true);
+
+ forceCheckpoint();
+
+ return null;
+ }, IgniteCheckedException.class, null);
+
+ stopAllGrids(true);
+
+ failFileIO.set(false);
+
+ nodes = startTestGrids(false);
+
+ checkEncryptedCaches(nodes.get1(), nodes.get2());
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testPhysicalRecoveryWithUpdates() throws Exception {
+ pageScanRate = 1.5;
+
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ createEncryptedCache(nodes.get1(), nodes.get2(), cacheName(), null);
+
+ loadData(50_000);
+
+ IgniteInternalFuture<?> addFut = GridTestUtils.runAsync(() -> loadData(100_000));
+
+ IgniteInternalFuture<?> updateFut = GridTestUtils.runAsync(() -> {
+ IgniteCache<Long, String> cache = grid(GRID_0).cache(cacheName());
+
+ while (!Thread.currentThread().isInterrupted()) {
+ for (long i = 50_000; i > 20_000; i--) {
+ String val = cache.get(i);
+
+ cache.put(i, val);
+ }
+ }
+ });
+
+ forceCheckpoint();
+
+ int grpId = CU.cacheId(cacheName());
+
+ nodes.get1().encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ forceCheckpoint();
+
+ failFileIO.set(true);
+
+ awaitEncryption(G.allGrids(), grpId, MAX_AWAIT_MILLIS);
+
+ addFut.get();
+ updateFut.cancel();
+
+ assertThrowsAnyCause(log, () -> {
+ forceCheckpoint();
+
+ return null;
+ }, IgniteCheckedException.class, null);
+
+ stopAllGrids(true);
+
+ failFileIO.set(false);
+
+ nodes = startTestGrids(false);
+
+ checkEncryptedCaches(nodes.get1(), nodes.get2());
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /**
+ * Ensures that re-encryption continues after a restart.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testLogicalRecovery() throws Exception {
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ IgniteEx node0 = nodes.get1();
+ IgniteEx node1 = nodes.get2();
+
+ createEncryptedCache(node0, node1, cacheName(), null, true);
+
+ loadData(100_000);
+
+ forceCheckpoint();
+
+ enableCheckpoints(G.allGrids(), false);
+
+ int grpId = CU.cacheId(cacheName());
+
+ node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ awaitEncryption(G.allGrids(), grpId, MAX_AWAIT_MILLIS);
+
+ assertEquals(1, node0.context().encryption().groupKey(grpId).id());
+ assertEquals(1, node1.context().encryption().groupKey(grpId).id());
+
+ stopAllGrids();
+
+ info(">>> Start grids (iteration 1)");
+
+ startTestGrids(false);
+
+ enableCheckpoints(G.allGrids(), false);
+
+ stopAllGrids();
+
+ info(">>> Start grids (iteration 2)");
+
+ startTestGrids(false);
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testCacheStopDuringReencryption() throws Exception {
+ pageScanRate = 1;
+
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ IgniteEx node0 = nodes.get1();
+ IgniteEx node1 = nodes.get2();
+
+ createEncryptedCache(node0, node1, cacheName(), null);
+
+ loadData(100_000);
+
+ IgniteCache<?, ?> cache = node0.cache(cacheName());
+
+ node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ int grpId = CU.cacheId(cacheName());
+
+ IgniteInternalFuture<Void> fut0 = node0.context().encryption().reencryptionFuture(grpId);
+
+ assertFalse(fut0.isDone());
+
+ assertTrue(isReencryptionInProgress(node0, grpId));
+
+ cache.destroy();
+
+ assertThrowsAnyCause(log, () -> {
+ fut0.get();
+
+ return null;
+ }, IgniteFutureCancelledCheckedException.class, null);
+
+ awaitPartitionMapExchange();
+
+ assertNull(node0.context().encryption().groupKeyIds(grpId));
+ assertNull(node1.context().encryption().groupKeyIds(grpId));
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testPartitionEvictionDuringReencryption() throws Exception {
+ backups = 1;
+ pageScanRate = 1;
+
+ CountDownLatch rebalanceFinished = new CountDownLatch(1);
+
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ IgniteEx node0 = nodes.get1();
+ IgniteEx node1 = nodes.get2();
+
+ createEncryptedCache(node0, node1, cacheName(), null);
+
+ loadData(100_000);
+
+ IgniteEx node2 = startGrid(GRID_2);
+
+ node2.events().localListen(evt -> {
+ rebalanceFinished.countDown();
+
+ return true;
+ }, EventType.EVT_CACHE_REBALANCE_STOPPED);
+
+ resetBaselineTopology();
+
+ rebalanceFinished.await();
+
+ stopGrid(GRID_2);
+
+ resetBaselineTopology();
+
+ int grpId = CU.cacheId(cacheName());
+
+ node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ stopAllGrids();
+
+ pageScanRate = DFLT_REENCRYPTION_RATE_MBPS;
+
+ startTestGrids(false);
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /**
+ * Test that partition files are reused correctly.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPartitionFileDestroy() throws Exception {
+ backups = 1;
+ pageScanRate = 1;
+ pageScanBatchSize = 10;
+
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ createEncryptedCache(nodes.get1(), nodes.get2(), cacheName(), null);
+
+ loadData(50_000);
+
+ forceCheckpoint();
+
+ nodes.get1().encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ startGrid(GRID_2);
+
+ // Trigger partitions eviction.
+ resetBaselineTopology();
+
+ awaitPartitionMapExchange(true, true, null);
+
+ forceCheckpoint();
+
+ assertTrue(isReencryptionInProgress(Collections.singleton(cacheName())));
+
+ checkGroupKey(CU.cacheId(cacheName()), INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /**
+ * Test that partition files are reused correctly.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPartitionFileDestroyAndRecreate() throws Exception {
+ backups = 1;
+ pageScanRate = 1;
+
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ createEncryptedCache(nodes.get1(), nodes.get2(), cacheName(), null);
+
+ loadData(50_000);
+
+ grid(GRID_0).encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ long walSegment = nodes.get1().context().cache().context().wal().currentSegment();
+
+ for (long n = 0; n <= walSegment; n++)
+ nodes.get1().context().encryption().onWalSegmentRemoved(n);
+
+ walSegment = nodes.get2().context().cache().context().wal().currentSegment();
+
+ for (long n = 0; n <= walSegment; n++)
+ nodes.get2().context().encryption().onWalSegmentRemoved(n);
+
+ // Force checkpoint to prevent logical recovery after key rotation.
+ forceCheckpoint();
+
+ startGrid(GRID_2);
+
+ // Trigger partitions eviction.
+ resetBaselineTopology();
+
+ awaitPartitionMapExchange(true, true, null);
+
+ // Trigger partitions re-create.
+ stopGrid(GRID_2);
+
+ resetBaselineTopology();
+
+ awaitPartitionMapExchange(true, true, null);
+
+ stopAllGrids();
+
+ nodes = startTestGrids(false);
+
+ checkEncryptedCaches(nodes.get1(), nodes.get2());
+
+ checkGroupKey(CU.cacheId(cacheName()), INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testNotBltNodeJoin() throws Exception {
+ backups = 1;
+ pageScanRate = 1;
+ pageScanBatchSize = 10;
+
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ createEncryptedCache(nodes.get1(), nodes.get2(), cacheName(), null);
+
+ loadData(50_000);
+
+ forceCheckpoint();
+
+ long startIdx1 = nodes.get1().context().cache().context().wal().currentSegment();
+ long startIdx2 = nodes.get2().context().cache().context().wal().currentSegment();
+
+ nodes.get1().encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ long endIdx1 = nodes.get1().context().cache().context().wal().currentSegment();
+ long endIdx2 = nodes.get2().context().cache().context().wal().currentSegment();
+
+ stopGrid(GRID_1);
+
+ resetBaselineTopology();
+
+ int grpId = CU.cacheId(cacheName());
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ startGrid(GRID_1);
+
+ resetBaselineTopology();
+
+ awaitPartitionMapExchange();
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ assertEquals(2, grid(GRID_0).context().encryption().groupKeyIds(grpId).size());
+ assertEquals(2, grid(GRID_1).context().encryption().groupKeyIds(grpId).size());
+
+ // Simulate that wal was removed.
+ for (long segment = startIdx1; segment <= endIdx1; segment++)
+ grid(GRID_0).context().encryption().onWalSegmentRemoved(segment);
+
+ assertEquals(1, grid(GRID_0).context().encryption().groupKeyIds(grpId).size());
+
+ for (long segment = startIdx2; segment <= endIdx2; segment++)
+ grid(GRID_1).context().encryption().onWalSegmentRemoved(segment);
+
+ assertEquals(1, grid(GRID_1).context().encryption().groupKeyIds(grpId).size());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testReencryptionStartsAfterNodeRestart() throws Exception {
+ pageScanRate = 0.000000001;
+
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ IgniteEx node0 = nodes.get1();
+ IgniteEx node1 = nodes.get2();
+
+ createEncryptedCache(node0, node1, cacheName(), null);
+
+ forceCheckpoint();
+
+ int grpId = CU.cacheId(cacheName());
+
+ node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ forceCheckpoint();
+
+ stopAllGrids();
+
+ nodes = startTestGrids(false);
+
+ node0 = nodes.get1();
+ node1 = nodes.get2();
+
+ assertTrue(isReencryptionInProgress(node0, grpId));
+ assertTrue(isReencryptionInProgress(node1, grpId));
+
+ stopAllGrids();
+
+ pageScanRate = DFLT_REENCRYPTION_RATE_MBPS;
+
+ startTestGrids(false);
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testReencryptionOnUnstableTopology() throws Exception {
+ backups = 1;
+ pageScanRate = 2;
+
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ IgniteEx node0 = nodes.get1();
+ IgniteEx node1 = nodes.get2();
+
+ startGrid(GRID_2);
+ startGrid(GRID_3);
+
+ resetBaselineTopology();
+
+ createEncryptedCache(node0, node1, cacheName(), null);
+
+ String cache2 = "encrypted-2";
+
+ createEncryptedCache(node0, node1, cache2, null);
+
+ loadData(cacheName(), 100_000);
+ loadData(cache2, 100_000);
+
+ List<String> cacheGroups = Arrays.asList(cacheName(), cache2);
+
+ node0.encryption().changeCacheGroupKey(cacheGroups).get();
+
+ while (isReencryptionInProgress(cacheGroups)) {
+ int rndNode = ThreadLocalRandom.current().nextInt(3);
+
+ String gridName = "grid-" + rndNode;
+
+ stopGrid(gridName);
+
+ startGrid(gridName);
+ }
+
+ stopAllGrids();
+
+ startGrid(GRID_0);
+ startGrid(GRID_1);
+ startGrid(GRID_2);
+ startGrid(GRID_3);
+
+ grid(GRID_0).cluster().state(ClusterState.ACTIVE);
+
+ awaitPartitionMapExchange();
+
+ checkGroupKey(CU.cacheId(cacheName()), INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ checkGroupKey(CU.cacheId(cache2), INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testChangeBaseline() throws Exception {
+ backups = 1;
+ pageScanRate = 2;
+
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ IgniteEx node0 = nodes.get1();
+ IgniteEx node1 = nodes.get2();
+
+ createEncryptedCache(node0, node1, cacheName(), null);
+
+ loadData(100_000);
+
+ node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ assertTrue(isReencryptionInProgress(Collections.singleton(cacheName())));
+
+ startGrid(GRID_2);
+
+ resetBaselineTopology();
+
+ startGrid(GRID_3);
+
+ resetBaselineTopology();
+
+ awaitPartitionMapExchange();
+
+ checkGroupKey(CU.cacheId(cacheName()), INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+
+ node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ stopGrid(GRID_2);
+
+ resetBaselineTopology();
+
+ awaitPartitionMapExchange();
+
+ checkGroupKey(CU.cacheId(cacheName()), INITIAL_KEY_ID + 2, MAX_AWAIT_MILLIS);
+
+ node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ startGrid(GRID_2);
+
+ resetBaselineTopology();
+
+ awaitPartitionMapExchange();
+
+ checkGroupKey(CU.cacheId(cacheName()), INITIAL_KEY_ID + 3, MAX_AWAIT_MILLIS);
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testKeyCleanup() throws Exception {
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ IgniteEx node0 = nodes.get1();
+ IgniteEx node1 = nodes.get2();
+
+ createEncryptedCache(node0, node1, cacheName(), null);
+
+ forceCheckpoint();
+
+ enableCheckpoints(G.allGrids(), false);
+
+ int grpId = CU.cacheId(cacheName());
+
+ long startIdx = node1.context().cache().context().wal().currentSegment();
+
+ node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ long endIdx = node1.context().cache().context().wal().currentSegment();
+
+ awaitEncryption(G.allGrids(), grpId, MAX_AWAIT_MILLIS);
+
+ // Simulate that wal was removed.
+ for (long segment = startIdx; segment <= endIdx; segment++)
+ node1.context().encryption().onWalSegmentRemoved(segment);
+
+ stopGrid(GRID_1);
+
+ node1 = startGrid(GRID_1);
+
+ enableCheckpoints(G.allGrids(), true);
+
+ node1.cluster().state(ClusterState.ACTIVE);
+
+ node1.resetLostPartitions(Collections.singleton(ENCRYPTED_CACHE));
+
+ checkEncryptedCaches(node0, node1);
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
+ }
+
+ /** @throws Exception If failed. */
+ @Test
+ public void testReencryptionMetrics() throws Exception {
+ pageScanRate = 0.000000001;
+
+ T2<IgniteEx, IgniteEx> nodes = startTestGrids(true);
+
+ IgniteEx node0 = nodes.get1();
+ IgniteEx node1 = nodes.get2();
+
+ createEncryptedCache(node0, node1, cacheName(), null);
+
+ node0.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
+
+ validateMetrics(node0, false);
+ validateMetrics(node1, false);
+
+ pageScanRate = DFLT_REENCRYPTION_RATE_MBPS;
+
+ stopAllGrids();
+
+ nodes = startTestGrids(false);
+
+ node0 = nodes.get1();
+ node1 = nodes.get2();
+
+ awaitEncryption(G.allGrids(), CU.cacheId(cacheName()), MAX_AWAIT_MILLIS);
+
+ forceCheckpoint();
+
+ validateMetrics(node0, true);
+ validateMetrics(node1, true);
+ }
+
+ /**
+ * @param node Grid.
+ * @param finished Expected reencryption status.
+ */
+ private void validateMetrics(IgniteEx node, boolean finished) {
+ MetricRegistry registry =
+ node.context().metric().registry(metricName(CacheGroupMetricsImpl.CACHE_GROUP_METRICS_PREFIX, cacheName()));
+
+ LongMetric pagesLeft = registry.findMetric("ReencryptionPagesLeft");
+
+ if (finished)
+ assertEquals(0, pagesLeft.value());
+ else
+ assertTrue(pagesLeft.value() > 0);
+
+ BooleanMetric reencryptionFinished = registry.findMetric("ReencryptionFinished");
+
+ assertEquals(finished, reencryptionFinished.value());
+ }
+
+ /**
+ * @param cacheGroups Cache group names.
+ * @return {@code True} If reencryption of the specified groups is not yet complete.
+ */
+ private boolean isReencryptionInProgress(Iterable<String> cacheGroups) {
+ for (Ignite node : G.allGrids()) {
+ for (String groupName : cacheGroups) {
+ if (isReencryptionInProgress((IgniteEx)node, CU.cacheId(groupName)))
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /** */
+ private static final class FailingFileIOFactory implements FileIOFactory {
+ /** */
+ private final FileIOFactory delegateFactory;
+
+ /** */
+ private final AtomicBoolean failFlag;
+
+ /**
+ * @param factory Delegate factory.
+ */
+ FailingFileIOFactory(FileIOFactory factory, AtomicBoolean failFlag) {
+ delegateFactory = factory;
+
+ this.failFlag = failFlag;
+ }
+
+ /** {@inheritDoc}*/
+ @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+ FileIO delegate = delegateFactory.create(file, modes);
+
+ return new FailingFileIO(delegate);
+ }
+
+ /** */
+ final class FailingFileIO extends FileIODecorator {
+ /**
+ * @param delegate File I/O delegate
+ */
+ public FailingFileIO(FileIO delegate) {
+ super(delegate);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int writeFully(ByteBuffer srcBuf, long position) throws IOException {
+ if (failFlag.get())
+ throw new IOException("Test exception.");
+
+ return delegate.writeFully(srcBuf, position);
+ }
+ }
+ }
+
+ /** */
+ private static class IndexedObject {
+ /** Id. */
+ @QuerySqlField(index = true)
+ private final long id;
+
+ /** Name. */
+ @QuerySqlField(index = true)
+ private final String name;
+
+ /**
+ * @param id Id.
+ */
+ public IndexedObject(long id, String name) {
+ this.id = id;
+ this.name = name;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ IndexedObject obj = (IndexedObject)o;
+
+ return id == obj.id && Objects.equals(name, obj.name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hash(name, id);
+ }
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheBigEntryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheBigEntryTest.java
index 9220202..613f376 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheBigEntryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheBigEntryTest.java
@@ -22,6 +22,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.encryption.GroupKey;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionKey;
@@ -59,7 +60,7 @@ public class EncryptedCacheBigEntryTest extends AbstractEncryptionTest {
int grpId = CU.cacheGroupId(cacheName(), null);
KeystoreEncryptionKey keyBeforeRestart =
- (KeystoreEncryptionKey)grids.get1().context().encryption().groupKey(grpId);
+ (KeystoreEncryptionKey)grids.get1().context().encryption().groupKey(grpId).key();
stopAllGrids();
@@ -67,7 +68,11 @@ public class EncryptedCacheBigEntryTest extends AbstractEncryptionTest {
checkEncryptedCaches(grids.get1(), grids.get2());
- KeystoreEncryptionKey keyAfterRestart = (KeystoreEncryptionKey)grids.get1().context().encryption().groupKey(grpId);
+ GroupKey grpKeyAfterRestart = grids.get1().context().encryption().groupKey(grpId);
+
+ assertNotNull(grpKeyAfterRestart);
+
+ KeystoreEncryptionKey keyAfterRestart = (KeystoreEncryptionKey)grpKeyAfterRestart.key();
assertNotNull(keyAfterRestart);
assertNotNull(keyAfterRestart.key());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheCreateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheCreateTest.java
index aaf880a..7f5f3ae 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheCreateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheCreateTest.java
@@ -28,6 +28,7 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.managers.encryption.GroupKey;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -89,8 +90,11 @@ public class EncryptedCacheCreateTest extends AbstractEncryptionTest {
assertNotNull(enc);
- KeystoreEncryptionKey key =
- (KeystoreEncryptionKey)grid.context().encryption().groupKey(CU.cacheGroupId(ENCRYPTED_CACHE, null));
+ GroupKey grpKey = grid.context().encryption().groupKey(CU.cacheGroupId(ENCRYPTED_CACHE, null));
+
+ assertNotNull(grpKey);
+
+ KeystoreEncryptionKey key = (KeystoreEncryptionKey)grpKey.key();
assertNotNull(key);
assertNotNull(key.key());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheDestroyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheDestroyTest.java
index 4fe9f59..1f5f0cf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheDestroyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheDestroyTest.java
@@ -19,13 +19,13 @@ package org.apache.ignite.internal.encryption;
import java.util.Collection;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.managers.encryption.GroupKey;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.CU;
-import org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionKey;
import org.junit.Test;
-import static org.apache.ignite.internal.managers.encryption.GridEncryptionManager.ENCRYPTION_KEY_PREFIX;
+import static org.apache.ignite.internal.managers.encryption.GridEncryptionManager.ENCRYPTION_KEYS_PREFIX;
/**
*/
@@ -114,17 +114,17 @@ public class EncryptedCacheDestroyTest extends AbstractEncryptionTest {
int grpId = CU.cacheGroupId(encCacheName, grpName);
- KeystoreEncryptionKey encKey = (KeystoreEncryptionKey)grid.context().encryption().groupKey(grpId);
+ GroupKey encKey = grid.context().encryption().groupKey(grpId);
MetaStorage metaStore = grid.context().cache().context().database().metaStorage();
if (keyShouldBeEmpty) {
assertNull(encKey);
- assertNull(metaStore.readRaw(ENCRYPTION_KEY_PREFIX + grpId));
+ assertNull(metaStore.readRaw(ENCRYPTION_KEYS_PREFIX + grpId));
} else {
assertNotNull(encKey);
- assertNotNull(metaStore.readRaw(ENCRYPTION_KEY_PREFIX + grpId));
+ assertNotNull(metaStore.readRaw(ENCRYPTION_KEYS_PREFIX + grpId));
}
}
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheGroupCreateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheGroupCreateTest.java
index 12fd8be..4caf303 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheGroupCreateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheGroupCreateTest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.managers.encryption.GridEncryptionManager;
+import org.apache.ignite.internal.managers.encryption.GroupKey;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionKey;
@@ -71,7 +72,11 @@ public class EncryptedCacheGroupCreateTest extends AbstractEncryptionTest {
GridEncryptionManager encMgr = encrypted2.context().kernalContext().encryption();
- KeystoreEncryptionKey key2 = (KeystoreEncryptionKey)encMgr.groupKey(CU.cacheGroupId(ENCRYPTED_CACHE, ENCRYPTED_GROUP));
+ GroupKey grpKey2 = encMgr.groupKey(CU.cacheGroupId(ENCRYPTED_CACHE, ENCRYPTED_GROUP));
+
+ assertNotNull(grpKey2);
+
+ KeystoreEncryptionKey key2 = (KeystoreEncryptionKey)grpKey2.key();
assertNotNull(key2);
assertNotNull(key2.key());
@@ -108,8 +113,11 @@ public class EncryptedCacheGroupCreateTest extends AbstractEncryptionTest {
assertNotNull(enc);
- KeystoreEncryptionKey key =
- (KeystoreEncryptionKey)grid.context().encryption().groupKey(CU.cacheGroupId(cacheName, grpName));
+ GroupKey grpKey = grid.context().encryption().groupKey(CU.cacheGroupId(cacheName, grpName));
+
+ assertNotNull(grpKey);
+
+ KeystoreEncryptionKey key = (KeystoreEncryptionKey)grpKey.key();
assertNotNull(key);
assertNotNull(key.key());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheRestartTest.java
index 9107ddf..dd2a50a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptedCacheRestartTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.encryption;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.managers.encryption.GroupKey;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionKey;
@@ -48,7 +49,8 @@ public class EncryptedCacheRestartTest extends AbstractEncryptionTest {
int grpId = CU.cacheGroupId(cacheName(), null);
- KeystoreEncryptionKey keyBeforeRestart = (KeystoreEncryptionKey)grids.get1().context().encryption().groupKey(grpId);
+ KeystoreEncryptionKey keyBeforeRestart =
+ (KeystoreEncryptionKey)grids.get1().context().encryption().groupKey(grpId).key();
stopAllGrids();
@@ -56,7 +58,11 @@ public class EncryptedCacheRestartTest extends AbstractEncryptionTest {
checkEncryptedCaches(grids.get1(), grids.get2());
- KeystoreEncryptionKey keyAfterRestart = (KeystoreEncryptionKey)grids.get1().context().encryption().groupKey(grpId);
+ GroupKey grpKeyAfterRestart = grids.get1().context().encryption().groupKey(grpId);
+
+ assertNotNull(grpKeyAfterRestart);
+
+ KeystoreEncryptionKey keyAfterRestart = (KeystoreEncryptionKey)grpKeyAfterRestart.key();
assertNotNull(keyAfterRestart);
assertNotNull(keyAfterRestart.key());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptionMXBeanTest.java b/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptionMXBeanTest.java
index ab5bf52..5844b5e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptionMXBeanTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/encryption/EncryptionMXBeanTest.java
@@ -20,10 +20,12 @@ package org.apache.ignite.internal.encryption;
import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.managers.encryption.EncryptionMXBeanImpl;
+import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.mxbean.EncryptionMXBean;
import org.junit.Test;
import static org.apache.ignite.cluster.ClusterState.ACTIVE_READ_ONLY;
+import static org.apache.ignite.internal.managers.encryption.GridEncryptionManager.INITIAL_KEY_ID;
import static org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionSpi.DEFAULT_MASTER_KEY_NAME;
import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
@@ -110,6 +112,28 @@ public class EncryptionMXBeanTest extends AbstractEncryptionTest {
assertEquals(MASTER_KEY_NAME_2, grid0.encryption().getMasterKeyName());
}
+ /** @throws Exception If failed. */
+ @Test
+ public void testCacheGroupKeyChange() throws Exception {
+ IgniteEx ignite = startGrid(GRID_0);
+
+ ignite.cluster().active(true);
+
+ createEncryptedCache(ignite, null, cacheName(), null);
+
+ EncryptionMXBean mBean = getMBean(GRID_0);
+
+ int grpId = CU.cacheId(cacheName());
+
+ assertEquals(INITIAL_KEY_ID, ignite.context().encryption().groupKey(grpId).id());
+
+ mBean.changeCacheGroupKey(cacheName());
+
+ assertEquals(INITIAL_KEY_ID + 1, ignite.context().encryption().groupKey(grpId).id());
+
+ checkGroupKey(grpId, INITIAL_KEY_ID + 1, getTestTimeout());
+ }
+
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
cleanPersistenceDir();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/encryption/MasterKeyChangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/encryption/MasterKeyChangeTest.java
index 81d489f..91dcd05 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/encryption/MasterKeyChangeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/encryption/MasterKeyChangeTest.java
@@ -42,7 +42,7 @@ import org.junit.Test;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_MASTER_KEY_NAME_TO_CHANGE_BEFORE_STARTUP;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
-import static org.apache.ignite.internal.managers.encryption.GridEncryptionManager.ENCRYPTION_KEY_PREFIX;
+import static org.apache.ignite.internal.managers.encryption.GridEncryptionManager.ENCRYPTION_KEYS_PREFIX;
import static org.apache.ignite.internal.managers.encryption.GridEncryptionManager.MASTER_KEY_NAME_PREFIX;
import static org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionSpi.DEFAULT_MASTER_KEY_NAME;
import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
@@ -59,6 +59,7 @@ public class MasterKeyChangeTest extends AbstractEncryptionTest {
IgniteConfiguration cfg = super.getConfiguration(name);
cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+ cfg.setConsistentId(name);
return cfg;
}
@@ -315,14 +316,14 @@ public class MasterKeyChangeTest extends AbstractEncryptionTest {
DynamicCacheDescriptor desc = grid0.context().cache().cacheDescriptor(cacheName());
- Serializable oldKey = metaStorage.read(ENCRYPTION_KEY_PREFIX + desc.groupId());
+ Serializable oldKey = metaStorage.read(ENCRYPTION_KEYS_PREFIX + desc.groupId());
assertNotNull(oldKey);
dbMgr.checkpointReadLock();
// 6. Simulate group key write error to MetaStore for one node to check recovery from WAL.
- metaStorage.write(ENCRYPTION_KEY_PREFIX + desc.groupId(), new byte[0]);
+ metaStorage.write(ENCRYPTION_KEYS_PREFIX + desc.groupId(), new byte[0]);
dbMgr.checkpointReadUnlock();
@@ -468,7 +469,7 @@ public class MasterKeyChangeTest extends AbstractEncryptionTest {
}
/** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
+ @Override protected void beforeTest() throws Exception {
stopAllGrids();
cleanPersistenceDir();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index b40d6c5..3dedd8f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -165,6 +165,11 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
}
/** {@inheritDoc} */
+ @Override public long currentSegment() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
@Override public int walArchiveSegments() {
return 0;
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/BasicRateLimiterTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/BasicRateLimiterTest.java
new file mode 100644
index 0000000..c136cb8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/BasicRateLimiterTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Rate limiter tests.
+ */
+public class BasicRateLimiterTest {
+ /**
+ * Check change speed at runtime.
+ */
+ @Test
+ public void checkSpeedLimitChange() throws IgniteInterruptedCheckedException {
+ BasicRateLimiter limiter = new BasicRateLimiter(2);
+
+ checkRate(limiter, 10);
+
+ limiter.setRate(3);
+
+ checkRate(limiter, 15);
+
+ limiter.setRate(0.5);
+
+ checkRate(limiter, 5);
+ }
+
+ /**
+ * Check the average rate of the limiter.
+ *
+ * @param limiter Rate limiter.
+ * @param totalOps Number of operations.
+ */
+ private void checkRate(BasicRateLimiter limiter, int totalOps) throws IgniteInterruptedCheckedException {
+ double permitsPerSec = limiter.getRate();
+ long startTime = System.currentTimeMillis();
+
+ for (int i = 0; i < totalOps; i++)
+ limiter.acquire(1);
+
+ long timeSpent = System.currentTimeMillis() - startTime;
+
+ // Rate limiter aims for an average rate of permits per second.
+ assertEquals(1, Math.round((double)timeSpent / 1000 / totalOps * permitsPerSec));
+ }
+
+ /**
+ * Check rate limit with multiple threads.
+ */
+ @Test
+ public void checkLimitMultithreaded() throws Exception {
+ int permitsPerSec = 1_000;
+ int totalOps = 10_000;
+
+ BasicRateLimiter limiter = new BasicRateLimiter(permitsPerSec);
+
+ int threads = Runtime.getRuntime().availableProcessors();
+
+ CyclicBarrier ready = new CyclicBarrier(threads + 1);
+
+ AtomicInteger cntr = new AtomicInteger();
+
+ IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(() -> {
+ ready.await();
+
+ do {
+ limiter.acquire(1);
+ }
+ while (!Thread.currentThread().isInterrupted() && cntr.incrementAndGet() < totalOps);
+
+ return null;
+ }, threads, "worker");
+
+ ready.await();
+
+ long startTime = System.currentTimeMillis();
+
+ fut.get();
+
+ long timeSpent = System.currentTimeMillis() - startTime;
+
+ // Rate limiter aims for an average rate of permits per second.
+ assertEquals(1, Math.round((double)timeSpent / 1000 / totalOps * permitsPerSec));
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 7e557a1e..8a35caa 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -107,6 +107,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
import org.apache.ignite.internal.processors.service.IgniteServiceProcessor;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.G;
@@ -2559,6 +2560,8 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
* @throws IgniteCheckedException If failed.
*/
protected void enableCheckpoints(Collection<Ignite> nodes, boolean enable) throws IgniteCheckedException {
+ GridCompoundFuture<Void, Void> fut = new GridCompoundFuture<>();
+
for (Ignite node : nodes) {
assert !node.cluster().localNode().isClient();
@@ -2569,8 +2572,12 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
GridCacheDatabaseSharedManager dbMgr0 = (GridCacheDatabaseSharedManager) dbMgr;
- dbMgr0.enableCheckpoints(enable).get();
+ fut.add(dbMgr0.enableCheckpoints(enable));
}
+
+ fut.markInitialized();
+
+ fut.get();
}
/**
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/wal/record/RecordUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/wal/record/RecordUtils.java
index f5da43d..27aa08a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/wal/record/RecordUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/wal/record/RecordUtils.java
@@ -26,12 +26,13 @@ import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
import org.apache.ignite.internal.pagemem.wal.record.ExchangeRecord;
-import org.apache.ignite.internal.pagemem.wal.record.MasterKeyChangeRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MasterKeyChangeRecordV2;
import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.MvccDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.MvccTxRecord;
import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.pagemem.wal.record.ReencryptionStartRecord;
import org.apache.ignite.internal.pagemem.wal.record.RollbackRecord;
import org.apache.ignite.internal.pagemem.wal.record.SnapshotRecord;
import org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord;
@@ -55,12 +56,14 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageCutRootRecord
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootInlineRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateIndexDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastAllocatedIndex;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastSuccessfulFullSnapshotId;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastSuccessfulSnapshotId;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateNextSnapshotId;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecordV2;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecordV3;
import org.apache.ignite.internal.pagemem.wal.record.delta.NewRootInitRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.PageListMetaResetCountRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListAddPageRecord;
@@ -110,11 +113,15 @@ import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_PAGE_UPDATE_RECORD;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_RECORD;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.ENCRYPTED_DATA_RECORD;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.ENCRYPTED_DATA_RECORD_V2;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.ENCRYPTED_RECORD;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.ENCRYPTED_RECORD_V2;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.EXCHANGE;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.HEADER_RECORD;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.INDEX_META_PAGE_DELTA_RECORD;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.INIT_NEW_PAGE_RECORD;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MASTER_KEY_CHANGE_RECORD;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MASTER_KEY_CHANGE_RECORD_V2;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MEMORY_RECOVERY;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.METASTORE_DATA_RECORD;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.META_PAGE_INIT;
@@ -136,9 +143,11 @@ import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.PAGE_LIST_META_RESET_COUNT_RECORD;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.PAGE_RECORD;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.PARTITION_DESTROY;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.PARTITION_META_PAGE_DELTA_RECORD_V3;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.PARTITION_META_PAGE_UPDATE_COUNTERS;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.PARTITION_META_PAGE_UPDATE_COUNTERS_V2;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.PART_META_UPDATE_STATE;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.REENCRYPTION_START_RECORD;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.RESERVED;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.ROLLBACK_TX_RECORD;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.ROTATED_ID_PART_RECORD;
@@ -189,6 +198,7 @@ public class RecordUtils {
put(PAGES_LIST_ADD_PAGE, RecordUtils::buildPagesListAddPageRecord);
put(PAGES_LIST_REMOVE_PAGE, RecordUtils::buildPagesListRemovePageRecord);
put(META_PAGE_INIT, RecordUtils::buildMetaPageInitRecord);
+ put(INDEX_META_PAGE_DELTA_RECORD, RecordUtils::buildMetaPageIndexDeltaRecord);
put(PARTITION_META_PAGE_UPDATE_COUNTERS, RecordUtils::buildMetaPageUpdatePartitionDataRecord);
put(MEMORY_RECOVERY, RecordUtils::buildMemoryRecoveryRecord);
put(TRACKING_PAGE_DELTA, RecordUtils::buildTrackingPageDeltaRecord);
@@ -209,13 +219,18 @@ public class RecordUtils {
put(RESERVED, RecordUtils::buildReservedRecord);
put(ROLLBACK_TX_RECORD, RecordUtils::buildRollbackRecord);
put(PARTITION_META_PAGE_UPDATE_COUNTERS_V2, RecordUtils::buildMetaPageUpdatePartitionDataRecordV2);
+ put(PARTITION_META_PAGE_DELTA_RECORD_V3, RecordUtils::buildMetaPageUpdatePartitionDataRecordV3);
put(MASTER_KEY_CHANGE_RECORD, RecordUtils::buildMasterKeyChangeRecord);
+ put(MASTER_KEY_CHANGE_RECORD_V2, RecordUtils::buildMasterKeyChangeRecordV2);
+ put(REENCRYPTION_START_RECORD, RecordUtils::buildEncryptionStatusRecord);
put(ROTATED_ID_PART_RECORD, RecordUtils::buildRotatedIdPartRecord);
put(MVCC_DATA_PAGE_MARK_UPDATED_RECORD, RecordUtils::buildDataPageMvccMarkUpdatedRecord);
put(MVCC_DATA_PAGE_TX_STATE_HINT_UPDATED_RECORD, RecordUtils::buildDataPageMvccUpdateTxStateHintRecord);
put(MVCC_DATA_PAGE_NEW_TX_STATE_HINT_UPDATED_RECORD, RecordUtils::buildDataPageMvccUpdateNewTxStateHintRecord);
put(ENCRYPTED_RECORD, RecordUtils::buildEncryptedRecord);
put(ENCRYPTED_DATA_RECORD, RecordUtils::buildEncryptedDataRecord);
+ put(ENCRYPTED_RECORD_V2, RecordUtils::buildEncryptedRecordV2);
+ put(ENCRYPTED_DATA_RECORD_V2, RecordUtils::buildEncryptedDataRecordV2);
put(MVCC_DATA_RECORD, RecordUtils::buildMvccDataRecord);
put(MVCC_TX_RECORD, RecordUtils::buildMvccTxRecord);
put(CONSISTENT_CUT, RecordUtils::buildConsistentCutRecord);
@@ -410,6 +425,11 @@ public class RecordUtils {
}
/** **/
+ public static MetaPageUpdateIndexDataRecord buildMetaPageIndexDeltaRecord() {
+ return new MetaPageUpdateIndexDataRecord(1, 1, 0, 0);
+ }
+
+ /** **/
public static MetaPageUpdatePartitionDataRecord buildMetaPageUpdatePartitionDataRecord() {
return new MetaPageUpdatePartitionDataRecord(1, 1, 1, 1, 1, 1, (byte)1, 1);
}
@@ -514,8 +534,23 @@ public class RecordUtils {
}
/** **/
- public static MasterKeyChangeRecord buildMasterKeyChangeRecord() {
- return new MasterKeyChangeRecord("", new HashMap<>());
+ public static MetaPageUpdatePartitionDataRecordV3 buildMetaPageUpdatePartitionDataRecordV3() {
+ return new MetaPageUpdatePartitionDataRecordV3(1, 1, 1, 1, 1, 1, (byte)1, 1, 1, 0, 0);
+ }
+
+ /** **/
+ public static UnsupportedWalRecord buildMasterKeyChangeRecord() {
+ return new UnsupportedWalRecord(MASTER_KEY_CHANGE_RECORD);
+ }
+
+ /** **/
+ public static MasterKeyChangeRecordV2 buildMasterKeyChangeRecordV2() {
+ return new MasterKeyChangeRecordV2("", Collections.emptyList());
+ }
+
+ /** **/
+ public static ReencryptionStartRecord buildEncryptionStatusRecord() {
+ return new ReencryptionStartRecord(Collections.emptyMap());
}
/** **/
@@ -549,6 +584,16 @@ public class RecordUtils {
}
/** **/
+ public static UnsupportedWalRecord buildEncryptedRecordV2() {
+ return new UnsupportedWalRecord(ENCRYPTED_RECORD_V2);
+ }
+
+ /** **/
+ public static UnsupportedWalRecord buildEncryptedDataRecordV2() {
+ return new UnsupportedWalRecord(ENCRYPTED_DATA_RECORD_V2);
+ }
+
+ /** **/
public static MvccDataRecord buildMvccDataRecord() {
return new MvccDataRecord(Collections.emptyList(), 1);
}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
index 210b1c7..6addefe 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
@@ -24,6 +24,8 @@ import org.apache.ignite.failure.SystemWorkersTerminationTest;
import org.apache.ignite.internal.ClusterBaselineNodesMetricsSelfTest;
import org.apache.ignite.internal.GridNodeMetricsLogPdsSelfTest;
import org.apache.ignite.internal.cluster.IgniteClusterIdTagTest;
+import org.apache.ignite.internal.encryption.CacheGroupKeyChangeTest;
+import org.apache.ignite.internal.encryption.CacheGroupReencryptionTest;
import org.apache.ignite.internal.encryption.EncryptedCacheBigEntryTest;
import org.apache.ignite.internal.encryption.EncryptedCacheCreateTest;
import org.apache.ignite.internal.encryption.EncryptedCacheDestroyTest;
@@ -74,6 +76,9 @@ import org.junit.runners.Suite;
MasterKeyChangeTest.class,
MasterKeyChangeConsistencyCheckTest.class,
+ CacheGroupKeyChangeTest.class,
+ CacheGroupReencryptionTest.class,
+
EncryptionMXBeanTest.class,
IgniteSnapshotManagerSelfTest.class,
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index e45ac07..1e110b1 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -20,6 +20,7 @@ package org.apache.ignite.testsuites;
import org.apache.ignite.internal.IgniteVersionUtilsSelfTest;
import org.apache.ignite.internal.pagemem.impl.PageIdUtilsSelfTest;
import org.apache.ignite.internal.processors.cache.GridCacheUtilsSelfTest;
+import org.apache.ignite.internal.util.BasicRateLimiterTest;
import org.apache.ignite.internal.util.DistributedProcessCoordinatorLeftTest;
import org.apache.ignite.internal.util.GridArraysSelfTest;
import org.apache.ignite.internal.util.GridConcurrentMultiPairQueueTest;
@@ -137,7 +138,9 @@ import org.junit.runners.Suite;
GridCountDownCallbackTest.class,
- DistributedProcessCoordinatorLeftTest.class
+ DistributedProcessCoordinatorLeftTest.class,
+
+ BasicRateLimiterTest.class
})
public class IgniteUtilSelfTestSuite {
}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
index c4b8d0f..1074af3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
@@ -17,6 +17,7 @@
package org.apache.ignite.testsuites;
+import org.apache.ignite.internal.encryption.CacheGroupReencryptionTest;
import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest;
import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest;
import org.apache.ignite.internal.processors.cache.index.ClientReconnectWithSqlTableConfiguredTest;
@@ -54,7 +55,8 @@ import org.junit.runners.Suite;
RebuildIndexTest.class,
IgniteClusterSnapshotWithIndexesTest.class,
ClientReconnectWithSqlTableConfiguredTest.class,
- MultipleParallelCacheDeleteDeadlockTest.class
+ MultipleParallelCacheDeleteDeadlockTest.class,
+ CacheGroupReencryptionTest.class
})
public class IgnitePdsWithIndexingTestSuite {
}
diff --git a/modules/spring/src/test/java/org/apache/ignite/encryption/SpringEncryptedCacheRestartTest.java b/modules/spring/src/test/java/org/apache/ignite/encryption/SpringEncryptedCacheRestartTest.java
index bad97a5..8866ab6 100644
--- a/modules/spring/src/test/java/org/apache/ignite/encryption/SpringEncryptedCacheRestartTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/encryption/SpringEncryptedCacheRestartTest.java
@@ -24,6 +24,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgnitionEx;
import org.apache.ignite.internal.encryption.EncryptedCacheRestartTest;
+import org.apache.ignite.internal.managers.encryption.GroupKey;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.typedef.T2;
@@ -81,13 +82,21 @@ public class SpringEncryptedCacheRestartTest extends EncryptedCacheRestartTest {
int grpId = CU.cacheGroupId(enc.name(), enc.configuration().getGroupName());
- KeystoreEncryptionKey key0 = (KeystoreEncryptionKey)g.get1().context().encryption().groupKey(grpId);
- KeystoreEncryptionKey key1 = (KeystoreEncryptionKey)g.get2().context().encryption().groupKey(grpId);
- KeystoreEncryptionKey key2 = (KeystoreEncryptionKey)g2.context().encryption().groupKey(grpId);
+ GroupKey grpKey0 = g.get1().context().encryption().groupKey(grpId);
+ GroupKey grpKey1 = g.get2().context().encryption().groupKey(grpId);
+ GroupKey grpKey2 = g2.context().encryption().groupKey(grpId);
- assertNotNull(cacheName, key0);
- assertNotNull(cacheName, key1);
- assertNotNull(cacheName, key2);
+ assertNotNull(cacheName, grpKey0);
+ assertNotNull(cacheName, grpKey1);
+ assertNotNull(cacheName, grpKey2);
+
+ KeystoreEncryptionKey key0 = (KeystoreEncryptionKey)grpKey0.key();
+ KeystoreEncryptionKey key1 = (KeystoreEncryptionKey)grpKey1.key();
+ KeystoreEncryptionKey key2 = (KeystoreEncryptionKey)grpKey2.key();
+
+ assertNotNull(cacheName, key0.key());
+ assertNotNull(cacheName, key1.key());
+ assertNotNull(cacheName, key2.key());
assertNotNull(cacheName, key0.key());
assertNotNull(cacheName, key1.key());
@@ -121,15 +130,23 @@ public class SpringEncryptedCacheRestartTest extends EncryptedCacheRestartTest {
assertNotNull(encrypted2);
- KeystoreEncryptionKey key = (KeystoreEncryptionKey)g0.context().encryption().groupKey(
+ GroupKey grpKey = g0.context().encryption().groupKey(
CU.cacheGroupId(encrypted.name(), encrypted.configuration().getGroupName()));
+ assertNotNull(grpKey);
+
+ KeystoreEncryptionKey key = (KeystoreEncryptionKey)grpKey.key();
+
assertNotNull(key);
assertNotNull(key.key());
- KeystoreEncryptionKey key2 = (KeystoreEncryptionKey)g0.context().encryption().groupKey(
+ GroupKey grpKey2 = g0.context().encryption().groupKey(
CU.cacheGroupId(encrypted2.name(), encrypted2.configuration().getGroupName()));
+ assertNotNull(grpKey2);
+
+ KeystoreEncryptionKey key2 = (KeystoreEncryptionKey)grpKey2.key();
+
assertNotNull(key2);
assertNotNull(key2.key());