You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/09/11 06:48:56 UTC
[31/50] [abbrv] hadoop git commit: HDFS-12400. Provide a way for NN
to drain the local key cache before re-encryption.
HDFS-12400. Provide a way for NN to drain the local key cache before re-encryption.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3a4d7d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3a4d7d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3a4d7d2
Branch: refs/heads/YARN-5972
Commit: b3a4d7d2a01051e166c06ee78e8c6e8df1341948
Parents: fa61375
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Sep 7 20:50:03 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Sep 7 20:51:02 2017 -0700
----------------------------------------------------------------------
.../crypto/key/KeyProviderCryptoExtension.java | 10 ++
.../server/namenode/FSDirEncryptionZoneOp.java | 5 +-
.../hdfs/server/namenode/FSNamesystem.java | 2 +
.../hdfs/server/namenode/TestReencryption.java | 103 +++++++------------
.../namenode/TestReencryptionWithKMS.java | 5 +
5 files changed, 56 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3a4d7d2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index 693c785..3ee3bd7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -555,6 +555,16 @@ public class KeyProviderCryptoExtension extends
}
/**
+ * Calls {@link CryptoExtension#drain(String)} for the given key name on the
+ * underlying {@link CryptoExtension}.
+ *
+ * @param keyName
+ */
+ public void drain(String keyName) {
+ getExtension().drain(keyName);
+ }
+
+ /**
* Batched version of {@link #reencryptEncryptedKey(EncryptedKeyVersion)}.
* <p>
* For each encrypted key version, re-encrypts an encrypted key version,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3a4d7d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
index ee2b0f4..e284b15 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
-import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileStatus;
@@ -698,9 +697,7 @@ final class FSDirEncryptionZoneOp {
// drain the local cache of the key provider.
// Do not invalidateCache on the server, since that's the responsibility
// when rolling the key version.
- if (dir.getProvider() instanceof CryptoExtension) {
- ((CryptoExtension) dir.getProvider()).drain(keyName);
- }
+ dir.getProvider().drain(keyName);
final EncryptedKeyVersion edek;
try {
edek = dir.getProvider().generateEncryptedKey(keyName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3a4d7d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 08c8562..d9f3c0e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -7111,6 +7111,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (keyVersionName == null) {
throw new IOException("Failed to get key version name for " + zone);
}
+ LOG.info("Re-encryption using key version " + keyVersionName
+ + " for zone " + zone);
}
writeLock();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3a4d7d2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
index 5612d65..33c52bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
@@ -91,7 +91,7 @@ public class TestReencryption {
private FileSystemTestHelper fsHelper;
private MiniDFSCluster cluster;
- private HdfsAdmin dfsAdmin;
+ protected HdfsAdmin dfsAdmin;
private DistributedFileSystem fs;
private FSNamesystem fsn;
private File testRootDir;
@@ -199,8 +199,7 @@ public class TestReencryption {
verifyZoneStatus(zone, null, 0);
// test re-encrypt after keyroll
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(2);
FileEncryptionInfo fei1 = getFileEncryptionInfo(encFile1);
@@ -316,8 +315,7 @@ public class TestReencryption {
final Path notReencrypted = new Path(zone, "f0");
final FileEncryptionInfo fei = getFileEncryptionInfo(lastReencryptedFile);
final FileEncryptionInfo feiLast = getFileEncryptionInfo(notReencrypted);
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// mark pause after first checkpoint (5 files)
getEzManager().pauseForTestingAfterNthSubmission(1);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -363,8 +361,7 @@ public class TestReencryption {
0xFEED);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// test zone deleted during re-encrypt's checkpointing
getEzManager().pauseForTestingAfterNthSubmission(1);
getEzManager().resetMetricsForTesting();
@@ -409,8 +406,7 @@ public class TestReencryption {
final Path encFile9 = new Path(zone, "9");
final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
@@ -443,8 +439,7 @@ public class TestReencryption {
fsWrapper.rename(new Path(zone, "f"), new Path(zone, "f1"));
// re-encrypt
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
@@ -495,8 +490,7 @@ public class TestReencryption {
final Path encFile9 = new Path(subdir, "9");
final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// mark pause after first checkpoint (5 files)
getEzManager().pauseForTestingAfterNthSubmission(1);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -540,8 +534,7 @@ public class TestReencryption {
final Path encFile9 = new Path(zone, "9");
final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
@@ -585,8 +578,7 @@ public class TestReencryption {
final Path encFile9 = new Path(zone, "9");
final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// disable re-encrypt for testing, and issue a command
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -645,8 +637,7 @@ public class TestReencryption {
final Path encFile9 = new Path(zone, "9");
final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// disable re-encrypt for testing, and issue a command
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -770,8 +761,7 @@ public class TestReencryption {
0xFEED);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// Disable re-encrypt, send re-encrypt on '/', verify queue
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zoneRoot, ReencryptAction.START);
@@ -816,8 +806,7 @@ public class TestReencryption {
.createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -883,8 +872,7 @@ public class TestReencryption {
.createFile(fs, new Path(subdir, "file" + i), len, (short) 1, 0xFEED);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -930,8 +918,7 @@ public class TestReencryption {
.createFile(fs, new Path(subdir, "file" + i), len, (short) 1, 0xFEED);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -984,8 +971,7 @@ public class TestReencryption {
0xFEED);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -1029,8 +1015,7 @@ public class TestReencryption {
0xFEED);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -1071,8 +1056,7 @@ public class TestReencryption {
.createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -1122,8 +1106,7 @@ public class TestReencryption {
.createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -1162,8 +1145,7 @@ public class TestReencryption {
.createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -1220,8 +1202,7 @@ public class TestReencryption {
.createFile(fs, new Path(subdir, "file" + i), len, (short) 1, 0xFEED);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -1283,8 +1264,7 @@ public class TestReencryption {
// test re-encrypt on snapshot dir
final Path encFile1 = new Path(zone, "0");
final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile1);
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
try {
dfsAdmin.reencryptEncryptionZone(zoneSnap, ReencryptAction.START);
fail("Reencrypt command on snapshot path should fail.");
@@ -1423,8 +1403,7 @@ public class TestReencryption {
fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// disable, test basic
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -1442,8 +1421,7 @@ public class TestReencryption {
assertExceptionContains("not under re-encryption", expected);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// test cancelling half-way
getEzManager().pauseForTestingAfterNthSubmission(1);
getEzManager().resumeReencryptForTesting();
@@ -1537,8 +1515,7 @@ public class TestReencryption {
// re-encrypt 10 files, so 2 callables. Hang 1, pause the updater so the
// callable is taken from the executor but not processed.
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForQueuedZones(1);
@@ -1593,8 +1570,7 @@ public class TestReencryption {
fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// disable, test basic
getEzManager().pauseReencryptUpdaterForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -1625,8 +1601,7 @@ public class TestReencryption {
}
// re-encrypt the zone
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
@@ -1678,8 +1653,7 @@ public class TestReencryption {
0xFEED);
}
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
+ rollKey(TEST_KEY);
// mark pause after first checkpoint (5 files)
getEzManager().pauseForTestingAfterNthSubmission(1);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@@ -1736,9 +1710,7 @@ public class TestReencryption {
}
// re-encrypt the zone
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
-
+ rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
assertEquals(0, injector.exceptionCount);
@@ -1790,9 +1762,7 @@ public class TestReencryption {
}
// re-encrypt the zone
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
-
+ rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
assertEquals(0, injector.exceptionCount);
@@ -1845,9 +1815,7 @@ public class TestReencryption {
}
// re-encrypt the zone
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
-
+ rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
assertEquals(0, injector.exceptionCount);
@@ -1899,9 +1867,7 @@ public class TestReencryption {
}
// re-encrypt the zone
- fsn.getProvider().rollNewVersion(TEST_KEY);
- fsn.getProvider().flush();
-
+ rollKey(TEST_KEY);
Whitebox.setInternalState(getUpdater(), "faultRetryInterval", 50);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
@@ -1929,4 +1895,11 @@ public class TestReencryption {
return (ReencryptionUpdater) Whitebox
.getInternalState(getHandler(), "reencryptionUpdater");
}
+
+ protected void rollKey(final String keyName) throws Exception {
+ dfsAdmin.getKeyProvider().rollNewVersion(keyName);
+ // need to flush for jceks provider to make the key version it returned
+ // after NN restart consistent.
+ dfsAdmin.getKeyProvider().flush();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3a4d7d2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionWithKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionWithKMS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionWithKMS.java
index af9c381..642d5e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionWithKMS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionWithKMS.java
@@ -88,4 +88,9 @@ public class TestReencryptionWithKMS extends TestReencryption{
KMSWebApp.getACLs().run();
testReencryptionBasic();
}
+
+ @Override
+ protected void rollKey(final String keyName) throws Exception {
+ dfsAdmin.getKeyProvider().rollNewVersion(keyName);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org