You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/08/09 16:54:30 UTC
hadoop git commit: HADOOP-14735. ITestS3AEncryptionSSEC failing in
parallel s3guard runs. Contributed by Steve Loughran.
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13345 35e4bcd43 -> b4c2ab278
HADOOP-14735. ITestS3AEncryptionSSEC failing in parallel s3guard runs.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4c2ab27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4c2ab27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4c2ab27
Branch: refs/heads/HADOOP-13345
Commit: b4c2ab278a4280192643469f900fa689b7b53d84
Parents: 35e4bcd
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 9 17:53:42 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 9 17:53:42 2017 +0100
----------------------------------------------------------------------
.../hadoop/fs/s3a/AbstractS3ATestBase.java | 13 +-
.../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 313 +++++++++----------
2 files changed, 156 insertions(+), 170 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c2ab27/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
index ca928f6..f0c389d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
@@ -112,10 +112,21 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase
*/
protected Path writeThenReadFile(String name, int len) throws IOException {
Path path = path(name);
+ writeThenReadFile(path, len);
+ return path;
+ }
+
+ /**
+ * Write a file, read it back, validate the dataset. Overwrites the file
+ * if it is present
+ * @param path path to file
+ * @param len length of file
+ * @throws IOException any IO problem
+ */
+ protected void writeThenReadFile(Path path, int len) throws IOException {
byte[] data = dataset(len, 'a', 'z');
writeDataset(getFileSystem(), path, data, data.length, 1024 * 1024, true);
ContractTestUtils.verifyFileContents(getFileSystem(), path, data);
- return path;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c2ab27/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
index 51c8c7b..8b7e031 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
@@ -18,20 +18,21 @@
package org.apache.hadoop.fs.s3a;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.rm;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeS3GuardState;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionTestsDisabled;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-
import java.io.IOException;
+import java.nio.file.AccessDeniedException;
+
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.junit.Test;
+import org.apache.hadoop.io.IOUtils;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Concrete class that extends {@link AbstractTestS3AEncryption}
@@ -39,17 +40,39 @@ import org.junit.Test;
*/
public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
+ private static final String SERVICE_AMAZON_S3_STATUS_CODE_403
+ = "Service: Amazon S3; Status Code: 403;";
+ private static final String KEY_1
+ = "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs=";
+ private static final String KEY_2
+ = "G61nz31Q7+zpjJWbakxfTOZW4VS0UmQWAq2YXhcTXoo=";
+ private static final String KEY_3
+ = "NTx0dUPrxoo9+LbNiT/gqf3z9jILqL6ilismFmJO50U=";
+ private static final String KEY_4
+ = "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8=";
+ private static final int TEST_FILE_LEN = 2048;
+
+ /**
+ * Filesystem created with a different key.
+ */
+ private FileSystem fsKeyB;
+
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
- S3ATestUtils.disableFilesystemCaching(conf);
+ disableFilesystemCaching(conf);
conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
getSSEAlgorithm().getMethod());
- conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY,
- "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs=");
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, KEY_1);
return conf;
}
+ @Override
+ public void teardown() throws Exception {
+ super.teardown();
+ IOUtils.closeStream(fsKeyB);
+ }
+
/**
* This will create and write to a file using encryption key A, then attempt
* to read from it again with encryption key B. This will not work as it
@@ -65,26 +88,25 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
assumeEnabled();
skipIfEncryptionTestsDisabled(getConfiguration());
- final Path[] path = new Path[1];
- intercept(java.nio.file.AccessDeniedException.class,
- "Service: Amazon S3; Status Code: 403;", () -> {
-
- int len = 2048;
- describe("Create an encrypted file of size " + len);
- String src = createFilename(len);
- path[0] = writeThenReadFile(src, len);
-
- //extract the test FS
- FileSystem fileSystem = createNewFileSystemWithSSECKey(
- "kX7SdwVc/1VXJr76kfKnkQ3ONYhxianyL2+C3rPVT9s=");
- byte[] data = dataset(len, 'a', 'z');
- ContractTestUtils.verifyFileContents(fileSystem, path[0], data);
- throw new Exception("Fail");
- });
+ intercept(AccessDeniedException.class,
+ SERVICE_AMAZON_S3_STATUS_CODE_403,
+ () -> {
+ int len = TEST_FILE_LEN;
+ describe("Create an encrypted file of size " + len);
+ Path src = path("testCreateFileAndReadWithDifferentEncryptionKey");
+ writeThenReadFile(src, len);
+
+ //extract the test FS
+ fsKeyB = createNewFileSystemWithSSECKey(
+ "kX7SdwVc/1VXJr76kfKnkQ3ONYhxianyL2+C3rPVT9s=");
+ byte[] data = dataset(len, 'a', 'z');
+ ContractTestUtils.verifyFileContents(fsKeyB, src, data);
+ return fsKeyB.getFileStatus(src);
+ });
}
/**
- * While each object has it's own key and should be distinct, this verifies
+ * While each object has its own key and should be distinct, this verifies
* that hadoop treats object keys as a filesystem path. So if a top level
* dir is encrypted with keyA, a sublevel dir cannot be accessed with a
* different keyB.
@@ -99,24 +121,18 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
skipIfEncryptionTestsDisabled(getConfiguration());
assumeS3GuardState(false, getConfiguration());
- final Path[] path = new Path[1];
- intercept(java.nio.file.AccessDeniedException.class,
- "Service: Amazon S3; Status Code: 403;", () -> {
-
- path[0] = S3ATestUtils.createTestPath(
- new Path(createFilename("dir/"))
- );
- Path nestedDirectory = S3ATestUtils.createTestPath(
- new Path(createFilename("dir/nestedDir/"))
- );
- FileSystem fsKeyB = createNewFileSystemWithSSECKey(
- "G61nz31Q7+zpjJWbakxfTOZW4VS0UmQWAq2YXhcTXoo=");
- getFileSystem().mkdirs(path[0]);
- fsKeyB.mkdirs(nestedDirectory);
-
- throw new Exception("Exception should be thrown.");
- });
- rm(getFileSystem(), path[0], true, false);
+ intercept(AccessDeniedException.class,
+ SERVICE_AMAZON_S3_STATUS_CODE_403,
+ () -> {
+ Path base = path("testCreateSubdirWithDifferentKey");
+ Path nestedDirectory = new Path(base, "nestedDir");
+ fsKeyB = createNewFileSystemWithSSECKey(
+ KEY_2);
+ getFileSystem().mkdirs(base);
+ fsKeyB.mkdirs(nestedDirectory);
+ // expected to fail
+ return fsKeyB.getFileStatus(nestedDirectory);
+ });
}
/**
@@ -132,20 +148,17 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
assumeEnabled();
skipIfEncryptionTestsDisabled(getConfiguration());
- final Path[] path = new Path[1];
- intercept(java.nio.file.AccessDeniedException.class,
- "Service: Amazon S3; Status Code: 403;", () -> {
-
- int len = 2048;
- String src = createFilename(len);
- path[0] = writeThenReadFile(src, len);
-
- FileSystem fsKeyB = createNewFileSystemWithSSECKey(
- "NTx0dUPrxoo9+LbNiT/gqf3z9jILqL6ilismFmJO50U=");
- fsKeyB.rename(path[0], new Path(createFilename("different-path.txt")));
-
- throw new Exception("Exception should be thrown.");
- });
+ intercept(AccessDeniedException.class,
+ SERVICE_AMAZON_S3_STATUS_CODE_403,
+ () -> {
+ int len = TEST_FILE_LEN;
+ Path src = path(createFilename(len));
+ writeThenReadFile(src, len);
+ fsKeyB = createNewFileSystemWithSSECKey(KEY_3);
+ Path dest = path(createFilename("different-path.txt"));
+ getFileSystem().mkdirs(dest.getParent());
+ return fsKeyB.rename(src, dest);
+ });
}
/**
@@ -159,11 +172,11 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
assumeEnabled();
skipIfEncryptionTestsDisabled(getConfiguration());
- String src = createFilename("original-path.txt");
- Path path = writeThenReadFile(src, 2048);
- Path newPath = path(createFilename("different-path.txt"));
- getFileSystem().rename(path, newPath);
- byte[] data = dataset(2048, 'a', 'z');
+ Path src = path("original-path.txt");
+ writeThenReadFile(src, TEST_FILE_LEN);
+ Path newPath = path("different-path.txt");
+ getFileSystem().rename(src, newPath);
+ byte[] data = dataset(TEST_FILE_LEN, 'a', 'z');
ContractTestUtils.verifyFileContents(getFileSystem(), newPath, data);
}
@@ -179,29 +192,24 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
skipIfEncryptionTestsDisabled(getConfiguration());
assumeS3GuardState(false, getConfiguration());
- Path nestedDirectory = S3ATestUtils.createTestPath(
- path(createFilename("/a/b/c/"))
- );
+ Path pathABC = path("testListEncryptedDir/a/b/c/");
+ Path pathAB = pathABC.getParent();
+ Path pathA = pathAB.getParent();
+
+ Path nestedDirectory = createTestPath(pathABC);
assertTrue(getFileSystem().mkdirs(nestedDirectory));
- FileSystem fsKeyB = createNewFileSystemWithSSECKey(
- "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8=");
+ fsKeyB = createNewFileSystemWithSSECKey(KEY_4);
- fsKeyB.listFiles(S3ATestUtils.createTestPath(
- path(createFilename("/a/"))
- ), true);
- fsKeyB.listFiles(S3ATestUtils.createTestPath(
- path(createFilename("/a/b/"))
- ), true);
+ fsKeyB.listFiles(pathA, true);
+ fsKeyB.listFiles(pathAB, true);
//Until this point, no exception is thrown about access
- intercept(java.nio.file.AccessDeniedException.class,
- "Service: Amazon S3; Status Code: 403;", () -> {
- fsKeyB.listFiles(S3ATestUtils.createTestPath(
- path(createFilename("/a/b/c/"))
- ), false);
- throw new Exception("Exception should be thrown.");
- });
+ intercept(AccessDeniedException.class,
+ SERVICE_AMAZON_S3_STATUS_CODE_403,
+ () -> {
+ fsKeyB.listFiles(pathABC, false);
+ });
Configuration conf = this.createConfiguration();
conf.unset(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM);
@@ -212,22 +220,13 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
FileSystem unencryptedFileSystem = contract.getTestFileSystem();
//unencrypted can access until the final directory
- unencryptedFileSystem.listFiles(S3ATestUtils.createTestPath(
- path(createFilename("/a/"))
- ), true);
- unencryptedFileSystem.listFiles(S3ATestUtils.createTestPath(
- path(createFilename("/a/b/"))
- ), true);
- intercept(org.apache.hadoop.fs.s3a.AWSS3IOException.class,
- "Bad Request (Service: Amazon S3; Status Code: 400; Error" +
- " Code: 400 Bad Request;", () -> {
-
- unencryptedFileSystem.listFiles(S3ATestUtils.createTestPath(
- path(createFilename("/a/b/c/"))
- ), false);
- throw new Exception("Exception should be thrown.");
- });
- rm(getFileSystem(), path(createFilename("/")), true, false);
+ unencryptedFileSystem.listFiles(pathA, true);
+ unencryptedFileSystem.listFiles(pathAB, true);
+ AWSS3IOException ex = intercept(AWSS3IOException.class,
+ () -> {
+ unencryptedFileSystem.listFiles(pathABC, false);
+ });
+ assertStatusCode(ex, 400);
}
/**
@@ -241,30 +240,25 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
skipIfEncryptionTestsDisabled(getConfiguration());
assumeS3GuardState(false, getConfiguration());
- Path nestedDirectory = S3ATestUtils.createTestPath(
- path(createFilename("/a/b/c/"))
- );
- assertTrue(getFileSystem().mkdirs(nestedDirectory));
+ Path pathABC = path("testListStatusEncryptedDir/a/b/c/");
+ Path pathAB = pathABC.getParent();
+ Path pathA = pathAB.getParent();
+ assertTrue(getFileSystem().mkdirs(pathABC));
- FileSystem fsKeyB = createNewFileSystemWithSSECKey(
- "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8=");
+ fsKeyB = createNewFileSystemWithSSECKey(KEY_4);
- fsKeyB.listStatus(S3ATestUtils.createTestPath(
- path(createFilename("/a/"))));
- fsKeyB.listStatus(S3ATestUtils.createTestPath(
- path(createFilename("/a/b/"))));
+ fsKeyB.listStatus(pathA);
+ fsKeyB.listStatus(pathAB);
//Until this point, no exception is thrown about access
- intercept(java.nio.file.AccessDeniedException.class,
- "Service: Amazon S3; Status Code: 403;", () -> {
- fsKeyB.listStatus(S3ATestUtils.createTestPath(
- path(createFilename("/a/b/c/"))));
-
- throw new Exception("Exception should be thrown.");
+ intercept(AccessDeniedException.class,
+ SERVICE_AMAZON_S3_STATUS_CODE_403,
+ () -> {
+ fsKeyB.listStatus(pathABC);
});
//Now try it with an unencrypted filesystem.
- Configuration conf = this.createConfiguration();
+ Configuration conf = createConfiguration();
conf.unset(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM);
conf.unset(Constants.SERVER_SIDE_ENCRYPTION_KEY);
@@ -273,20 +267,14 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
FileSystem unencryptedFileSystem = contract.getTestFileSystem();
//unencrypted can access until the final directory
- unencryptedFileSystem.listStatus(S3ATestUtils.createTestPath(
- path(createFilename("/a/"))));
- unencryptedFileSystem.listStatus(S3ATestUtils.createTestPath(
- path(createFilename("/a/b/"))));
-
- intercept(org.apache.hadoop.fs.s3a.AWSS3IOException.class,
- "Bad Request (Service: Amazon S3; Status Code: 400; Error Code: 400" +
- " Bad Request;", () -> {
-
- unencryptedFileSystem.listStatus(S3ATestUtils.createTestPath(
- path(createFilename("/a/b/c/"))));
- throw new Exception("Exception should be thrown.");
+ unencryptedFileSystem.listStatus(pathA);
+ unencryptedFileSystem.listStatus(pathAB);
+
+ AWSS3IOException ex = intercept(AWSS3IOException.class,
+ () -> {
+ unencryptedFileSystem.listStatus(pathABC);
});
- rm(getFileSystem(), path(createFilename("/")), true, false);
+ assertStatusCode(ex, 400);
}
/**
@@ -299,31 +287,23 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
assumeEnabled();
skipIfEncryptionTestsDisabled(getConfiguration());
assumeS3GuardState(false, getConfiguration());
+ Path pathABC = path("testListStatusEncryptedFile/a/b/c/");
+ assertTrue(getFileSystem().mkdirs(pathABC));
- Path nestedDirectory = S3ATestUtils.createTestPath(
- path(createFilename("/a/b/c/"))
- );
- assertTrue(getFileSystem().mkdirs(nestedDirectory));
-
- String src = createFilename("/a/b/c/fileToStat.txt");
- Path fileToStat = writeThenReadFile(src, 2048);
+ Path fileToStat = new Path(pathABC, "fileToStat.txt");
+ writeThenReadFile(fileToStat, TEST_FILE_LEN);
- FileSystem fsKeyB = createNewFileSystemWithSSECKey(
- "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8=");
+ fsKeyB = createNewFileSystemWithSSECKey(KEY_4);
//Until this point, no exception is thrown about access
- intercept(java.nio.file.AccessDeniedException.class,
- "Service: Amazon S3; Status Code: 403;", () -> {
- fsKeyB.listStatus(S3ATestUtils.createTestPath(fileToStat));
-
- throw new Exception("Exception should be thrown.");
- });
- rm(getFileSystem(), path(createFilename("/")), true, false);
+ intercept(AccessDeniedException.class,
+ SERVICE_AMAZON_S3_STATUS_CODE_403,
+ () -> {
+ fsKeyB.listStatus(fileToStat);
+ });
}
-
-
/**
* It is possible to delete directories without the proper encryption key and
* the hierarchy above it.
@@ -335,31 +315,25 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
assumeEnabled();
skipIfEncryptionTestsDisabled(getConfiguration());
assumeS3GuardState(false, getConfiguration());
-
- Path nestedDirectory = S3ATestUtils.createTestPath(
- path(createFilename("/a/b/c/"))
- );
- assertTrue(getFileSystem().mkdirs(nestedDirectory));
- String src = createFilename("/a/b/c/filetobedeleted.txt");
- Path fileToDelete = writeThenReadFile(src, 2048);
-
- FileSystem fsKeyB = createNewFileSystemWithSSECKey(
- "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8=");
- intercept(java.nio.file.AccessDeniedException.class,
- "Forbidden (Service: Amazon S3; Status Code: 403; Error Code: " +
- "403 Forbidden", () -> {
-
- fsKeyB.delete(fileToDelete, false);
- throw new Exception("Exception should be thrown.");
- });
+ Path pathABC = path("testDeleteEncryptedObjectWithDifferentKey/a/b/c/");
+
+ Path pathAB = pathABC.getParent();
+ Path pathA = pathAB.getParent();
+ assertTrue(getFileSystem().mkdirs(pathABC));
+ Path fileToDelete = new Path(pathABC, "filetobedeleted.txt");
+ writeThenReadFile(fileToDelete, TEST_FILE_LEN);
+ fsKeyB = createNewFileSystemWithSSECKey(KEY_4);
+ intercept(AccessDeniedException.class,
+ SERVICE_AMAZON_S3_STATUS_CODE_403,
+ () -> {
+ fsKeyB.delete(fileToDelete, false);
+ });
//This is possible
- fsKeyB.delete(S3ATestUtils.createTestPath(
- path(createFilename("/a/b/c/"))), true);
- fsKeyB.delete(S3ATestUtils.createTestPath(
- path(createFilename("/a/b/"))), true);
- fsKeyB.delete(S3ATestUtils.createTestPath(
- path(createFilename("/a/"))), true);
+ fsKeyB.delete(pathABC, true);
+ fsKeyB.delete(pathAB, true);
+ fsKeyB.delete(pathA, true);
+ assertPathDoesNotExist("expected recursive delete", fileToDelete);
}
private FileSystem createNewFileSystemWithSSECKey(String sseCKey) throws
@@ -377,4 +351,5 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
protected S3AEncryptionMethods getSSEAlgorithm() {
return S3AEncryptionMethods.SSE_C;
}
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org