You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2019/04/24 13:56:30 UTC
[hadoop] branch trunk updated: HADOOP-16252. Add prefix to dynamo
tables in tests.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e1c5ddf HADOOP-16252. Add prefix to dynamo tables in tests.
e1c5ddf is described below
commit e1c5ddf2aa854951142e234462978245cdb99e1d
Author: Ben Roling <be...@gmail.com>
AuthorDate: Wed Apr 24 14:55:58 2019 +0100
HADOOP-16252. Add prefix to dynamo tables in tests.
Contributed by Ben Roling.
---
.../src/site/markdown/tools/hadoop-aws/testing.md | 6 ++++++
.../apache/hadoop/fs/s3a/AbstractS3ATestBase.java | 5 +++++
.../org/apache/hadoop/fs/s3a/S3ATestConstants.java | 4 ++++
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 10 +++++++++
.../s3a/s3guard/AbstractS3GuardToolTestBase.java | 6 ++++--
.../fs/s3a/s3guard/ITestDynamoDBMetadataStore.java | 24 ++++++++++++++--------
.../fs/s3a/s3guard/ITestS3GuardConcurrentOps.java | 4 +++-
.../fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java | 11 ++++++----
8 files changed, 55 insertions(+), 15 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
index 37a5302..707694c8 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
@@ -1074,6 +1074,12 @@ If the `s3guard` profile *is* set,
property should be configured, and the name of that table should be different
than what is used for fs.s3a.s3guard.ddb.table. The test table is destroyed
and modified multiple times during the test.
+ 1. Several of the tests create and destroy DynamoDB tables. The table names
+ are prefixed with the value defined by
+ `fs.s3a.s3guard.test.dynamo.table.prefix` (default="s3guard.test."). The user
+ executing the tests will need sufficient privilege to create and destroy such
+ tables. If the tests abort uncleanly, these tables may be left behind,
+ incurring AWS charges.
### Scale Testing MetadataStore Directly
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
index 484d2dc..c1942a9 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
@@ -33,6 +33,7 @@ import java.io.IOException;
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestDynamoTablePrefix;
/**
* An extension of the contract test base set up for S3A tests.
@@ -129,6 +130,10 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase
ContractTestUtils.verifyFileContents(getFileSystem(), path, data);
}
+ protected String getTestTableName(String suffix) {
+ return getTestDynamoTablePrefix(getConfiguration()) + suffix;
+ }
+
/**
* Assert that an exception failed with a specific status code.
* @param e exception
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
index 5f28c30..9d6e1ce 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
@@ -158,6 +158,10 @@ public interface S3ATestConstants {
String TEST_S3GUARD_IMPLEMENTATION_DYNAMO = "dynamo";
String TEST_S3GUARD_IMPLEMENTATION_NONE = "none";
+ String TEST_S3GUARD_DYNAMO_TABLE_PREFIX =
+ "fs.s3a.s3guard.test.dynamo.table.prefix";
+ String TEST_S3GUARD_DYNAMO_TABLE_PREFIX_DEFAULT = "s3guard.test.";
+
/**
* Timeout in Milliseconds for standard tests: {@value}.
*/
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 5068524..260ecdd 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -709,6 +709,16 @@ public final class S3ATestUtils {
}
/**
+ * Get the prefix for DynamoDB table names used in tests.
+ * @param conf configuration to scan.
+ * @return the table name prefix
+ */
+ public static String getTestDynamoTablePrefix(final Configuration conf) {
+ return getTestProperty(conf, TEST_S3GUARD_DYNAMO_TABLE_PREFIX,
+ TEST_S3GUARD_DYNAMO_TABLE_PREFIX_DEFAULT);
+ }
+
+ /**
* Remove any values from a bucket.
* @param bucket bucket whose overrides are to be removed. Can be null/empty
* @param conf config
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index d4e4122..ad4691a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -59,6 +59,7 @@ import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CREATE_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestDynamoTablePrefix;
import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
@@ -71,7 +72,7 @@ import static org.apache.hadoop.test.LambdaTestUtils.intercept;
public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
protected static final String OWNER = "hdfs";
- protected static final String DYNAMODB_TABLE = "dynamodb://ireland-team";
+ protected static final String DYNAMODB_TABLE = "ireland-team";
protected static final String S3A_THIS_BUCKET_DOES_NOT_EXIST
= "s3a://this-bucket-does-not-exist-00000000000";
@@ -520,7 +521,8 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
ByteArrayOutputStream buf = new ByteArrayOutputStream();
S3GuardTool.Diff cmd = new S3GuardTool.Diff(fs.getConf());
cmd.setStore(ms);
- exec(0, "", cmd, buf, "diff", "-meta", DYNAMODB_TABLE, testPath.toString());
+ String table = "dynamo://" + getTestTableName(DYNAMODB_TABLE);
+ exec(0, "", cmd, buf, "diff", "-meta", table, testPath.toString());
Set<Path> actualOnS3 = new HashSet<>();
Set<Path> actualOnMS = new HashSet<>();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java
index 2f8146d..158f13d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java
@@ -269,7 +269,8 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
@Test
public void testInitialize() throws IOException {
final S3AFileSystem s3afs = this.fileSystem;
- final String tableName = "testInitialize";
+ final String tableName =
+ getTestTableName("testInitialize");
final Configuration conf = s3afs.getConf();
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
@@ -293,7 +294,8 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
*/
@Test
public void testInitializeWithConfiguration() throws IOException {
- final String tableName = "testInitializeWithConfiguration";
+ final String tableName =
+ getTestTableName("testInitializeWithConfiguration");
final Configuration conf = getFileSystem().getConf();
conf.unset(S3GUARD_DDB_TABLE_NAME_KEY);
String savedRegion = conf.get(S3GUARD_DDB_REGION_KEY,
@@ -430,7 +432,7 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
*/
@Test
public void testTableVersionRequired() throws Exception {
- String tableName = "testTableVersionRequired";
+ String tableName = getTestTableName("testTableVersionRequired");
Configuration conf = getFileSystem().getConf();
int maxRetries = conf.getInt(S3GUARD_DDB_MAX_RETRIES,
S3GUARD_DDB_MAX_RETRIES_DEFAULT);
@@ -457,7 +459,7 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
*/
@Test
public void testTableVersionMismatch() throws Exception {
- String tableName = "testTableVersionMismatch";
+ String tableName = getTestTableName("testTableVersionMismatch");
Configuration conf = getFileSystem().getConf();
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
@@ -484,7 +486,8 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
*/
@Test
public void testFailNonexistentTable() throws IOException {
- final String tableName = "testFailNonexistentTable";
+ final String tableName =
+ getTestTableName("testFailNonexistentTable");
final S3AFileSystem s3afs = getFileSystem();
final Configuration conf = s3afs.getConf();
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
@@ -600,7 +603,8 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
@Test
public void testProvisionTable() throws Exception {
- final String tableName = "testProvisionTable-" + UUID.randomUUID();
+ final String tableName
+ = getTestTableName("testProvisionTable-" + UUID.randomUUID());
Configuration conf = getFileSystem().getConf();
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
@@ -631,7 +635,7 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
@Test
public void testDeleteTable() throws Exception {
- final String tableName = "testDeleteTable";
+ final String tableName = getTestTableName("testDeleteTable");
Path testPath = new Path(new Path(fsUri), "/" + tableName);
final S3AFileSystem s3afs = getFileSystem();
final Configuration conf = s3afs.getConf();
@@ -666,7 +670,8 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
propKey -> conf.unset(S3GUARD_DDB_TABLE_TAG + propKey)
);
- String tableName = "testTableTagging-" + UUID.randomUUID();
+ String tableName =
+ getTestTableName("testTableTagging-" + UUID.randomUUID());
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
conf.set(S3GUARD_DDB_TABLE_CREATE_KEY, "true");
@@ -765,4 +770,7 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
() -> dynamoDB.getTable(tableName).describe());
}
+ private String getTestTableName(String suffix) {
+ return getTestDynamoTablePrefix(s3AContract.getConf()) + suffix;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
index 22a1efd..fa7d1dc 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
@@ -106,7 +106,9 @@ public class ITestS3GuardConcurrentOps extends AbstractS3ATestBase {
try {
DynamoDB db = ms.getDynamoDB();
- String tableName = "testConcurrentTableCreations" + new Random().nextInt();
+ String tableName =
+ getTestTableName("testConcurrentTableCreations" +
+ new Random().nextInt());
conf.setBoolean(Constants.S3GUARD_DDB_TABLE_CREATE_KEY, true);
conf.set(Constants.S3GUARD_DDB_TABLE_NAME_KEY, tableName);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
index aaaae2e..98c1e99 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Init;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_TAG;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestDynamoTablePrefix;
import static org.apache.hadoop.fs.s3a.S3AUtils.setBucketOption;
import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
@@ -80,7 +81,8 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
@Test
public void testInvalidRegion() throws Exception {
- final String testTableName = "testInvalidRegion" + new Random().nextInt();
+ final String testTableName =
+ getTestTableName("testInvalidRegion" + new Random().nextInt());
final String testRegion = "invalidRegion";
// Initialize MetadataStore
final Init initCmd = new Init(getFileSystem().getConf());
@@ -117,7 +119,7 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
);
conf.set(S3GUARD_DDB_TABLE_NAME_KEY,
- "testDynamoTableTagging-" + UUID.randomUUID());
+ getTestTableName("testDynamoTableTagging-" + UUID.randomUUID()));
S3GuardTool.Init cmdR = new S3GuardTool.Init(conf);
Map<String, String> tagMap = new HashMap<>();
tagMap.put("hello", "dynamo");
@@ -165,7 +167,8 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
@Test
public void testDynamoDBInitDestroyCycle() throws Throwable {
- String testTableName = "testDynamoDBInitDestroy" + new Random().nextInt();
+ String testTableName =
+ getTestTableName("testDynamoDBInitDestroy" + new Random().nextInt());
String testS3Url = path(testTableName).toString();
S3AFileSystem fs = getFileSystem();
DynamoDB db = null;
@@ -284,7 +287,7 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
public void testDestroyUnknownTable() throws Throwable {
run(S3GuardTool.Destroy.NAME,
"-region", "us-west-2",
- "-meta", DYNAMODB_TABLE);
+ "-meta", "dynamodb://" + getTestTableName(DYNAMODB_TABLE));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org