You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ga...@apache.org on 2020/11/26 12:51:02 UTC
[hadoop] branch trunk updated: HADOOP-17385.
ITestS3ADeleteCost.testDirMarkersFileCreation failure (#2473). Contributed
by Steve Loughran
This is an automated email from the ASF dual-hosted git repository.
gabota pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 67dc092 HADOOP-17385. ITestS3ADeleteCost.testDirMarkersFileCreation failure (#2473). Contributed by Steve Loughran
67dc092 is described below
commit 67dc0928c1accd0093317b64f551cc1b63f01d9e
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Thu Nov 26 12:50:33 2020 +0000
HADOOP-17385. ITestS3ADeleteCost.testDirMarkersFileCreation failure (#2473). Contributed by Steve Loughran
The addition of deprecated S3A configuration options in HADOOP-17318
triggered a reload of default (xml resource) configurations, which breaks
tests which fail if there's a per-bucket setting inconsistent with test
setup.
Creating an S3AFS instance before creating the Configuration() instance
for test runs gets that reload out the way before test setup takes
place.
Along with the fix, extra changes in the failing test suite to fail
fast when marker policy isn't as expected, and to log FS state better.
Rather than create and discard an instance, add a new static method
to S3AFS and invoke it in test setup. This forces the load
Change-Id: Id52b1c46912c6fedd2ae270e2b1eb2222a360329
---
.../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 9 +++++++++
.../test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java | 4 ++++
.../test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java | 3 +++
.../apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java | 9 +++++++++
.../org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java | 1 +
5 files changed, 26 insertions(+)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index f2cd2cd..a02947b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -4955,6 +4955,15 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
/**
+ * This is purely for testing, as it force initializes all static
+ * initializers. See HADOOP-17385 for details.
+ */
+ @InterfaceAudience.Private
+ public static void initializeClass() {
+ LOG.debug("Initialize S3A class");
+ }
+
+ /**
* The implementation of context accessors.
*/
private class ContextAccessorsImpl implements ContextAccessors {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
index 1311020..0d3dd4c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.contract.s3a;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
/**
@@ -53,6 +54,9 @@ public class S3AContract extends AbstractBondedFSContract {
*/
public S3AContract(Configuration conf, boolean addContractResource) {
super(conf);
+ // Force deprecated key load through the
+ // static initializers. See: HADOOP-17385
+ S3AFileSystem.initializeClass();
//insert the base features
if (addContractResource) {
addConfResource(CONTRACT_XML);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
index da679cd..73a503a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
@@ -60,6 +60,9 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase
// filesystems which add default configuration resources to do it before
// our tests start adding/removing options. See HADOOP-16626.
FileSystem.getLocal(new Configuration());
+ // Force deprecated key load through the
+ // static initializers. See: HADOOP-17385
+ S3AFileSystem.initializeClass();
super.setup();
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
index d7e277f..2b2fb7c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.Statistic;
import org.apache.hadoop.fs.s3a.Tristate;
+import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy;
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
import static org.apache.hadoop.fs.s3a.Constants.*;
@@ -141,6 +142,14 @@ public class AbstractS3ACostTest extends AbstractS3ATestBase {
isDeleting = !isKeeping;
+ // check that the FS has the expected state
+ DirectoryPolicy markerPolicy = fs.getDirectoryMarkerPolicy();
+ Assertions.assertThat(markerPolicy.getMarkerPolicy())
+ .describedAs("Marker policy for filesystem %s", fs)
+ .isEqualTo(isKeepingMarkers()
+ ? DirectoryPolicy.MarkerPolicy.Keep
+ : DirectoryPolicy.MarkerPolicy.Delete);
+
// insert new metrics so as to keep the list sorted
costValidator = OperationCostValidator.builder(getFileSystem())
.withMetrics(
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
index 618b491..0b5afc6 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
@@ -267,6 +267,7 @@ public class ITestS3ADeleteCost extends AbstractS3ACostTest {
verifyMetrics(() -> {
file(new Path(srcDir, "source.txt"));
+ LOG.info("Metrics: {}\n{}", getMetricSummary(), getFileSystem());
return "after touch(fs, srcFilePath) " + getMetricSummary();
},
with(DIRECTORIES_CREATED, 0),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org