You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2016/10/26 16:28:27 UTC
[3/3] hadoop git commit: HADOOP-13502. Split fs.contract.is-blobstore
flag into more descriptive flags for use by contract tests. Contributed by
Chris Nauroth.
HADOOP-13502. Split fs.contract.is-blobstore flag into more descriptive flags for use by contract tests. Contributed by Chris Nauroth.
(cherry picked from commit 1f8490a5bacd98d0d371447ada3b31f93ca40a4e)
(cherry picked from commit 082d69ee663d5c1a59f32c40b9fbde9996886de9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4df17809
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4df17809
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4df17809
Branch: refs/heads/branch-2.8
Commit: 4df1780991db8262dce64d7bfeae38b99751a42a
Parents: c940c68
Author: Chris Nauroth <cn...@apache.org>
Authored: Wed Oct 26 08:57:29 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Wed Oct 26 08:57:46 2016 -0700
----------------------------------------------------------------------
.gitignore | 1 +
.../fs/contract/AbstractContractCreateTest.java | 18 +++++++++---------
.../hadoop/fs/contract/ContractOptions.java | 18 ++++++++++++++++++
.../src/test/resources/contract/ftp.xml | 7 +------
.../hadoop-aws/src/test/resources/contract/s3.xml | 10 ++++++++++
.../src/test/resources/contract/s3a.xml | 5 +++++
.../src/test/resources/contract/s3n.xml | 10 ++++++++++
.../src/test/resources/contract/swift.xml | 10 ++++++++++
8 files changed, 64 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 5925ec4..eb1fc96 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,4 +29,5 @@ yarnregistry.pdf
hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml
hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml
hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml
+hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
patchprocess/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java
index 9344225..84dc775 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java
@@ -121,7 +121,7 @@ public abstract class AbstractContractCreateTest extends
try {
assertIsDirectory(path);
} catch (AssertionError failure) {
- if (isSupported(IS_BLOBSTORE)) {
+ if (isSupported(CREATE_OVERWRITES_DIRECTORY)) {
// file/directory hack surfaces here
throw new AssumptionViolatedException(failure.toString(), failure);
}
@@ -137,10 +137,10 @@ public abstract class AbstractContractCreateTest extends
FileStatus status = getFileSystem().getFileStatus(path);
boolean isDir = status.isDirectory();
- if (!isDir && isSupported(IS_BLOBSTORE)) {
- // object store: downgrade to a skip so that the failure is visible
- // in test results
- skip("Object store allows a file to overwrite a directory");
+ if (!isDir && isSupported(CREATE_OVERWRITES_DIRECTORY)) {
+ // For some file systems, downgrade to a skip so that the failure is
+ // visible in test results.
+ skip("This Filesystem allows a file to overwrite a directory");
}
fail("write of file over dir succeeded");
} catch (FileAlreadyExistsException expected) {
@@ -170,10 +170,10 @@ public abstract class AbstractContractCreateTest extends
1024)) {
if (!getFileSystem().exists(path)) {
- if (isSupported(IS_BLOBSTORE)) {
- // object store: downgrade to a skip so that the failure is visible
- // in test results
- skip("Filesystem is an object store and newly created files are not immediately visible");
+ if (isSupported(CREATE_VISIBILITY_DELAYED)) {
+ // For some file systems, downgrade to a skip so that the failure is
+ // visible in test results.
+ skip("This Filesystem delays visibility of newly created files");
}
assertPathExists("expected path to be visible before anything written",
path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
index c8af062..b778f0d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
@@ -37,16 +37,33 @@ public interface ContractOptions {
String FS_CONTRACT_KEY = "fs.contract.";
/**
+ * Flag to indicate that a newly created file may overwrite a pre-existing
+ * directory.
+ * {@value}
+ */
+ String CREATE_OVERWRITES_DIRECTORY = "create-overwrites-directory";
+
+ /**
+ * Flag to indicate that a newly created file is not made visible in the
+ * namespace immediately. Instead, the file becomes visible at a later point
+ * in the file creation lifecycle, such as when the client closes it.
+ * {@value}
+ */
+ String CREATE_VISIBILITY_DELAYED = "create-visibility-delayed";
+
+ /**
* Is a filesystem case sensitive.
* Some of the filesystems that say "no" here may mean
* that it varies from platform to platform -the localfs being the key
* example.
+ * {@value}
*/
String IS_CASE_SENSITIVE = "is-case-sensitive";
/**
* Blobstore flag. Implies it's not a real directory tree and
* consistency is below that which Hadoop expects
+ * {@value}
*/
String IS_BLOBSTORE = "is-blobstore";
@@ -196,6 +213,7 @@ public interface ContractOptions {
/**
* Limit for #of random seeks to perform.
* Keep low for remote filesystems for faster tests
+ * {@value}
*/
String TEST_RANDOM_SEEK_COUNT = "test.random-seek-count";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml b/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml
index 9c3e380..f95446e 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml
@@ -27,11 +27,6 @@
</property>
<property>
- <name>fs.contract.is-blobstore</name>
- <value>false</value>
- </property>
-
- <property>
<name>fs.contract.is-case-sensitive</name>
<value>true</value>
</property>
@@ -81,4 +76,4 @@
<value>false</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
index 4b742c1..49acd3b 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
@@ -37,6 +37,16 @@
</property>
<property>
+ <name>fs.contract.create-overwrites-directory</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.create-visibility-delayed</name>
+ <value>true</value>
+ </property>
+
+ <property>
<name>fs.contract.is-case-sensitive</name>
<value>true</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
index be1e7ca..a534f0a 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
@@ -38,6 +38,11 @@
</property>
<property>
+ <name>fs.contract.create-visibility-delayed</name>
+ <value>true</value>
+ </property>
+
+ <property>
<name>fs.contract.is-case-sensitive</name>
<value>true</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
index cb8aca7..0c6b8c6 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
@@ -38,6 +38,16 @@
</property>
<property>
+ <name>fs.contract.create-overwrites-directory</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.create-visibility-delayed</name>
+ <value>true</value>
+ </property>
+
+ <property>
<name>fs.contract.is-case-sensitive</name>
<value>true</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4df17809/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml b/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml
index 12a67e0..fbf3a17 100644
--- a/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml
+++ b/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml
@@ -38,6 +38,16 @@
</property>
<property>
+ <name>fs.contract.create-overwrites-directory</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.create-visibility-delayed</name>
+ <value>true</value>
+ </property>
+
+ <property>
<name>fs.contract.is-case-sensitive</name>
<value>true</value>
</property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org