You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2016/07/25 14:50:54 UTC
[1/4] hadoop git commit: HADOOP-13188 S3A file-create should throw
error rather than overwrite directories. Contributed by Steve Loughran
Repository: hadoop
Updated Branches:
refs/heads/branch-2 271e7bb00 -> b757eff30
refs/heads/branch-2.8 5f9bffbb6 -> 71c8bdd50
refs/heads/branch-3.0.0-alpha1 f0c4a9e0b -> eecba9211
refs/heads/trunk be9e46b42 -> 86ae21889
HADOOP-13188 S3A file-create should throw error rather than overwrite directories. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71c8bdd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71c8bdd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71c8bdd5
Branch: refs/heads/branch-2.8
Commit: 71c8bdd50c290fc161a29518cc8c99e947754354
Parents: 5f9bffb
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jul 25 15:48:32 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jul 25 15:48:49 2016 +0100
----------------------------------------------------------------------
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 19 +++++++++++++++++--
.../fs/contract/s3a/TestS3AContractCreate.java | 5 -----
2 files changed, 17 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8bdd5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 5ef1105..c45f4a5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -598,9 +598,24 @@ public class S3AFileSystem extends FileSystem {
boolean overwrite, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
String key = pathToKey(f);
+ S3AFileStatus status = null;
+ try {
+ // get the status or throw an FNFE
+ status = getFileStatus(f);
+
+ // if the thread reaches here, there is something at the path
+ if (status.isDirectory()) {
+ // path references a directory: automatic error
+ throw new FileAlreadyExistsException(f + " is a directory");
+ }
+ if (!overwrite) {
+ // path references a file and overwrite is disabled
+ throw new FileAlreadyExistsException(f + " already exists");
+ }
+ LOG.debug("Overwriting file {}", f);
+ } catch (FileNotFoundException e) {
+ // this means the file is not found
- if (!overwrite && exists(f)) {
- throw new FileAlreadyExistsException(f + " already exists");
}
instrumentation.fileCreated();
if (getConf().getBoolean(FAST_UPLOAD, DEFAULT_FAST_UPLOAD)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8bdd5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
index 1d95ddf..035252b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
@@ -30,9 +30,4 @@ public class TestS3AContractCreate extends AbstractContractCreateTest {
return new S3AContract(conf);
}
- @Override
- public void testOverwriteEmptyDirectory() throws Throwable {
- ContractTestUtils.skip(
- "blobstores can't distinguish empty directories from files");
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/4] hadoop git commit: HADOOP-13188 S3A file-create should throw
error rather than overwrite directories. Contributed by Steve Loughran
Posted by st...@apache.org.
HADOOP-13188 S3A file-create should throw error rather than overwrite directories. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86ae2188
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86ae2188
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86ae2188
Branch: refs/heads/trunk
Commit: 86ae218893d018638e937c2528c8e84336254da7
Parents: be9e46b
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jul 25 15:48:32 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jul 25 15:50:23 2016 +0100
----------------------------------------------------------------------
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 19 +++++++++++++++++--
.../fs/contract/s3a/TestS3AContractCreate.java | 5 -----
2 files changed, 17 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ae2188/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 075f36c..902f73a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -588,9 +588,24 @@ public class S3AFileSystem extends FileSystem {
boolean overwrite, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
String key = pathToKey(f);
+ S3AFileStatus status = null;
+ try {
+ // get the status or throw an FNFE
+ status = getFileStatus(f);
+
+ // if the thread reaches here, there is something at the path
+ if (status.isDirectory()) {
+ // path references a directory: automatic error
+ throw new FileAlreadyExistsException(f + " is a directory");
+ }
+ if (!overwrite) {
+ // path references a file and overwrite is disabled
+ throw new FileAlreadyExistsException(f + " already exists");
+ }
+ LOG.debug("Overwriting file {}", f);
+ } catch (FileNotFoundException e) {
+ // this means the file is not found
- if (!overwrite && exists(f)) {
- throw new FileAlreadyExistsException(f + " already exists");
}
instrumentation.fileCreated();
if (getConf().getBoolean(FAST_UPLOAD, DEFAULT_FAST_UPLOAD)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ae2188/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
index 1d95ddf..035252b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
@@ -30,9 +30,4 @@ public class TestS3AContractCreate extends AbstractContractCreateTest {
return new S3AContract(conf);
}
- @Override
- public void testOverwriteEmptyDirectory() throws Throwable {
- ContractTestUtils.skip(
- "blobstores can't distinguish empty directories from files");
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/4] hadoop git commit: HADOOP-13188 S3A file-create should throw
error rather than overwrite directories. Contributed by Steve Loughran
Posted by st...@apache.org.
HADOOP-13188 S3A file-create should throw error rather than overwrite directories. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b757eff3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b757eff3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b757eff3
Branch: refs/heads/branch-2
Commit: b757eff30cdcab5c496707a8529ca1fa473108e3
Parents: 271e7bb
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jul 25 15:48:32 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jul 25 15:49:54 2016 +0100
----------------------------------------------------------------------
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 19 +++++++++++++++++--
.../fs/contract/s3a/TestS3AContractCreate.java | 5 -----
2 files changed, 17 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b757eff3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 5ef1105..c45f4a5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -598,9 +598,24 @@ public class S3AFileSystem extends FileSystem {
boolean overwrite, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
String key = pathToKey(f);
+ S3AFileStatus status = null;
+ try {
+ // get the status or throw an FNFE
+ status = getFileStatus(f);
+
+ // if the thread reaches here, there is something at the path
+ if (status.isDirectory()) {
+ // path references a directory: automatic error
+ throw new FileAlreadyExistsException(f + " is a directory");
+ }
+ if (!overwrite) {
+ // path references a file and overwrite is disabled
+ throw new FileAlreadyExistsException(f + " already exists");
+ }
+ LOG.debug("Overwriting file {}", f);
+ } catch (FileNotFoundException e) {
+ // this means the file is not found
- if (!overwrite && exists(f)) {
- throw new FileAlreadyExistsException(f + " already exists");
}
instrumentation.fileCreated();
if (getConf().getBoolean(FAST_UPLOAD, DEFAULT_FAST_UPLOAD)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b757eff3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
index 1d95ddf..035252b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
@@ -30,9 +30,4 @@ public class TestS3AContractCreate extends AbstractContractCreateTest {
return new S3AContract(conf);
}
- @Override
- public void testOverwriteEmptyDirectory() throws Throwable {
- ContractTestUtils.skip(
- "blobstores can't distinguish empty directories from files");
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[4/4] hadoop git commit: HADOOP-13188 S3A file-create should throw
error rather than overwrite directories. Contributed by Steve Loughran
Posted by st...@apache.org.
HADOOP-13188 S3A file-create should throw error rather than overwrite directories. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eecba921
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eecba921
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eecba921
Branch: refs/heads/branch-3.0.0-alpha1
Commit: eecba921116e5c8a6b104bf7be73845e1fc58e82
Parents: f0c4a9e
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jul 25 15:48:32 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jul 25 15:50:32 2016 +0100
----------------------------------------------------------------------
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 19 +++++++++++++++++--
.../fs/contract/s3a/TestS3AContractCreate.java | 5 -----
2 files changed, 17 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eecba921/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 075f36c..902f73a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -588,9 +588,24 @@ public class S3AFileSystem extends FileSystem {
boolean overwrite, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
String key = pathToKey(f);
+ S3AFileStatus status = null;
+ try {
+ // get the status or throw an FNFE
+ status = getFileStatus(f);
+
+ // if the thread reaches here, there is something at the path
+ if (status.isDirectory()) {
+ // path references a directory: automatic error
+ throw new FileAlreadyExistsException(f + " is a directory");
+ }
+ if (!overwrite) {
+ // path references a file and overwrite is disabled
+ throw new FileAlreadyExistsException(f + " already exists");
+ }
+ LOG.debug("Overwriting file {}", f);
+ } catch (FileNotFoundException e) {
+ // this means the file is not found
- if (!overwrite && exists(f)) {
- throw new FileAlreadyExistsException(f + " already exists");
}
instrumentation.fileCreated();
if (getConf().getBoolean(FAST_UPLOAD, DEFAULT_FAST_UPLOAD)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eecba921/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
index 1d95ddf..035252b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
@@ -30,9 +30,4 @@ public class TestS3AContractCreate extends AbstractContractCreateTest {
return new S3AContract(conf);
}
- @Override
- public void testOverwriteEmptyDirectory() throws Throwable {
- ContractTestUtils.skip(
- "blobstores can't distinguish empty directories from files");
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org