You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2016/05/04 00:06:00 UTC
[13/50] hadoop git commit: HADOOP-12891. S3AFileSystem should
configure Multipart Copy threshold and chunk size. (Andrew Olson via stevel)
HADOOP-12891. S3AFileSystem should configure Multipart Copy threshold and chunk size. (Andrew Olson via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5df89f9a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5df89f9a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5df89f9a
Branch: refs/heads/branch-2.8
Commit: 5df89f9a8a4adc16820f28289878b5f800878447
Parents: adf937f
Author: Steve Loughran <st...@apache.org>
Authored: Fri Apr 22 11:24:24 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Apr 22 11:24:24 2016 +0100
----------------------------------------------------------------------
.../hadoop-common/src/main/resources/core-default.xml | 5 ++++-
.../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 ++
.../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 4 +++-
3 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df89f9a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 83dd37d..51e2df4 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -840,7 +840,10 @@
<property>
<name>fs.s3a.multipart.threshold</name>
<value>2147483647</value>
- <description>Threshold before uploads or copies use parallel multipart operations.</description>
+ <description>How big (in bytes) to split upload or copy operations up into.
+ This also controls the partition size in renamed files, as rename() involves
+ copying the source file(s)
+ </description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df89f9a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 83d1d53..4cd2783 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -318,6 +318,8 @@ public class S3AFileSystem extends FileSystem {
TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
transferConfiguration.setMinimumUploadPartSize(partSize);
transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
+ transferConfiguration.setMultipartCopyPartSize(partSize);
+ transferConfiguration.setMultipartCopyThreshold(multiPartThreshold);
transfers = new TransferManager(s3, threadPoolExecutor);
transfers.setConfiguration(transferConfiguration);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df89f9a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 15b9837..9016264 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -315,7 +315,9 @@ this capability.
<property>
<name>fs.s3a.multipart.size</name>
<value>104857600</value>
- <description>How big (in bytes) to split upload or copy operations up into.</description>
+ <description>How big (in bytes) to split upload or copy operations up into.
+ This also controls the partition size in renamed files, as rename() involves
+ copying the source file(s)</description>
</property>
<property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org