You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2015/02/05 13:20:37 UTC
[2/2] hadoop git commit: HADOOP-11463 Replace method-local
TransferManager object with S3AFileSystem#transfers. (Ted Yu via stevel)
HADOOP-11463 Replace method-local TransferManager object with S3AFileSystem#transfers. (Ted Yu via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e7ad4f0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e7ad4f0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e7ad4f0
Branch: refs/heads/trunk
Commit: 4e7ad4f0a88bd36bc91db6b1bd311d7f5c6bebee
Parents: 20660b7
Author: Steve Loughran <st...@apache.org>
Authored: Thu Feb 5 12:19:49 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Feb 5 12:20:23 2015 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 30 ++++++++------------
2 files changed, 15 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e7ad4f0/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 84c9200..9f136f5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -544,6 +544,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11492. Bump up curator version to 2.7.1. (Arun Suresh and
Karthik Kambatla via kasha)
+ HADOOP-11463 Replace method-local TransferManager object with
+ S3AFileSystem#transfers. (Ted Yu via stevel)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e7ad4f0/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 0e4d54f..22350bc 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -292,7 +292,6 @@ public class S3AFileSystem extends FileSystem {
Date purgeBefore = new Date(new Date().getTime() - purgeExistingMultipartAge*1000);
transfers.abortMultipartUploads(bucket, purgeBefore);
- transfers.shutdownNow(false);
}
serverSideEncryptionAlgorithm = conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM);
@@ -995,13 +994,6 @@ public class S3AFileSystem extends FileSystem {
LocalFileSystem local = getLocal(getConf());
File srcfile = local.pathToFile(src);
- TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
- transferConfiguration.setMinimumUploadPartSize(partSize);
- transferConfiguration.setMultipartUploadThreshold(partSizeThreshold);
-
- TransferManager transfers = new TransferManager(s3);
- transfers.setConfiguration(transferConfiguration);
-
final ObjectMetadata om = new ObjectMetadata();
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
om.setServerSideEncryption(serverSideEncryptionAlgorithm);
@@ -1029,8 +1021,6 @@ public class S3AFileSystem extends FileSystem {
statistics.incrementWriteOps(1);
} catch (InterruptedException e) {
throw new IOException("Got interrupted, cancelling");
- } finally {
- transfers.shutdownNow(false);
}
// This will delete unnecessary fake parent directories
@@ -1041,6 +1031,18 @@ public class S3AFileSystem extends FileSystem {
}
}
+ @Override
+ public void close() throws IOException {
+ try {
+ super.close();
+ } finally {
+ if (transfers != null) {
+ transfers.shutdownNow(true);
+ transfers = null;
+ }
+ }
+ }
+
/**
* Override getCononicalServiceName because we don't support token in S3A
*/
@@ -1055,12 +1057,6 @@ public class S3AFileSystem extends FileSystem {
LOG.debug("copyFile " + srcKey + " -> " + dstKey);
}
- TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
- transferConfiguration.setMultipartCopyPartSize(partSize);
-
- TransferManager transfers = new TransferManager(s3);
- transfers.setConfiguration(transferConfiguration);
-
ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey);
final ObjectMetadata dstom = srcom.clone();
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
@@ -1089,8 +1085,6 @@ public class S3AFileSystem extends FileSystem {
statistics.incrementWriteOps(1);
} catch (InterruptedException e) {
throw new IOException("Got interrupted, cancelling");
- } finally {
- transfers.shutdownNow(false);
}
}