You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2018/10/02 21:40:44 UTC
[44/50] [abbrv] hadoop git commit: HDFS-13936. Multipart upload to
HDFS to support 0 byte upload. Contributed by Ewan Higgs.
HDFS-13936. Multipart upload to HDFS to support 0 byte upload. Contributed by Ewan Higgs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fab6886
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fab6886
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fab6886
Branch: refs/heads/HDFS-13532
Commit: 6fab6886f652492573734b832ca0375459a82775
Parents: a383ac4
Author: Ewan Higgs <eh...@apache.org>
Authored: Tue Oct 2 14:03:28 2018 +0200
Committer: Ewan Higgs <eh...@apache.org>
Committed: Tue Oct 2 14:05:35 2018 +0200
----------------------------------------------------------------------
.../hadoop/fs/FileSystemMultipartUploader.java | 25 +++++++++++++++-----
.../AbstractContractMultipartUploaderTest.java | 22 +++++++++++++++++
2 files changed, 41 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fab6886/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
index 690194d..94c7861 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
@@ -100,6 +100,14 @@ public class FileSystemMultipartUploader extends MultipartUploader {
return fs.getPathHandle(status);
}
+ private long totalPartsLen(List<Path> partHandles) throws IOException {
+ long totalLen = 0;
+ for (Path p: partHandles) {
+ totalLen += fs.getFileStatus(p).getLen();
+ }
+ return totalLen;
+ }
+
@Override
@SuppressWarnings("deprecation") // rename w/ OVERWRITE
public PathHandle complete(Path filePath,
@@ -127,12 +135,17 @@ public class FileSystemMultipartUploader extends MultipartUploader {
.collect(Collectors.toList());
Path collectorPath = createCollectorPath(filePath);
- Path filePathInsideCollector = mergePaths(collectorPath,
- new Path(Path.SEPARATOR + filePath.getName()));
- fs.create(filePathInsideCollector).close();
- fs.concat(filePathInsideCollector,
- partHandles.toArray(new Path[handles.size()]));
- fs.rename(filePathInsideCollector, filePath, Options.Rename.OVERWRITE);
+ boolean emptyFile = totalPartsLen(partHandles) == 0;
+ if (emptyFile) {
+ fs.create(filePath).close();
+ } else {
+ Path filePathInsideCollector = mergePaths(collectorPath,
+ new Path(Path.SEPARATOR + filePath.getName()));
+ fs.create(filePathInsideCollector).close();
+ fs.concat(filePathInsideCollector,
+ partHandles.toArray(new Path[handles.size()]));
+ fs.rename(filePathInsideCollector, filePath, Options.Rename.OVERWRITE);
+ }
fs.delete(collectorPath, true);
return getPathHandle(filePath);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fab6886/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
index 85a6861..7cee5a6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
@@ -165,6 +165,28 @@ public abstract class AbstractContractMultipartUploaderTest extends
}
/**
+ * Assert that a multipart upload is successful when a single empty part is
+ * uploaded.
+ * @throws Exception failure
+ */
+ @Test
+ public void testMultipartUploadEmptyPart() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path file = path("testMultipartUpload");
+ MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
+ UploadHandle uploadHandle = mpu.initialize(file);
+ List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ MessageDigest origDigest = DigestUtils.getMd5Digest();
+ byte[] payload = new byte[0];
+ origDigest.update(payload);
+ InputStream is = new ByteArrayInputStream(payload);
+ PartHandle partHandle = mpu.putPart(file, is, 0, uploadHandle,
+ payload.length);
+ partHandles.add(Pair.of(0, partHandle));
+ completeUpload(file, mpu, uploadHandle, partHandles, origDigest, 0);
+ }
+
+ /**
* Assert that a multipart upload is successful even when the parts are
* given in the reverse order.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org