You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by ga...@apache.org on 2018/01/09 20:57:18 UTC
[1/2] jclouds git commit: JCLOUDS-1369: improving slicing with many
parts
Repository: jclouds
Updated Branches:
refs/heads/master 52c92a9eb -> 8cd68a350
JCLOUDS-1369: improving slicing with many parts
Previously with GCS and its maximum 32 parts, the slicing algorithm
would sliced 3.2 GB blobs into (31) 32 MB parts and (1) 2.3 GB part.
With this algorithm it creates (31) 100 MB parts and (1) smaller part.
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/8cd68a35
Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/8cd68a35
Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/8cd68a35
Branch: refs/heads/master
Commit: 8cd68a35033312cd54822bc3f8e46ed629810712
Parents: d05be89
Author: Andrew Gaul <ga...@apache.org>
Authored: Thu Jan 4 15:41:35 2018 -0800
Committer: Andrew Gaul <ga...@apache.org>
Committed: Tue Jan 9 12:56:00 2018 -0800
----------------------------------------------------------------------
.../internal/MultipartUploadSlicingAlgorithm.java | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8cd68a35/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
----------------------------------------------------------------------
diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
index eb4074f..f8cc9f3 100644
--- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
+++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
@@ -103,14 +103,10 @@ public final class MultipartUploadSlicingAlgorithm {
unitPartSize = maximumPartSize;
parts = (int)(length / unitPartSize);
}
- if (parts > maximumNumberOfParts) { // if splits in too many parts or
- // cannot be split
- unitPartSize = minimumPartSize; // take the minimum part size
- parts = (int)(length / unitPartSize);
- }
- if (parts > maximumNumberOfParts) { // if still splits in too many parts
- parts = maximumNumberOfParts - 1; // limit them. do not care about not
- // covering
+ if (parts > maximumNumberOfParts) {
+ partSize = length / maximumNumberOfParts;
+ unitPartSize = partSize;
+ parts = maximumNumberOfParts;
}
long remainder = length % unitPartSize;
if (remainder == 0 && parts > 0) {
[2/2] jclouds git commit: JCLOUDS-1368: Correct use of slicing
algorithm
Posted by ga...@apache.org.
JCLOUDS-1368: Correct use of slicing algorithm
MultipartUploadSlicingAlgorithm creates multiple equal-sized parts and
a remaining amount. BaseBlobStore.putMultipartBlob used this
interface incorrectly, which could create more parts than intended
since the remaining size could be larger than the part size. This
manifested with Google Cloud Storage which only allows 32 parts.
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/d05be896
Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/d05be896
Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/d05be896
Branch: refs/heads/master
Commit: d05be89614b4539b2ac16e911439dcb3df2e23df
Parents: 52c92a9
Author: Andrew Gaul <ga...@apache.org>
Authored: Thu Jan 4 14:52:10 2018 -0800
Committer: Andrew Gaul <ga...@apache.org>
Committed: Tue Jan 9 12:56:00 2018 -0800
----------------------------------------------------------------------
.../org/jclouds/blobstore/internal/BaseBlobStore.java | 12 +++++++++++-
.../internal/MultipartUploadSlicingAlgorithm.java | 8 +++-----
2 files changed, 14 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds/blob/d05be896/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java
----------------------------------------------------------------------
diff --git a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java
index 087e9f6..97d4553 100644
--- a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java
+++ b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java
@@ -354,14 +354,24 @@ public abstract class BaseBlobStore implements BlobStore {
MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), overrides);
try {
long contentLength = blob.getMetadata().getContentMetadata().getContentLength();
+ // TODO: inject MultipartUploadSlicingAlgorithm to override default part size
MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts());
long partSize = algorithm.calculateChunkSize(contentLength);
int partNumber = 1;
- for (Payload payload : slicer.slice(blob.getPayload(), partSize)) {
+ // TODO: for InputStream payloads, this buffers all parts in-memory!
+ while (partNumber < algorithm.getParts()) {
+ Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), partSize);
BlobUploader b =
new BlobUploader(mpu, partNumber++, payload);
parts.add(executor.submit(b));
+ algorithm.addCopied(partSize);
+ }
+ if (algorithm.getRemaining() != 0) {
+ Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), algorithm.getRemaining());
+ BlobUploader b =
+ new BlobUploader(mpu, partNumber, payload);
+ parts.add(executor.submit(b));
}
return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts)));
} catch (RuntimeException re) {
http://git-wip-us.apache.org/repos/asf/jclouds/blob/d05be896/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
----------------------------------------------------------------------
diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
index 455ab62..eb4074f 100644
--- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
+++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
@@ -132,8 +132,7 @@ public final class MultipartUploadSlicingAlgorithm {
this.copied = copied;
}
- @VisibleForTesting
- protected int getParts() {
+ public int getParts() {
return parts;
}
@@ -141,7 +140,7 @@ public final class MultipartUploadSlicingAlgorithm {
return ++part;
}
- protected void addCopied(long copied) {
+ public void addCopied(long copied) {
this.copied += copied;
}
@@ -156,8 +155,7 @@ public final class MultipartUploadSlicingAlgorithm {
return chunkSize;
}
- @VisibleForTesting
- protected long getRemaining() {
+ public long getRemaining() {
return remaining;
}