You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by ga...@apache.org on 2017/05/08 21:44:17 UTC
[19/24] jclouds git commit: JCLOUDS-1267: Allow B2 streaming uploads
JCLOUDS-1267: Allow B2 streaming uploads
B2 now supports uploads without calculating the SHA-1 hash. This
allows uploading without a repeatable payload.
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/66caf6d9
Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/66caf6d9
Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/66caf6d9
Branch: refs/heads/master
Commit: 66caf6d954043d453b8c5261d23687d62b670955
Parents: 8408242
Author: Andrew Gaul <ga...@apache.org>
Authored: Fri Apr 7 17:28:35 2017 -0700
Committer: Andrew Gaul <ga...@apache.org>
Committed: Fri Apr 7 19:48:58 2017 -0700
----------------------------------------------------------------------
.../jclouds/b2/binders/UploadFileBinder.java | 5 +++
.../jclouds/b2/binders/UploadPartBinder.java | 6 ++++
.../org/jclouds/b2/blobstore/B2BlobStore.java | 32 +++++---------------
.../org/jclouds/b2/features/MultipartApi.java | 2 +-
.../java/org/jclouds/b2/features/ObjectApi.java | 3 +-
.../integration/B2BlobIntegrationLiveTest.java | 20 ------------
6 files changed, 20 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds/blob/66caf6d9/providers/b2/src/main/java/org/jclouds/b2/binders/UploadFileBinder.java
----------------------------------------------------------------------
diff --git a/providers/b2/src/main/java/org/jclouds/b2/binders/UploadFileBinder.java b/providers/b2/src/main/java/org/jclouds/b2/binders/UploadFileBinder.java
index 23ddf05..aa4ebc9 100644
--- a/providers/b2/src/main/java/org/jclouds/b2/binders/UploadFileBinder.java
+++ b/providers/b2/src/main/java/org/jclouds/b2/binders/UploadFileBinder.java
@@ -33,10 +33,15 @@ public final class UploadFileBinder implements MapBinder {
public <R extends HttpRequest> R bindToRequest(R request, Map<String, Object> postParams) {
UploadUrlResponse uploadUrl = (UploadUrlResponse) postParams.get("uploadUrl");
String fileName = (String) postParams.get("fileName");
+ String contentSha1 = (String) postParams.get("contentSha1");
+ if (contentSha1 == null) {
+ contentSha1 = "do_not_verify";
+ }
Map<String, String> fileInfo = (Map<String, String>) postParams.get("fileInfo");
HttpRequest.Builder builder = request.toBuilder()
.endpoint(uploadUrl.uploadUrl())
.replaceHeader(HttpHeaders.AUTHORIZATION, uploadUrl.authorizationToken())
+ .replaceHeader(B2Headers.CONTENT_SHA1, contentSha1)
.replaceHeader(B2Headers.FILE_NAME, escaper.escape(fileName));
for (Map.Entry<String, String> entry : fileInfo.entrySet()) {
builder.replaceHeader(B2Headers.FILE_INFO_PREFIX + entry.getKey(), escaper.escape(entry.getValue()));
http://git-wip-us.apache.org/repos/asf/jclouds/blob/66caf6d9/providers/b2/src/main/java/org/jclouds/b2/binders/UploadPartBinder.java
----------------------------------------------------------------------
diff --git a/providers/b2/src/main/java/org/jclouds/b2/binders/UploadPartBinder.java b/providers/b2/src/main/java/org/jclouds/b2/binders/UploadPartBinder.java
index 0275412..e89e2df 100644
--- a/providers/b2/src/main/java/org/jclouds/b2/binders/UploadPartBinder.java
+++ b/providers/b2/src/main/java/org/jclouds/b2/binders/UploadPartBinder.java
@@ -20,6 +20,7 @@ import java.util.Map;
import org.jclouds.http.HttpRequest;
import org.jclouds.b2.domain.GetUploadPartResponse;
+import org.jclouds.b2.reference.B2Headers;
import org.jclouds.rest.MapBinder;
import com.google.common.net.HttpHeaders;
@@ -28,9 +29,14 @@ public final class UploadPartBinder implements MapBinder {
@Override
public <R extends HttpRequest> R bindToRequest(R request, Map<String, Object> postParams) {
GetUploadPartResponse uploadUrl = (GetUploadPartResponse) postParams.get("response");
+ String contentSha1 = (String) postParams.get("contentSha1");
+ if (contentSha1 == null) {
+ contentSha1 = "do_not_verify";
+ }
return (R) request.toBuilder()
.endpoint(uploadUrl.uploadUrl())
.replaceHeader(HttpHeaders.AUTHORIZATION, uploadUrl.authorizationToken())
+ .replaceHeader(B2Headers.CONTENT_SHA1, contentSha1)
.build();
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/66caf6d9/providers/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java
----------------------------------------------------------------------
diff --git a/providers/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java b/providers/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java
index 00953b6..aa40fc5 100644
--- a/providers/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java
+++ b/providers/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java
@@ -16,7 +16,6 @@
*/
package org.jclouds.b2.blobstore;
-import java.io.IOException;
import java.net.URI;
import java.util.Date;
import java.util.List;
@@ -67,7 +66,6 @@ import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.collect.Memoized;
import org.jclouds.domain.Location;
-import org.jclouds.io.ByteStreams2;
import org.jclouds.io.ContentMetadata;
import org.jclouds.io.ContentMetadataBuilder;
import org.jclouds.io.MutableContentMetadata;
@@ -78,14 +76,12 @@ import org.jclouds.io.payloads.BaseMutableContentMetadata;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
-import com.google.common.base.Throwables;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
-import com.google.common.hash.Hashing;
import com.google.common.net.HttpHeaders;
import com.google.common.util.concurrent.UncheckedExecutionException;
@@ -254,8 +250,6 @@ public final class B2BlobStore extends BaseBlobStore {
@Override
public String putBlob(String container, Blob blob, PutOptions options) {
- Preconditions.checkArgument(blob.getPayload().isRepeatable(), "B2 requires repeatable payload to calculate SHA1");
-
if (options.getBlobAccess() != BlobAccess.PRIVATE) {
throw new UnsupportedOperationException("B2 only supports private access blobs");
}
@@ -263,13 +257,6 @@ public final class B2BlobStore extends BaseBlobStore {
if (options.isMultipart()) {
return putMultipartBlob(container, blob, options);
} else {
- String contentSha1;
- try {
- contentSha1 = ByteStreams2.hashAndClose(blob.getPayload().openStream(), Hashing.sha1()).toString();
- } catch (IOException ioe) {
- throw Throwables.propagate(ioe);
- }
-
String name = blob.getMetadata().getName();
// B2 versions all files so we store the original fileId to delete it after the upload succeeds
@@ -277,7 +264,7 @@ public final class B2BlobStore extends BaseBlobStore {
Bucket bucket = getBucket(container);
UploadUrlResponse uploadUrl = api.getObjectApi().getUploadUrl(bucket.bucketId());
- UploadFileResponse uploadFile = api.getObjectApi().uploadFile(uploadUrl, name, contentSha1, blob.getMetadata().getUserMetadata(), blob.getPayload());
+ UploadFileResponse uploadFile = api.getObjectApi().uploadFile(uploadUrl, name, null, blob.getMetadata().getUserMetadata(), blob.getPayload());
if (oldFileId != null) {
api.getObjectApi().deleteFileVersion(name, oldFileId);
@@ -397,20 +384,15 @@ public final class B2BlobStore extends BaseBlobStore {
@Override
public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) {
- Preconditions.checkArgument(payload.isRepeatable());
-
- String contentSha1;
- try {
- contentSha1 = ByteStreams2.hashAndClose(payload.openStream(), Hashing.sha1()).toString();
- } catch (IOException ioe) {
- throw Throwables.propagate(ioe);
- }
-
GetUploadPartResponse getUploadPart = api.getMultipartApi().getUploadPartUrl(mpu.id());
- UploadPartResponse uploadPart = api.getMultipartApi().uploadPart(getUploadPart, partNumber, contentSha1, payload);
+ UploadPartResponse uploadPart = api.getMultipartApi().uploadPart(getUploadPart, partNumber, null, payload);
Date lastModified = null; // B2 does not return Last-Modified
- return MultipartPart.create(uploadPart.partNumber(), uploadPart.contentLength(), uploadPart.contentSha1(), lastModified);
+ String contentSha1 = uploadPart.contentSha1();
+ if (contentSha1.startsWith("unverified:")) {
+ contentSha1 = contentSha1.substring("unverified:".length());
+ }
+ return MultipartPart.create(uploadPart.partNumber(), uploadPart.contentLength(), contentSha1, lastModified);
}
@Override
http://git-wip-us.apache.org/repos/asf/jclouds/blob/66caf6d9/providers/b2/src/main/java/org/jclouds/b2/features/MultipartApi.java
----------------------------------------------------------------------
diff --git a/providers/b2/src/main/java/org/jclouds/b2/features/MultipartApi.java b/providers/b2/src/main/java/org/jclouds/b2/features/MultipartApi.java
index 5affc4e..510fd0f 100644
--- a/providers/b2/src/main/java/org/jclouds/b2/features/MultipartApi.java
+++ b/providers/b2/src/main/java/org/jclouds/b2/features/MultipartApi.java
@@ -84,7 +84,7 @@ public interface MultipartApi {
@Named("b2_upload_part")
@POST
@MapBinder(UploadPartBinder.class)
- UploadPartResponse uploadPart(@PayloadParam("response") GetUploadPartResponse response, @HeaderParam("X-Bz-Part-Number") int partNumber, @HeaderParam("X-Bz-Content-Sha1") String sha1, @PayloadParam("payload") Payload payload);
+ UploadPartResponse uploadPart(@PayloadParam("response") GetUploadPartResponse response, @HeaderParam("X-Bz-Part-Number") int partNumber, @Nullable @PayloadParam("contentSha1") String sha1, @PayloadParam("payload") Payload payload);
@Named("b2_list_parts")
@POST
http://git-wip-us.apache.org/repos/asf/jclouds/blob/66caf6d9/providers/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java
----------------------------------------------------------------------
diff --git a/providers/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java b/providers/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java
index 4864864..c8379eb 100644
--- a/providers/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java
+++ b/providers/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java
@@ -24,7 +24,6 @@ import java.util.Map;
import javax.inject.Named;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
-import javax.ws.rs.HeaderParam;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@@ -68,7 +67,7 @@ public interface ObjectApi {
@POST
@MapBinder(UploadFileBinder.class)
@Consumes(APPLICATION_JSON)
- UploadFileResponse uploadFile(@PayloadParam("uploadUrl") UploadUrlResponse uploadUrl, @PayloadParam("fileName") String fileName, @HeaderParam("X-Bz-Content-Sha1") String contentSha1, @PayloadParam("fileInfo") Map<String, String> fileInfo, Payload payload);
+ UploadFileResponse uploadFile(@PayloadParam("uploadUrl") UploadUrlResponse uploadUrl, @PayloadParam("fileName") String fileName, @Nullable @PayloadParam("contentSha1") String contentSha1, @PayloadParam("fileInfo") Map<String, String> fileInfo, Payload payload);
@Named("b2_delete_file_version")
@POST
http://git-wip-us.apache.org/repos/asf/jclouds/blob/66caf6d9/providers/b2/src/test/java/org/jclouds/b2/blobstore/integration/B2BlobIntegrationLiveTest.java
----------------------------------------------------------------------
diff --git a/providers/b2/src/test/java/org/jclouds/b2/blobstore/integration/B2BlobIntegrationLiveTest.java b/providers/b2/src/test/java/org/jclouds/b2/blobstore/integration/B2BlobIntegrationLiveTest.java
index ae2326e..846b25b 100644
--- a/providers/b2/src/test/java/org/jclouds/b2/blobstore/integration/B2BlobIntegrationLiveTest.java
+++ b/providers/b2/src/test/java/org/jclouds/b2/blobstore/integration/B2BlobIntegrationLiveTest.java
@@ -123,26 +123,6 @@ public final class B2BlobIntegrationLiveTest extends BaseBlobIntegrationTest {
}
@Override
- public void testPutInputStream() throws Exception {
- try {
- super.testPutInputStream();
- failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
- } catch (IllegalArgumentException iae) {
- throw new SkipException("B2 requires repeatable payloads to calculate SHA1 hash", iae);
- }
- }
-
- @Override
- public void testPutMultipartInputStream() throws Exception {
- try {
- super.testPutMultipartInputStream();
- failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
- } catch (IllegalArgumentException iae) {
- throw new SkipException("B2 requires repeatable payloads to calculate SHA1 hash", iae);
- }
- }
-
- @Override
public void testPutObjectStream() throws InterruptedException, IOException, ExecutionException {
try {
super.testPutObjectStream();