You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by ga...@apache.org on 2015/05/01 22:36:35 UTC
[2/9] jclouds git commit: JCLOUDS-894: Expose multipart component
operations
JCLOUDS-894: Expose multipart component operations
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/71cdba15
Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/71cdba15
Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/71cdba15
Branch: refs/heads/master
Commit: 71cdba15717f93c0d1bde642b74f120029c9609a
Parents: c4175df
Author: Andrew Gaul <ga...@apache.org>
Authored: Fri Feb 20 16:28:21 2015 -0800
Committer: Andrew Gaul <ga...@apache.org>
Committed: Fri May 1 13:07:23 2015 -0700
----------------------------------------------------------------------
blobstore/pom.xml | 5 +
.../java/org/jclouds/blobstore/BlobStore.java | 28 ++++++
.../jclouds/blobstore/domain/MultipartPart.java | 31 +++++++
.../blobstore/domain/MultipartUpload.java | 32 +++++++
.../blobstore/util/ForwardingBlobStore.java | 44 +++++++++
.../blobstore/util/ReadOnlyBlobStore.java | 31 +++++++
.../internal/BaseBlobIntegrationTest.java | 98 ++++++++++++++++++++
7 files changed, 269 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds/blob/71cdba15/blobstore/pom.xml
----------------------------------------------------------------------
diff --git a/blobstore/pom.xml b/blobstore/pom.xml
index f1175ec..235f1e3 100644
--- a/blobstore/pom.xml
+++ b/blobstore/pom.xml
@@ -59,6 +59,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
<groupId>com.google.auto.service</groupId>
<artifactId>auto-service</artifactId>
<optional>true</optional>
http://git-wip-us.apache.org/repos/asf/jclouds/blob/71cdba15/blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java
----------------------------------------------------------------------
diff --git a/blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java
index cf81c02..3bbe708 100644
--- a/blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java
+++ b/blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java
@@ -16,6 +16,7 @@
*/
package org.jclouds.blobstore;
+import java.util.List;
import java.util.Set;
import com.google.common.annotations.Beta;
@@ -25,6 +26,8 @@ import org.jclouds.blobstore.domain.BlobAccess;
import org.jclouds.blobstore.domain.BlobBuilder;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.ContainerAccess;
+import org.jclouds.blobstore.domain.MultipartPart;
+import org.jclouds.blobstore.domain.MultipartUpload;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.options.CopyOptions;
@@ -33,6 +36,7 @@ import org.jclouds.blobstore.options.GetOptions;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.domain.Location;
+import org.jclouds.io.Payload;
import org.jclouds.javax.annotation.Nullable;
/**
@@ -328,4 +332,28 @@ public interface BlobStore {
*/
long countBlobs(String container, ListContainerOptions options);
+ @Beta
+ MultipartUpload initiateMultipartUpload(String container, BlobMetadata blob);
+
+ @Beta
+ // TODO: take parts?
+ void abortMultipartUpload(MultipartUpload mpu);
+
+ @Beta
+ String completeMultipartUpload(MultipartUpload mpu, List<MultipartPart> parts);
+
+ @Beta
+ MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload);
+
+ @Beta
+ List<MultipartPart> listMultipartUpload(MultipartUpload mpu);
+
+ @Beta
+ long getMinimumMultipartPartSize();
+
+ @Beta
+ long getMaximumMultipartPartSize();
+
+ @Beta
+ int getMaximumNumberOfParts();
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/71cdba15/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartPart.java
----------------------------------------------------------------------
diff --git a/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartPart.java b/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartPart.java
new file mode 100644
index 0000000..0b17411
--- /dev/null
+++ b/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartPart.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jclouds.blobstore.domain;
+
+import com.google.auto.value.AutoValue;
+
+@AutoValue
+public abstract class MultipartPart {
+ public abstract int partNumber();
+ public abstract long partSize();
+ public abstract String partETag();
+
+ public static MultipartPart create(int partNumber, long partSize, String partETag) {
+ return new AutoValue_MultipartPart(partNumber, partSize, partETag);
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/71cdba15/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartUpload.java
----------------------------------------------------------------------
diff --git a/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartUpload.java b/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartUpload.java
new file mode 100644
index 0000000..d70c3ec
--- /dev/null
+++ b/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartUpload.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jclouds.blobstore.domain;
+
+import com.google.auto.value.AutoValue;
+
+@AutoValue
+public abstract class MultipartUpload {
+ public abstract String containerName();
+ public abstract String blobName();
+ public abstract String id();
+ public abstract BlobMetadata blobMetadata();
+
+ public static MultipartUpload create(String containerName, String blobName, String id, BlobMetadata blobMetadata) {
+ return new AutoValue_MultipartUpload(containerName, blobName, id, blobMetadata);
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/71cdba15/blobstore/src/main/java/org/jclouds/blobstore/util/ForwardingBlobStore.java
----------------------------------------------------------------------
diff --git a/blobstore/src/main/java/org/jclouds/blobstore/util/ForwardingBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/util/ForwardingBlobStore.java
index 60c2d7a..77f01f6 100644
--- a/blobstore/src/main/java/org/jclouds/blobstore/util/ForwardingBlobStore.java
+++ b/blobstore/src/main/java/org/jclouds/blobstore/util/ForwardingBlobStore.java
@@ -19,6 +19,7 @@ package org.jclouds.blobstore.util;
import static com.google.common.base.Preconditions.checkNotNull;
+import java.util.List;
import java.util.Set;
import com.google.common.collect.ForwardingObject;
@@ -30,6 +31,8 @@ import org.jclouds.blobstore.domain.BlobAccess;
import org.jclouds.blobstore.domain.BlobBuilder;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.ContainerAccess;
+import org.jclouds.blobstore.domain.MultipartPart;
+import org.jclouds.blobstore.domain.MultipartUpload;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.options.CopyOptions;
@@ -38,6 +41,7 @@ import org.jclouds.blobstore.options.GetOptions;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.domain.Location;
+import org.jclouds.io.Payload;
public abstract class ForwardingBlobStore extends ForwardingObject
implements BlobStore {
@@ -214,4 +218,44 @@ public abstract class ForwardingBlobStore extends ForwardingObject
public long countBlobs(String container, ListContainerOptions options) {
return delegate().countBlobs(container, options);
}
+
+ @Override
+ public MultipartUpload initiateMultipartUpload(String container, BlobMetadata blobMetadata) {
+ return delegate().initiateMultipartUpload(container, blobMetadata);
+ }
+
+ @Override
+ public void abortMultipartUpload(MultipartUpload mpu) {
+ delegate().abortMultipartUpload(mpu);
+ }
+
+ @Override
+ public String completeMultipartUpload(MultipartUpload mpu, List<MultipartPart> parts) {
+ return delegate().completeMultipartUpload(mpu, parts);
+ }
+
+ @Override
+ public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) {
+ return delegate().uploadMultipartPart(mpu, partNumber, payload);
+ }
+
+ @Override
+ public List<MultipartPart> listMultipartUpload(MultipartUpload mpu) {
+ return delegate().listMultipartUpload(mpu);
+ }
+
+ @Override
+ public long getMinimumMultipartPartSize() {
+ return delegate().getMinimumMultipartPartSize();
+ }
+
+ @Override
+ public long getMaximumMultipartPartSize() {
+ return delegate().getMaximumMultipartPartSize();
+ }
+
+ @Override
+ public int getMaximumNumberOfParts() {
+ return delegate().getMaximumNumberOfParts();
+ }
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/71cdba15/blobstore/src/main/java/org/jclouds/blobstore/util/ReadOnlyBlobStore.java
----------------------------------------------------------------------
diff --git a/blobstore/src/main/java/org/jclouds/blobstore/util/ReadOnlyBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/util/ReadOnlyBlobStore.java
index 16e481e..0fd06ee 100644
--- a/blobstore/src/main/java/org/jclouds/blobstore/util/ReadOnlyBlobStore.java
+++ b/blobstore/src/main/java/org/jclouds/blobstore/util/ReadOnlyBlobStore.java
@@ -17,17 +17,23 @@
package org.jclouds.blobstore.util;
+import java.util.List;
+
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobAccess;
import org.jclouds.blobstore.domain.BlobBuilder;
+import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.ContainerAccess;
+import org.jclouds.blobstore.domain.MultipartPart;
+import org.jclouds.blobstore.domain.MultipartUpload;
import org.jclouds.blobstore.options.CopyOptions;
import org.jclouds.blobstore.options.CreateContainerOptions;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.domain.Location;
+import org.jclouds.io.Payload;
public final class ReadOnlyBlobStore extends ForwardingBlobStore {
public static BlobStore newReadOnlyBlobStore(BlobStore blobStore) {
@@ -128,4 +134,29 @@ public final class ReadOnlyBlobStore extends ForwardingBlobStore {
BlobAccess access) {
throw new UnsupportedOperationException("Read-only BlobStore");
}
+
+ @Override
+ public MultipartUpload initiateMultipartUpload(String container, BlobMetadata blobMetadata) {
+ throw new UnsupportedOperationException("Read-only BlobStore");
+ }
+
+ @Override
+ public void abortMultipartUpload(MultipartUpload mpu) {
+ throw new UnsupportedOperationException("Read-only BlobStore");
+ }
+
+ @Override
+ public String completeMultipartUpload(MultipartUpload mpu, List<MultipartPart> parts) {
+ throw new UnsupportedOperationException("Read-only BlobStore");
+ }
+
+ @Override
+ public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) {
+ throw new UnsupportedOperationException("Read-only BlobStore");
+ }
+
+ @Override
+ public List<MultipartPart> listMultipartUpload(MultipartUpload mpu) {
+ throw new UnsupportedOperationException("Read-only BlobStore");
+ }
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/71cdba15/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
----------------------------------------------------------------------
diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
index 23de246..1dc5d81 100644
--- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
+++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
@@ -37,6 +37,7 @@ import java.io.InputStream;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.Date;
+import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.Callable;
@@ -54,6 +55,8 @@ import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobAccess;
import org.jclouds.blobstore.domain.BlobBuilder.PayloadBlobBuilder;
import org.jclouds.blobstore.domain.BlobMetadata;
+import org.jclouds.blobstore.domain.MultipartPart;
+import org.jclouds.blobstore.domain.MultipartUpload;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.StorageType;
@@ -63,6 +66,7 @@ import org.jclouds.crypto.Crypto;
import org.jclouds.encryption.internal.JCECrypto;
import org.jclouds.http.HttpResponseException;
import org.jclouds.io.ContentMetadataBuilder;
+import org.jclouds.io.ByteStreams2;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.io.payloads.ByteSourcePayload;
@@ -79,6 +83,7 @@ import org.testng.annotations.Test;
import com.google.common.base.Charsets;
import com.google.common.base.Predicate;
import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
@@ -816,6 +821,99 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
}
}
+ @Test(groups = { "integration", "live" })
+ public void testMultipartUploadNoPartsAbort() throws Exception {
+ BlobStore blobStore = view.getBlobStore();
+ String container = getContainerName();
+ try {
+ String name = "blob-name";
+ Blob blob = blobStore.blobBuilder(name).build();
+ MultipartUpload mpu = blobStore.initiateMultipartUpload(container, blob.getMetadata());
+
+ List<MultipartPart> parts = blobStore.listMultipartUpload(mpu);
+ assertThat(parts).isEqualTo(ImmutableList.of());
+
+ blobStore.abortMultipartUpload(mpu);
+
+ blob = blobStore.getBlob(container, name);
+ assertThat(blob).isNull();
+ } finally {
+ returnContainer(container);
+ }
+ }
+
+ @Test(groups = { "integration", "live" })
+ public void testMultipartUploadSinglePart() throws Exception {
+ BlobStore blobStore = view.getBlobStore();
+ String container = getContainerName();
+ try {
+ String name = "blob-name";
+ PayloadBlobBuilder blobBuilder = blobStore.blobBuilder(name)
+ .userMetadata(ImmutableMap.of("key1", "value1", "key2", "value2"))
+ // TODO: fake payload to add content metadata
+ .payload(new byte[0]);
+ addContentMetadata(blobBuilder);
+ Blob blob = blobBuilder.build();
+ MultipartUpload mpu = blobStore.initiateMultipartUpload(container, blob.getMetadata());
+
+ ByteSource byteSource = TestUtils.randomByteSource().slice(0, 1);
+ Payload payload = Payloads.newByteSourcePayload(byteSource);
+ payload.getContentMetadata().setContentLength(byteSource.size());
+ MultipartPart part = blobStore.uploadMultipartPart(mpu, 1, payload);
+
+ List<MultipartPart> parts = blobStore.listMultipartUpload(mpu);
+ assertThat(parts).isEqualTo(ImmutableList.of(part));
+
+ blobStore.completeMultipartUpload(mpu, ImmutableList.of(part));
+
+ Blob newBlob = blobStore.getBlob(container, name);
+ assertThat(newBlob).isNotNull();
+ assertThat(ByteStreams2.toByteArrayAndClose(newBlob.getPayload().openStream())).isEqualTo(byteSource.read());
+ checkContentMetadata(newBlob);
+ checkUserMetadata(newBlob.getMetadata().getUserMetadata(), blob.getMetadata().getUserMetadata());
+ } finally {
+ returnContainer(container);
+ }
+ }
+
+ @Test(groups = { "integration", "live" })
+ public void testMultipartUploadMultipleParts() throws Exception {
+ BlobStore blobStore = view.getBlobStore();
+ String container = getContainerName();
+ try {
+ String name = "blob-name";
+ PayloadBlobBuilder blobBuilder = blobStore.blobBuilder(name)
+ .userMetadata(ImmutableMap.of("key1", "value1", "key2", "value2"))
+ // TODO: fake payload to add content metadata
+ .payload(new byte[0]);
+ addContentMetadata(blobBuilder);
+ Blob blob = blobBuilder.build();
+ MultipartUpload mpu = blobStore.initiateMultipartUpload(container, blob.getMetadata());
+
+ ByteSource byteSource = TestUtils.randomByteSource().slice(0, blobStore.getMinimumMultipartPartSize() + 1);
+ ByteSource byteSource1 = byteSource.slice(0, blobStore.getMinimumMultipartPartSize());
+ ByteSource byteSource2 = byteSource.slice(blobStore.getMinimumMultipartPartSize(), 1);
+ Payload payload1 = Payloads.newByteSourcePayload(byteSource1);
+ Payload payload2 = Payloads.newByteSourcePayload(byteSource2);
+ payload1.getContentMetadata().setContentLength(byteSource1.size());
+ payload2.getContentMetadata().setContentLength(byteSource2.size());
+ MultipartPart part1 = blobStore.uploadMultipartPart(mpu, 1, payload1);
+ MultipartPart part2 = blobStore.uploadMultipartPart(mpu, 2, payload2);
+
+ List<MultipartPart> parts = blobStore.listMultipartUpload(mpu);
+ assertThat(parts).isEqualTo(ImmutableList.of(part1, part2));
+
+ blobStore.completeMultipartUpload(mpu, ImmutableList.of(part1, part2));
+
+ Blob newBlob = blobStore.getBlob(container, name);
+ assertThat(ByteStreams2.toByteArrayAndClose(newBlob.getPayload().openStream())).isEqualTo(byteSource.read());
+ checkContentMetadata(newBlob);
+ checkUserMetadata(newBlob.getMetadata().getUserMetadata(), blob.getMetadata().getUserMetadata());
+ } finally {
+ returnContainer(container);
+ }
+ }
+
protected void validateMetadata(BlobMetadata metadata) throws IOException {
assert metadata.getContentMetadata().getContentType().startsWith("text/plain") : metadata.getContentMetadata()
.getContentType();