You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by ga...@apache.org on 2014/12/06 22:54:02 UTC
[2/2] jclouds git commit: JCLOUDS-671: Use Iterable payload slicer
for Azure
JCLOUDS-671: Use Iterable payload slicer for Azure
Payload slicer has a method that returns an iterable of payloads that
works on non-repeatable InputStreams that was introduced to fix
multi-part uploads in Swift (JCLOUDS-356). This commit applies the
same method to fix multi-part uploads for Azure blob store.
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/66c4963b
Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/66c4963b
Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/66c4963b
Branch: refs/heads/master
Commit: 66c4963b49a1ddd985e3b57309e465ff1922cc26
Parents: 06653f1
Author: Eric Jutrzenka <er...@gmail.com>
Authored: Sat Nov 29 14:43:50 2014 -0800
Committer: Andrew Gaul <ga...@apache.org>
Committed: Sat Dec 6 13:52:27 2014 -0800
----------------------------------------------------------------------
.../strategy/AzureBlobBlockUploadStrategy.java | 18 ++++-------
.../AzureBlobBlockUploadStrategyTest.java | 34 ++++++++++++++------
2 files changed, 30 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds/blob/66c4963b/providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/strategy/AzureBlobBlockUploadStrategy.java
----------------------------------------------------------------------
diff --git a/providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/strategy/AzureBlobBlockUploadStrategy.java b/providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/strategy/AzureBlobBlockUploadStrategy.java
index 950dcf0..74ebda9 100644
--- a/providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/strategy/AzureBlobBlockUploadStrategy.java
+++ b/providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/strategy/AzureBlobBlockUploadStrategy.java
@@ -29,9 +29,8 @@ import org.jclouds.logging.Logger;
import javax.annotation.Resource;
import javax.inject.Named;
-
-import java.security.SecureRandom;
import java.util.List;
+import java.util.UUID;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -62,23 +61,18 @@ public class AzureBlobBlockUploadStrategy implements MultipartUploadStrategy {
checkNotNull(length,
"please invoke payload.getContentMetadata().setContentLength(length) prior to azure block upload");
checkArgument(length <= (MAX_NUMBER_OF_BLOCKS * MAX_BLOCK_SIZE));
- long offset = 0L;
List<String> blockIds = Lists.newArrayList();
long bytesWritten = 0;
- while (offset < length) {
- long chunkSize = MAX_BLOCK_SIZE;
- if (length - offset < MAX_BLOCK_SIZE) {
- chunkSize = length % MAX_BLOCK_SIZE;
- }
- bytesWritten += chunkSize;
- Payload block = slicer.slice(payload, offset, chunkSize);
- offset += MultipartUploadStrategy.MAX_BLOCK_SIZE;
- String blockName = blobName + "-" + offset + "-" + new SecureRandom().nextInt();
+
+ for (Payload block : slicer.slice(payload, MAX_BLOCK_SIZE)) {
+ String blockName = blobName + "-" + UUID.randomUUID().toString();
byte blockIdBytes[] = Hashing.md5().hashBytes(blockName.getBytes()).asBytes();
String blockId = BaseEncoding.base64().encode(blockIdBytes);
blockIds.add(blockId);
client.putBlock(container, blobName, blockId, block);
+ bytesWritten += block.getContentMetadata().getContentLength();
}
+
checkState(bytesWritten == length, "Wrote %s bytes, but we wanted to write %s bytes", bytesWritten, length);
return client.putBlockList(container, blobName, blockIds);
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/66c4963b/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/strategy/AzureBlobBlockUploadStrategyTest.java
----------------------------------------------------------------------
diff --git a/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/strategy/AzureBlobBlockUploadStrategyTest.java b/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/strategy/AzureBlobBlockUploadStrategyTest.java
index 615b3f4..aac7d30 100644
--- a/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/strategy/AzureBlobBlockUploadStrategyTest.java
+++ b/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/strategy/AzureBlobBlockUploadStrategyTest.java
@@ -18,6 +18,7 @@
package org.jclouds.azureblob.blobstore.strategy;
import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableList;
import com.google.common.io.ByteSource;
import org.easymock.EasyMock;
import org.jclouds.azureblob.AzureBlobClient;
@@ -30,6 +31,7 @@ import org.jclouds.io.Payload;
import org.jclouds.io.PayloadSlicer;
import org.jclouds.io.Payloads;
import org.jclouds.io.payloads.BaseMutableContentMetadata;
+import org.jclouds.io.payloads.ByteSourcePayload;
import org.testng.annotations.Test;
import java.util.List;
@@ -39,7 +41,6 @@ import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import static org.testng.Assert.assertEquals;
@@ -50,26 +51,31 @@ public class AzureBlobBlockUploadStrategyTest {
public void testExecute() throws Exception {
String container = "test-container";
String blobName = "test-blob";
- long oneMB = 1048576L;
+ byte[] blobData = "ABCD".getBytes(Charsets.UTF_8);
AzureBlobClient client = createMock(AzureBlobClient.class);
PayloadSlicer slicer = createMock(PayloadSlicer.class);
MutableBlobMetadata metadata = new MutableBlobMetadataImpl();
MutableContentMetadata contentMetadata = new BaseMutableContentMetadata();
- contentMetadata.setContentLength(MultipartUploadStrategy.MAX_BLOCK_SIZE * 3 + oneMB);
+ contentMetadata.setContentLength((long)blobData.length);
metadata.setName(blobName);
metadata.setContentMetadata(contentMetadata);
Blob blob = new BlobImpl(metadata);
- ByteSource bytes = ByteSource.wrap("ABCD".getBytes(Charsets.UTF_8));
+ ByteSource bytes = ByteSource.wrap(blobData);
Payload payload = Payloads.newByteSourcePayload(bytes);
payload.setContentMetadata(contentMetadata);
blob.setPayload(payload);
- expect(slicer.slice(payload, 0, MultipartUploadStrategy.MAX_BLOCK_SIZE)).andReturn(payload);
- expect(slicer.slice(payload, MultipartUploadStrategy.MAX_BLOCK_SIZE, MultipartUploadStrategy.MAX_BLOCK_SIZE)).andReturn(payload);
- expect(slicer.slice(payload, MultipartUploadStrategy.MAX_BLOCK_SIZE * 2, MultipartUploadStrategy.MAX_BLOCK_SIZE)).andReturn(payload);
- expect(slicer.slice(payload, MultipartUploadStrategy.MAX_BLOCK_SIZE * 3, oneMB)).andReturn(payload);
- client.putBlock(eq(container), eq(blobName), anyObject(String.class), eq(payload));
- expectLastCall().times(4);
+ List<Payload> payloads = ImmutableList.of(
+ createBlockPayload(new byte[]{blobData[0]}),
+ createBlockPayload(new byte[]{blobData[1]}),
+ createBlockPayload(new byte[]{blobData[2]}),
+ createBlockPayload(new byte[]{blobData[3]}));
+
+ expect(slicer.slice(payload, MultipartUploadStrategy.MAX_BLOCK_SIZE)).andReturn(payloads);
+ client.putBlock(eq(container), eq(blobName), anyObject(String.class), eq(payloads.get(0)));
+ client.putBlock(eq(container), eq(blobName), anyObject(String.class), eq(payloads.get(1)));
+ client.putBlock(eq(container), eq(blobName), anyObject(String.class), eq(payloads.get(2)));
+ client.putBlock(eq(container), eq(blobName), anyObject(String.class), eq(payloads.get(3)));
expect(client.putBlockList(eq(container), eq(blobName), EasyMock.<List<String>>anyObject())).andReturn("Fake ETAG");
AzureBlobBlockUploadStrategy strat = new AzureBlobBlockUploadStrategy(client, slicer);
@@ -102,4 +108,12 @@ public class AzureBlobBlockUploadStrategyTest {
AzureBlobBlockUploadStrategy strat = new AzureBlobBlockUploadStrategy(client, slicer);
strat.execute(container, blob);
}
+
+ private Payload createBlockPayload(byte[] blockData) {
+ ByteSourcePayload payload = Payloads.newByteSourcePayload(ByteSource.wrap(blockData));
+ MutableContentMetadata contentMetadata = new BaseMutableContentMetadata();
+ contentMetadata.setContentLength((long) blockData.length);
+ payload.setContentMetadata(contentMetadata);
+ return payload;
+ }
}