You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by ga...@apache.org on 2014/07/04 10:11:05 UTC
git commit: JCLOUDS-619: Introduce MultipartNamingStrategy to
generate part names correctly.
Repository: jclouds
Updated Branches:
refs/heads/master 05c37c2c7 -> a39eadce5
JCLOUDS-619: Introduce MultipartNamingStrategy to generate part names correctly.
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/a39eadce
Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/a39eadce
Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/a39eadce
Branch: refs/heads/master
Commit: a39eadce50e3bc37abc0e47564ef1698529bec81
Parents: 05c37c2
Author: Markus von Rüden <mv...@opennms.com>
Authored: Tue Jul 1 14:24:50 2014 +0200
Committer: Andrew Gaul <ga...@apache.org>
Committed: Fri Jul 4 01:06:46 2014 -0700
----------------------------------------------------------------------
.../internal/MultipartNamingStrategy.java | 30 ++++++++
.../SequentialMultipartUploadStrategy.java | 11 +--
.../SwiftBlobIntegrationLiveTest.java | 73 ++++++++++++++++----
.../internal/MultipartNamingStrategyTest.java | 71 +++++++++++++++++++
4 files changed, 169 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds/blob/a39eadce/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategy.java
----------------------------------------------------------------------
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategy.java
new file mode 100644
index 0000000..7ffebc5
--- /dev/null
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategy.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.openstack.swift.blobstore.strategy.internal;
+
+import javax.inject.Singleton;
+
+@Singleton
+public class MultipartNamingStrategy {
+
+ private static final String PART_SEPARATOR = "/";
+
+ protected String getPartName(String key, int partNumber, int totalParts) {
+ int base = (int) Math.log10(totalParts) + 1;
+ return String.format("%s%s%0" + base + "d", key, PART_SEPARATOR, partNumber);
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/a39eadce/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
----------------------------------------------------------------------
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
index 1874ccd..5f21ddc 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
@@ -34,7 +34,6 @@ import org.jclouds.openstack.swift.blobstore.functions.BlobToObject;
import com.google.inject.Inject;
public class SequentialMultipartUploadStrategy implements MultipartUploadStrategy {
- private static final String PART_SEPARATOR = "/";
@Resource
@Named(BlobStoreConstants.BLOBSTORE_LOGGER)
@@ -45,15 +44,18 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
private final BlobToObject blob2Object;
private final MultipartUploadSlicingAlgorithm algorithm;
private final PayloadSlicer slicer;
+ private final MultipartNamingStrategy namingStrategy;
@Inject
public SequentialMultipartUploadStrategy(CommonSwiftClient client, Provider<BlobBuilder> blobBuilders,
- BlobToObject blob2Object, MultipartUploadSlicingAlgorithm algorithm, PayloadSlicer slicer) {
+ BlobToObject blob2Object, MultipartUploadSlicingAlgorithm algorithm, PayloadSlicer slicer,
+ MultipartNamingStrategy namingStrategy) {
this.client = checkNotNull(client, "client");
this.blobBuilders = checkNotNull(blobBuilders, "blobBuilders");
this.blob2Object = checkNotNull(blob2Object, "blob2Object");
this.algorithm = checkNotNull(algorithm, "algorithm");
this.slicer = checkNotNull(slicer, "slicer");
+ this.namingStrategy = checkNotNull(namingStrategy, "namingStrategy");
}
@Override
@@ -68,10 +70,11 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
if (partCount > 0) {
for (Payload part : slicer.slice(payload, chunkSize)) {
int partNum = algorithm.getNextPart();
+ String partName = namingStrategy.getPartName(key, partNum, partCount);
Blob blobPart = blobBuilders.get()
- .name(key + PART_SEPARATOR + partNum)
+ .name(partName)
.payload(part)
- .contentDisposition(key + PART_SEPARATOR + partNum)
+ .contentDisposition(partName)
.build();
client.putObject(container, blob2Object.apply(blobPart));
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/a39eadce/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
----------------------------------------------------------------------
diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
index fca7145..ecd2370 100644
--- a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
+++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
@@ -25,11 +25,14 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
+import java.util.Random;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.io.ByteSources;
+import org.jclouds.io.Payload;
import org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties;
import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload;
import org.testng.ITestContext;
@@ -76,18 +79,18 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
throw new SkipException("not yet implemented");
}
- @BeforeClass(groups = { "integration", "live" }, dependsOnMethods = "setupContext")
- @Override
- public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
- super.setUpResourcesOnThisThread(testContext);
- oneHundredOneConstitutions = getTestDataSupplier();
- }
+ @BeforeClass(groups = {"integration", "live"}, dependsOnMethods = "setupContext")
+ @Override
+ public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
+ super.setUpResourcesOnThisThread(testContext);
+ oneHundredOneConstitutions = getTestDataSupplier();
+ }
@Override
protected void checkContentDisposition(Blob blob, String contentDisposition) {
- // This works for Swift Server 1.4.4/SWauth 1.0.3 but was null in previous versions.
- // TODO: Better testing for the different versions.
- super.checkContentDisposition(blob, contentDisposition);
+ // This works for Swift Server 1.4.4/SWauth 1.0.3 but was null in previous versions.
+ // TODO: Better testing for the different versions.
+ super.checkContentDisposition(blob, contentDisposition);
}
// not supported in swift
@@ -121,7 +124,40 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
"A multipart blob wasn't actually created - " +
"there was only 1 extra blob but there should be one manifest blob and multiple chunk blobs");
} finally {
- returnContainer(containerName);
+ returnContainer(containerName);
+ }
+ }
+
+ /**
+ * Checks that when there are more than 9 chunks the object names
+ * are set correctly so that the order of the object names matches
+ * the upload order.
+ * See issue https://issues.apache.org/jira/browse/JCLOUDS-619
+ */
+ @Test(groups = {"integration", "live"})
+ public void testMultipartChunkedFilenames() throws InterruptedException, IOException {
+ String containerName = getContainerName();
+ try {
+ BlobStore blobStore = view.getBlobStore();
+ String objectName = "object.txt";
+ long countBefore = blobStore.countBlobs(containerName);
+
+ // we want 11 parts
+ ByteSource inputSource = createByteSource(PART_SIZE * 11);
+ addMultipartBlobToContainer(containerName, objectName, inputSource);
+
+ // did we create enough parts?
+ long countAfter = blobStore.countBlobs(containerName);
+ assertNotEquals(countAfter, countBefore, "No blob was created");
+ assertEquals(countAfter, countBefore + 12,
+ "12 parts (11 objects + 1 manifest) were expected.");
+
+ // download and check if correct
+ Blob read = blobStore.getBlob(containerName, objectName);
+ Payload readPayload = read.getPayload();
+ assertTrue(inputSource.contentEquals(ByteSources.asByteSource(readPayload.openStream())));
+ } finally {
+ returnContainer(containerName);
}
}
@@ -158,15 +194,28 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
protected void addMultipartBlobToContainer(String containerName, String key) throws IOException {
File fileToUpload = createFileBiggerThan(PART_SIZE);
+ addMultipartBlobToContainer(containerName, key, Files.asByteSource(fileToUpload));
+ }
+ protected void addMultipartBlobToContainer(String containerName, String key, ByteSource byteSource) throws IOException {
BlobStore blobStore = view.getBlobStore();
blobStore.createContainerInLocation(null, containerName);
Blob blob = blobStore.blobBuilder(key)
- .payload(fileToUpload)
- .build();
+ .payload(byteSource)
+ .contentLength(byteSource.size())
+ .build();
blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart());
}
+ private ByteSource createByteSource(long size) throws IOException {
+ final Random random = new Random();
+ final byte[] randomBytes = new byte[(int) MultipartUpload.MIN_PART_SIZE];
+ random.nextBytes(randomBytes);
+ ByteSource byteSource = ByteSources.repeatingArrayByteSource(randomBytes).slice(0, size);
+ assertEquals(byteSource.size(), size);
+ return byteSource;
+ }
+
@SuppressWarnings("unchecked")
private File createFileBiggerThan(long partSize) throws IOException {
long copiesNeeded = (partSize / getOneHundredOneConstitutionsLength()) + 1;
http://git-wip-us.apache.org/repos/asf/jclouds/blob/a39eadce/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategyTest.java
----------------------------------------------------------------------
diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategyTest.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategyTest.java
new file mode 100644
index 0000000..364c410
--- /dev/null
+++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategyTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.openstack.swift.blobstore.strategy.internal;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+@Test(testName = "MultipartNamingStrategyTest")
+public class MultipartNamingStrategyTest {
+
+ @Test
+ public void testGetPartNameFirstOneHundred() {
+ final MultipartNamingStrategy strategy = new MultipartNamingStrategy();
+ final String key = "file.txt";
+ final int numberParts = 100;
+
+ // check the first 100
+ for (int i = 0; i < numberParts; i++) {
+ String partName = strategy.getPartName(key, i + 1, numberParts);
+ assertEquals(String.format("file.txt/%03d", i + 1), partName);
+ }
+ }
+
+ @Test
+ public void testGetPartNameChoices() {
+ final MultipartNamingStrategy strategy = new MultipartNamingStrategy();
+ final String key = "file.txt";
+
+ // check less than 10 parts
+ assertEquals(strategy.getPartName(key, 1, 5), "file.txt/1");
+ assertEquals(strategy.getPartName(key, 2, 5), "file.txt/2");
+ assertEquals(strategy.getPartName(key, 5, 5), "file.txt/5");
+
+ // check <= 10 parts
+ assertEquals(strategy.getPartName(key, 1, 10), "file.txt/01");
+ assertEquals(strategy.getPartName(key, 2, 10), "file.txt/02");
+ assertEquals(strategy.getPartName(key, 10, 10), "file.txt/10");
+
+ // check <= 100 parts
+ assertEquals(strategy.getPartName(key, 1, 100), "file.txt/001");
+ assertEquals(strategy.getPartName(key, 9, 100), "file.txt/009");
+ assertEquals(strategy.getPartName(key, 10, 100), "file.txt/010");
+ assertEquals(strategy.getPartName(key, 99, 100), "file.txt/099");
+ assertEquals(strategy.getPartName(key, 100, 100), "file.txt/100");
+
+ // check <= 5000 parts
+ assertEquals(strategy.getPartName(key, 1, 5000), "file.txt/0001");
+ assertEquals(strategy.getPartName(key, 10, 5000), "file.txt/0010");
+ assertEquals(strategy.getPartName(key, 99, 5000), "file.txt/0099");
+ assertEquals(strategy.getPartName(key, 100, 5000), "file.txt/0100");
+ assertEquals(strategy.getPartName(key, 999, 5000), "file.txt/0999");
+ assertEquals(strategy.getPartName(key, 4999, 5000), "file.txt/4999");
+ assertEquals(strategy.getPartName(key, 5000, 500), "file.txt/5000");
+ }
+}
+