You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by ev...@apache.org on 2013/08/25 17:20:41 UTC
git commit: JCLOUDS-251: Swift: Delete chunks when deleting a
multipart blob
Updated Branches:
refs/heads/master d05e77b8b -> d60d2681d
JCLOUDS-251: Swift: Delete chunks when deleting a multipart blob
Also:
- Make SwiftBlobIntegrationLiveTest.testMultipartChunkedFileStream more realistic by uploading a file large enough to be split into parts.
- JavaDoc fixes for SwiftBlobStore: don't reference nonexistent methods.
Project: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/commit/d60d2681
Tree: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/tree/d60d2681
Diff: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/diff/d60d2681
Branch: refs/heads/master
Commit: d60d2681d1d7d7a38bbb65d5408672ee71b15727
Parents: d05e77b
Author: Francis Devereux <fr...@bright-interactive.co.uk>
Authored: Thu Aug 22 15:27:25 2013 +0100
Committer: Everett Toews <ev...@rackspace.com>
Committed: Sun Aug 25 10:20:22 2013 -0500
----------------------------------------------------------------------
.../CloudFilesBlobIntegrationLiveTest.java | 23 +++++++
.../swift/blobstore/SwiftBlobStore.java | 56 ++++++++++++++-
.../domain/MutableObjectInfoWithMetadata.java | 3 +
...DelegatingMutableObjectInfoWithMetadata.java | 10 +++
.../MutableObjectInfoWithMetadataImpl.java | 21 +++++-
.../functions/ParseObjectInfoFromHeaders.java | 2 +
.../swift/blobstore/SwiftBlobStoreTest.java | 45 +++++++++++++
.../SwiftBlobIntegrationLiveTest.java | 71 ++++++++++++++++----
.../internal/BaseBlobIntegrationTest.java | 7 ++
9 files changed, 220 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java
----------------------------------------------------------------------
diff --git a/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java b/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java
index a56e9f8..3d7f42b 100644
--- a/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java
+++ b/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java
@@ -16,10 +16,15 @@
*/
package org.jclouds.cloudfiles.blobstore.integration;
+import java.io.IOException;
+
+import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.openstack.swift.blobstore.integration.SwiftBlobIntegrationLiveTest;
import org.testng.annotations.Test;
+import static org.testng.Assert.assertEquals;
+
/**
*
* @author Adrian Cole
@@ -38,4 +43,22 @@ public class CloudFilesBlobIntegrationLiveTest extends SwiftBlobIntegrationLiveT
.getMetadata().getContentMetadata().getContentDisposition();
}
+ @Test(groups = { "integration", "live" })
+ public void testChunksAreDeletedWhenMultipartBlobIsDeleted() throws IOException, InterruptedException {
+ String containerName = getContainerName();
+ try {
+ BlobStore blobStore = view.getBlobStore();
+
+ long countBefore = blobStore.countBlobs(containerName);
+ String blobName = "deleteme.txt";
+ addMultipartBlobToContainer(containerName, blobName);
+
+ blobStore.removeBlob(containerName, blobName);
+ long countAfter = blobStore.countBlobs(containerName);
+
+ assertEquals(countAfter, countBefore);
+ } finally {
+ returnContainer(containerName);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
----------------------------------------------------------------------
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
index 8c0ac70..2940d6f 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
@@ -16,8 +16,10 @@
*/
package org.jclouds.openstack.swift.blobstore;
+import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.blobstore.util.BlobStoreUtils.createParentIfNeededAsync;
+import static org.jclouds.openstack.swift.options.ListContainerOptions.Builder.withPrefix;
import java.util.Set;
@@ -25,6 +27,7 @@ import javax.inject.Inject;
import javax.inject.Provider;
import javax.inject.Singleton;
+import com.google.common.annotations.VisibleForTesting;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobMetadata;
@@ -50,8 +53,11 @@ import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob;
import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata;
import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy;
import org.jclouds.openstack.swift.domain.ContainerMetadata;
+import org.jclouds.openstack.swift.domain.MutableObjectInfoWithMetadata;
+import org.jclouds.openstack.swift.domain.ObjectInfo;
import com.google.common.base.Function;
+import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.collect.Iterables;
@@ -118,7 +124,7 @@ public class SwiftBlobStore extends BaseBlobStore {
}
/**
- * This implementation invokes {@link CommonSwiftClient#putBucketInRegion}
+ * This implementation invokes {@link CommonSwiftClient#createContainer}
*
* @param location
* currently ignored
@@ -145,7 +151,7 @@ public class SwiftBlobStore extends BaseBlobStore {
}
/**
- * This implementation invokes {@link CommonSwiftClient#blobExists}
+ * This implementation invokes {@link CommonSwiftClient#objectExists}
*
* @param container
* container name
@@ -225,7 +231,53 @@ public class SwiftBlobStore extends BaseBlobStore {
*/
@Override
public void removeBlob(String container, String key) {
+ String objectManifest = getObjectManifestOrNull(container, key);
+
sync.removeObject(container, key);
+
+ if (!Strings.isNullOrEmpty(objectManifest)) {
+ removeObjectsWithPrefix(objectManifest);
+ }
+ }
+
+ private String getObjectManifestOrNull(String container, String key) {
+ MutableObjectInfoWithMetadata objectInfo = sync.getObjectInfo(container, key);
+ return objectInfo == null ? null : objectInfo.getObjectManifest();
+ }
+
+ private void removeObjectsWithPrefix(String containerAndPrefix) {
+ String[] parts = splitContainerAndKey(containerAndPrefix);
+
+ String container = parts[0];
+ String prefix = parts[1];
+
+ removeObjectsWithPrefix(container, prefix);
+ }
+
+ @VisibleForTesting
+ static String[] splitContainerAndKey(String containerAndKey) {
+ String[] parts = containerAndKey.split("/", 2);
+ checkArgument(parts.length == 2,
+ "No / separator found in \"%s\"",
+ containerAndKey);
+ return parts;
+ }
+
+ private void removeObjectsWithPrefix(String container, String prefix) {
+ String nextMarker = null;
+ do {
+ org.jclouds.openstack.swift.options.ListContainerOptions listContainerOptions =
+ withPrefix(prefix);
+ if (nextMarker != null) {
+ listContainerOptions = listContainerOptions.afterMarker(nextMarker);
+ }
+
+ PageSet<ObjectInfo> chunks = sync.listObjects(container, listContainerOptions);
+ for (ObjectInfo chunk : chunks) {
+ sync.removeObject(container, chunk.getName());
+ }
+ nextMarker = chunks.getNextMarker();
+ } while (nextMarker != null);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java
----------------------------------------------------------------------
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java
index ee326a4..103292f 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java
@@ -48,4 +48,7 @@ public interface MutableObjectInfoWithMetadata extends ObjectInfo {
Map<String, String> getMetadata();
+ String getObjectManifest();
+
+ void setObjectManifest(String objectManifest);
}
http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java
----------------------------------------------------------------------
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java
index 57d3444..9b57686 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java
@@ -147,4 +147,14 @@ public class DelegatingMutableObjectInfoWithMetadata extends BaseMutableContentM
public URI getUri() {
return delegate.getUri();
}
+
+ @Override
+ public void setObjectManifest(String objectManifest) {
+ delegate.setObjectManifest(objectManifest);
+ }
+
+ @Override
+ public String getObjectManifest() {
+ return delegate.getObjectManifest();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java
----------------------------------------------------------------------
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java
index dc09819..484888d 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java
@@ -41,6 +41,7 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM
private byte[] hash;
private String contentType = MediaType.APPLICATION_OCTET_STREAM;
private Date lastModified;
+ private String objectManifest;
private final Map<String, String> metadata = Maps.newLinkedHashMap();
/**
@@ -121,6 +122,7 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM
int result = 1;
result = prime * result + ((container == null) ? 0 : container.hashCode());
result = prime * result + ((name == null) ? 0 : name.hashCode());
+ result = prime * result + ((objectManifest == null) ? 0 : objectManifest.hashCode());
return result;
}
@@ -143,6 +145,11 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM
return false;
} else if (!name.equals(other.name))
return false;
+ if (objectManifest == null) {
+ if (other.objectManifest != null)
+ return false;
+ } else if (!objectManifest.equals(other.objectManifest))
+ return false;
return true;
}
@@ -197,9 +204,19 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM
}
@Override
+ public String getObjectManifest() {
+ return objectManifest;
+ }
+
+ @Override
+ public void setObjectManifest(String objectManifest) {
+ this.objectManifest = objectManifest;
+ }
+
+ @Override
public String toString() {
- return String.format("[name=%s, container=%s, uri=%s, bytes=%s, contentType=%s, lastModified=%s, hash=%s]", name,
- container, uri, bytes, contentType, lastModified, Arrays.toString(hash));
+ return String.format("[name=%s, container=%s, uri=%s, bytes=%s, contentType=%s, lastModified=%s, hash=%s, objectManifest=%s]",
+ name, container, uri, bytes, contentType, lastModified, Arrays.toString(hash), objectManifest);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java
----------------------------------------------------------------------
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java
index bd082fe..d9f0297 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java
@@ -65,6 +65,8 @@ public class ParseObjectInfoFromHeaders implements Function<HttpResponse, Mutabl
if (eTagHeader != null) {
to.setHash(ETagUtils.convertHexETagToByteArray(eTagHeader));
}
+ to.setObjectManifest(from.getFirstHeaderOrNull("X-Object-Manifest"));
+
return to;
}
http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStoreTest.java
----------------------------------------------------------------------
diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStoreTest.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStoreTest.java
new file mode 100644
index 0000000..439d7e0
--- /dev/null
+++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStoreTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.openstack.swift.blobstore;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+@Test(groups = "unit")
+public class SwiftBlobStoreTest {
+ @Test
+ public void testSplitContainerAndKey() {
+ String container = "test-container";
+ String key = "key/with/some/slashes/in/it/and/a/trailing/slash/";
+
+ String containerAndKey = container + "/" + key;
+
+ String[] split = SwiftBlobStore.splitContainerAndKey(containerAndKey);
+ String actualContainer = split[0];
+ String actualKey = split[1];
+
+ assertEquals(actualContainer, container);
+ assertEquals(actualKey, key);
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class,
+ expectedExceptionsMessageRegExp = "No / separator found in \"not-a-container-and-key\"")
+ public void testSplitContainerAndKeyWithNoSeparator() {
+ SwiftBlobStore.splitContainerAndKey("not-a-container-and-key");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
----------------------------------------------------------------------
diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
index 57e381a..365dc57 100644
--- a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
+++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
@@ -21,11 +21,13 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
+import com.google.common.io.ByteStreams;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties;
+import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload;
import org.testng.ITestContext;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
@@ -34,6 +36,9 @@ import org.testng.annotations.Test;
import com.google.common.io.Files;
import com.google.common.io.InputSupplier;
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertTrue;
+
/**
*
* @author James Murty
@@ -41,15 +46,21 @@ import com.google.common.io.InputSupplier;
*/
@Test(groups = "live")
public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
+ /**
+ * Use the minimum part size to minimise the file size that we have to
+ * upload to get a multipart blob thereby make the test run faster
+ */
+ private static final long PART_SIZE = MultipartUpload.MIN_PART_SIZE;
+
@Override
protected Properties setupProperties() {
Properties props = super.setupProperties();
setIfTestSystemPropertyPresent(props, KeystoneProperties.CREDENTIAL_TYPE);
+ props.setProperty("jclouds.mpu.parts.size", String.valueOf(PART_SIZE));
return props;
}
private InputSupplier<InputStream> oneHundredOneConstitutions;
- private byte[] oneHundredOneConstitutionsMD5;
public SwiftBlobIntegrationLiveTest() {
provider = System.getProperty("test.swift.provider", "swift");
@@ -66,7 +77,6 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext);
oneHundredOneConstitutions = getTestDataSupplier();
- oneHundredOneConstitutionsMD5 = md5Supplier(oneHundredOneConstitutions);
}
@Override
@@ -91,18 +101,51 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
{ "asteri*k" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } };
}
+ @Test(groups = { "integration", "live" })
public void testMultipartChunkedFileStream() throws IOException, InterruptedException {
- Files.copy(oneHundredOneConstitutions, new File("target/const.txt"));
- String containerName = getContainerName();
-
- try {
- BlobStore blobStore = view.getBlobStore();
- blobStore.createContainerInLocation(null, containerName);
- Blob blob = blobStore.blobBuilder("const.txt")
- .payload(new File("target/const.txt")).contentMD5(oneHundredOneConstitutionsMD5).build();
- blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart());
- } finally {
- returnContainer(containerName);
- }
+ String containerName = getContainerName();
+ try {
+ BlobStore blobStore = view.getBlobStore();
+ long countBefore = blobStore.countBlobs(containerName);
+
+ addMultipartBlobToContainer(containerName, "const.txt");
+
+ long countAfter = blobStore.countBlobs(containerName);
+ assertNotEquals(countBefore, countAfter,
+ "No blob was created");
+ assertTrue(countAfter - countBefore > 1,
+ "A multipart blob wasn't actually created - " +
+ "there was only 1 extra blob but there should be one manifest blob and multiple chunk blobs");
+ } finally {
+ returnContainer(containerName);
+ }
+ }
+
+ protected void addMultipartBlobToContainer(String containerName, String key) throws IOException {
+ File fileToUpload = createFileBiggerThan(PART_SIZE);
+
+ BlobStore blobStore = view.getBlobStore();
+ blobStore.createContainerInLocation(null, containerName);
+ Blob blob = blobStore.blobBuilder(key)
+ .payload(fileToUpload)
+ .build();
+ blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart());
+ }
+
+ @SuppressWarnings("unchecked")
+ private File createFileBiggerThan(long partSize) throws IOException {
+ long copiesNeeded = (partSize / getOneHundredOneConstitutionsLength()) + 1;
+
+ InputSupplier<InputStream> temp = ByteStreams.join(oneHundredOneConstitutions);
+
+ for (int i = 0; i < copiesNeeded; i++) {
+ temp = ByteStreams.join(temp, oneHundredOneConstitutions);
+ }
+
+ File fileToUpload = new File("target/lots-of-const.txt");
+ Files.copy(temp, fileToUpload);
+
+ assertTrue(fileToUpload.length() > partSize);
+ return fileToUpload;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
----------------------------------------------------------------------
diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
index fceeb85..19ec3f5 100644
--- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
+++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
@@ -119,6 +119,13 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
return temp;
}
+ public static long getOneHundredOneConstitutionsLength() throws IOException {
+ if (oneHundredOneConstitutionsLength == 0) {
+ getTestDataSupplier();
+ }
+ return oneHundredOneConstitutionsLength;
+ }
+
/**
* Attempt to capture the issue detailed in
* http://groups.google.com/group/jclouds/browse_thread/thread/4a7c8d58530b287f