You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by ga...@apache.org on 2017/11/07 18:56:03 UTC
[2/2] jclouds git commit: JCLOUDS-1281: Swift dynamic large objects
JCLOUDS-1281: Swift dynamic large objects
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/eaf3c779
Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/eaf3c779
Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/eaf3c779
Branch: refs/heads/master
Commit: eaf3c779dce63ddc808d5ff8fe3ed9aade10b2f2
Parents: b2ced53
Author: Archana Chinnaiah <ar...@oc5007853722.ibm.com>
Authored: Tue May 23 17:14:16 2017 +0530
Committer: Andrew Gaul <ga...@apache.org>
Committed: Tue Nov 7 10:50:10 2017 -0800
----------------------------------------------------------------------
.../jclouds/openstack/swift/v1/SwiftApi.java | 6 +
.../v1/features/DynamicLargeObjectApi.java | 97 ++++++++++++++
.../features/DynamicLargeObjectApiLiveTest.java | 126 +++++++++++++++++++
.../features/DynamicLargeObjectApiMockTest.java | 122 ++++++++++++++++++
4 files changed, 351 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds/blob/eaf3c779/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java
----------------------------------------------------------------------
diff --git a/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java b/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java
index 1d4d1d0..76a8e9f 100644
--- a/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java
+++ b/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java
@@ -27,6 +27,7 @@ import org.jclouds.location.functions.RegionToEndpoint;
import org.jclouds.openstack.swift.v1.features.AccountApi;
import org.jclouds.openstack.swift.v1.features.BulkApi;
import org.jclouds.openstack.swift.v1.features.ContainerApi;
+import org.jclouds.openstack.swift.v1.features.DynamicLargeObjectApi;
import org.jclouds.openstack.swift.v1.features.ObjectApi;
import org.jclouds.openstack.swift.v1.features.StaticLargeObjectApi;
import org.jclouds.rest.annotations.Delegate;
@@ -67,4 +68,9 @@ public interface SwiftApi extends Closeable {
@Path("/{containerName}")
StaticLargeObjectApi getStaticLargeObjectApi(@EndpointParam(parser = RegionToEndpoint.class) String region,
@PathParam("containerName") String containerName);
+
+ @Delegate
+ @Path("/{containerName}")
+ DynamicLargeObjectApi getDynamicLargeObjectApi(@EndpointParam(parser = RegionToEndpoint.class) String region,
+ @PathParam("containerName") String containerName);
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/eaf3c779/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApi.java
----------------------------------------------------------------------
diff --git a/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApi.java b/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApi.java
new file mode 100644
index 0000000..044e457
--- /dev/null
+++ b/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApi.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
+
+import java.util.Map;
+
+import javax.inject.Named;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+
+import org.jclouds.openstack.keystone.v2_0.filters.AuthenticateRequest;
+import org.jclouds.openstack.swift.v1.binders.BindMetadataToHeaders.BindObjectMetadataToHeaders;
+import org.jclouds.openstack.swift.v1.binders.BindToHeaders;
+import org.jclouds.openstack.swift.v1.domain.SwiftObject;
+import org.jclouds.openstack.swift.v1.functions.ETagHeader;
+import org.jclouds.rest.annotations.BinderParam;
+import org.jclouds.rest.annotations.Headers;
+import org.jclouds.rest.annotations.RequestFilters;
+import org.jclouds.rest.annotations.ResponseParser;
+
+import com.google.common.annotations.Beta;
+
+/**
+ * Provides access to the OpenStack Object Storage (Swift) Dynamic Large Object
+ * API features.
+ * <p/>
+ * This API is new to jclouds and hence is in Beta.
+ */
+@Beta
+@RequestFilters(AuthenticateRequest.class)
+@Consumes(APPLICATION_JSON)
+@Path("/{objectName}")
+public interface DynamicLargeObjectApi {
+ /**
+ * Creates or updates a dynamic large object's manifest.
+ *
+ * @param objectName
+ * corresponds to {@link SwiftObject#getName()}.
+ * @param metadata
+ * corresponds to {@link SwiftObject#getMetadata()}.
+ * @param headers
+ * Binds the map to headers, without prefixing/escaping the header
+ * name/key.
+ *
+ * @return {@link SwiftObject#getEtag()} of the object, which is the MD5
+ * checksum of the concatenated ETag values of the {@code segments}.
+ *
+ * @see {@code StaticLargeObjectApi}
+ */
+ @Deprecated
+ @Named("dynamicLargeObject:putManifest")
+ @PUT
+ @ResponseParser(ETagHeader.class)
+ @Headers(keys = "X-Object-Manifest", values = "{containerName}/{objectName}/")
+ String putManifest(@PathParam("objectName") String objectName,
+ @BinderParam(BindObjectMetadataToHeaders.class) Map<String, String> metadata,
+ @BinderParam(BindToHeaders.class) Map<String, String> headers);
+
+ /**
+ * Creates or updates a dynamic large object's manifest.
+ *
+ * @param objectName
+ * corresponds to {@link SwiftObject#getName()}.
+ * @param metadata
+ * corresponds to {@link SwiftObject#getMetadata()}.
+ *
+ * @return {@link SwiftObject#getEtag()} of the object, which is the etag
+ * of 0 sized object.
+ *
+ * @see {@code StaticLargeObjectApi}
+ */
+ @Deprecated
+ @Named("dynamicLargeObject:putManifest")
+ @PUT
+ @ResponseParser(ETagHeader.class)
+ @Headers(keys = "X-Object-Manifest", values = "{containerName}/{objectName}/")
+ String putManifest(@PathParam("objectName") String objectName,
+ @BinderParam(BindObjectMetadataToHeaders.class) Map<String, String> metadata);
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/eaf3c779/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiLiveTest.java
----------------------------------------------------------------------
diff --git a/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiLiveTest.java b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiLiveTest.java
new file mode 100644
index 0000000..aaf37d3
--- /dev/null
+++ b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiLiveTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.jclouds.io.Payloads.newByteSourcePayload;
+import static org.testng.Assert.assertNotNull;
+
+import java.io.IOException;
+
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.openstack.swift.v1.blobstore.RegionScopedBlobStoreContext;
+import org.jclouds.openstack.swift.v1.domain.ObjectList;
+import org.jclouds.openstack.swift.v1.domain.SwiftObject;
+import org.jclouds.openstack.swift.v1.internal.BaseSwiftApiLiveTest;
+import org.jclouds.openstack.swift.v1.options.ListContainerOptions;
+import org.jclouds.utils.TestUtils;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.ByteSource;
+
+@Test(groups = "live", testName = "DynamicLargeObjectApiLiveTest", singleThreaded = true)
+public class DynamicLargeObjectApiLiveTest extends BaseSwiftApiLiveTest {
+
+ private String defaultName = getClass().getSimpleName();
+ private static final ByteSource megOf1s = TestUtils.randomByteSource().slice(0, 1024 * 1024);
+ private static final ByteSource megOf2s = TestUtils.randomByteSource().slice(0, 1024 * 1024);
+ private String objectName = "myObject";
+
+ @Test
+ public void testReplaceManifest() throws Exception {
+ for (String regionId : regions) {
+ assertReplaceManifest(regionId, defaultName);
+ uploadLargeFile(regionId);
+ }
+ }
+
+ @SuppressWarnings("deprecation")
+ @Test
+ public void uploadLargeFile(String regionId) throws IOException, InterruptedException {
+ long total_size = 0;
+ RegionScopedBlobStoreContext ctx = RegionScopedBlobStoreContext.class.cast(view);
+ BlobStore blobStore = ctx.getBlobStore();
+ String defaultContainerName = getContainerName();
+ // configure the blobstore to use multipart uploading of the file
+ for (int partNumber = 0; partNumber < 3; partNumber++) {
+ String objName = String.format("%s/%s/%s", objectName, "dlo", partNumber);
+ String data = "data" + partNumber;
+ ByteSource payload = ByteSource.wrap(data.getBytes(Charsets.UTF_8));
+ Blob blob = blobStore.blobBuilder(objName)
+ .payload(payload)
+ .build();
+ String etag = blobStore.putBlob(defaultContainerName, blob);
+ assertNotNull(etag);
+ total_size += data.length();
+ }
+ getApi().getDynamicLargeObjectApi(regionId, defaultContainerName).putManifest(objectName,
+ ImmutableMap.of("myfoo", "Bar"));
+ SwiftObject bigObject = getApi().getObjectApi(regionId, defaultContainerName).get(objectName);
+ assertThat(bigObject.getETag()).isEqualTo("54bc1337d7a51660c40db39759cc1944");
+ assertThat(bigObject.getPayload().getContentMetadata().getContentLength()).isEqualTo(total_size);
+ assertThat(getApi().getContainerApi(regionId).get(defaultContainerName).getObjectCount()).isEqualTo(Long.valueOf(4));
+ }
+
+ @SuppressWarnings("deprecation")
+ protected void assertReplaceManifest(String regionId, String name) throws InterruptedException {
+ String containerName = getContainerName();
+ ObjectApi objectApi = getApi().getObjectApi(regionId, containerName);
+
+ String etag1s = objectApi.put(name + "/1", newByteSourcePayload(megOf1s));
+ awaitConsistency();
+ assertMegabyteAndETagMatches(regionId, containerName, name + "/1", etag1s);
+
+ String etag2s = objectApi.put(name + "/2", newByteSourcePayload(megOf2s));
+ awaitConsistency();
+ assertMegabyteAndETagMatches(regionId, containerName, name + "/2", etag2s);
+
+ awaitConsistency();
+ String etagOfEtags = getApi().getDynamicLargeObjectApi(regionId, containerName).putManifest(name,
+ ImmutableMap.of("myfoo", "Bar"));
+
+ assertNotNull(etagOfEtags);
+
+ awaitConsistency();
+
+ SwiftObject bigObject = getApi().getObjectApi(regionId, containerName).get(name);
+ assertThat(bigObject.getPayload().getContentMetadata().getContentLength()).isEqualTo(Long.valueOf(2 * 1024L * 1024L));
+ assertThat(bigObject.getMetadata()).isEqualTo(ImmutableMap.of("myfoo", "Bar"));
+ // segments are visible
+ assertThat(getApi().getContainerApi(regionId).get(containerName).getObjectCount()).isEqualTo(Long.valueOf(3));
+ }
+
+ protected void assertMegabyteAndETagMatches(String regionId, String containerName, String name, String etag1s) {
+ SwiftObject object1s = getApi().getObjectApi(regionId, containerName).get(name);
+ assertThat(object1s.getETag()).isEqualTo(etag1s);
+ assertThat(object1s.getPayload().getContentMetadata().getContentLength()).isEqualTo(Long.valueOf(1024L * 1024L));
+ }
+
+ protected void deleteAllObjectsInContainerDLO(String regionId, final String containerName) {
+ ObjectList objects = getApi().getObjectApi(regionId, containerName).list(new ListContainerOptions());
+ if (objects == null) {
+ return;
+ }
+ for (SwiftObject object : objects) {
+ String name = containerName + "/" + object.getName();
+ getApi().getObjectApi(regionId, containerName).delete(name);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/eaf3c779/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiMockTest.java
----------------------------------------------------------------------
diff --git a/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiMockTest.java b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiMockTest.java
new file mode 100644
index 0000000..c0420ee
--- /dev/null
+++ b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiMockTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jclouds.openstack.swift.v1.features;
+
+import static com.google.common.net.HttpHeaders.ETAG;
+import static org.jclouds.openstack.swift.v1.reference.SwiftHeaders.OBJECT_METADATA_PREFIX;
+import static org.testng.Assert.assertEquals;
+
+import org.jclouds.io.Payloads;
+import org.jclouds.openstack.swift.v1.SwiftApi;
+import org.jclouds.openstack.v2_0.internal.BaseOpenStackMockTest;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.net.HttpHeaders;
+import com.squareup.okhttp.mockwebserver.MockResponse;
+import com.squareup.okhttp.mockwebserver.MockWebServer;
+import com.squareup.okhttp.mockwebserver.RecordedRequest;
+
+@Test(groups = "unit", testName = "DynamicLargeObjectApiMockTest")
+public final class DynamicLargeObjectApiMockTest extends BaseOpenStackMockTest<SwiftApi> {
+
+ String containerName = "myContainer";
+ String objectName = "myObjectTest";
+
+ @SuppressWarnings("deprecation")
+ @Test
+ public void uploadLargeFile() throws Exception {
+ MockWebServer server = mockOpenStackServer();
+ server.enqueue(addCommonHeaders(new MockResponse().setBody(stringFromResource("/access.json"))));
+ server.enqueue(new MockResponse().setBody("").addHeader(ETAG, "89d903bc35dede724fd52c51437ff5fd"));
+ server.enqueue(new MockResponse().setBody("").addHeader(ETAG, "d41d8cd98f00b204e9800998ecf8427e"));
+ server.enqueue(addCommonHeaders(new MockResponse().addHeader("X-Object-Manifest", "myContainer/myObject")));
+
+ try {
+ SwiftApi api = api(server.getUrl("/").toString(), "openstack-swift");
+ assertEquals(api.getObjectApi("DFW", containerName).put(objectName.concat("1"), Payloads.newPayload("data1")),
+ "89d903bc35dede724fd52c51437ff5fd");
+ assertEquals(api.getDynamicLargeObjectApi("DFW", containerName).putManifest(objectName,
+ ImmutableMap.of("MyFoo", "Bar"), ImmutableMap.of("MyFoo", "Bar")), "d41d8cd98f00b204e9800998ecf8427e");
+
+ assertEquals(server.getRequestCount(), 3);
+ assertAuthentication(server);
+
+ RecordedRequest uploadRequest = server.takeRequest();
+ assertEquals(uploadRequest.getRequestLine(),
+ "PUT /v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer/myObjectTest1 HTTP/1.1");
+ assertEquals(new String(uploadRequest.getBody()), "data1");
+
+ RecordedRequest uploadRequestManifest = server.takeRequest();
+ assertRequest(uploadRequestManifest, "PUT",
+ "/v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer/myObjectTest");
+ assertEquals(uploadRequestManifest.getHeader(OBJECT_METADATA_PREFIX + "MyFoo"), "Bar");
+
+ } finally {
+ server.shutdown();
+ }
+ }
+
+ @SuppressWarnings("deprecation")
+ public void testReplaceManifest() throws Exception {
+ MockWebServer server = mockOpenStackServer();
+ server.enqueue(addCommonHeaders(new MockResponse().setBody(stringFromResource("/access.json"))));
+ server.enqueue(addCommonHeaders(new MockResponse().addHeader(HttpHeaders.ETAG, "\"abcd\"")));
+ server.enqueue(addCommonHeaders(new MockResponse().addHeader("X-Object-Manifest", "myContainer/myObject")));
+
+ try {
+ SwiftApi api = api(server.getUrl("/").toString(), "openstack-swift");
+ assertEquals(api.getDynamicLargeObjectApi("DFW", "myContainer").putManifest("myObject",
+ ImmutableMap.of("MyFoo", "Bar"), ImmutableMap.of("MyFoo", "Bar")), "abcd");
+
+ assertEquals(server.getRequestCount(), 2);
+ assertAuthentication(server);
+
+ RecordedRequest replaceRequest = server.takeRequest();
+ assertRequest(replaceRequest, "PUT",
+ "/v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer/myObject");
+ assertEquals(replaceRequest.getHeader(OBJECT_METADATA_PREFIX + "myfoo"), "Bar");
+ } finally {
+ server.shutdown();
+ }
+ }
+
+ @SuppressWarnings("deprecation")
+ public void testReplaceManifestUnicodeUTF8() throws Exception {
+ MockWebServer server = mockOpenStackServer();
+ server.enqueue(addCommonHeaders(new MockResponse().setBody(stringFromResource("/access.json"))));
+ server.enqueue(addCommonHeaders(new MockResponse().addHeader(HttpHeaders.ETAG, "\"abcd\"")));
+ server.enqueue(addCommonHeaders(new MockResponse().addHeader("X-Object-Manifest", "myContainer/myObject")));
+
+ try {
+ SwiftApi api = api(server.getUrl("/").toString(), "openstack-swift");
+ assertEquals(api.getDynamicLargeObjectApi("DFW", "myContainer").putManifest("unic₪de",
+ ImmutableMap.of("MyFoo", "Bar"), ImmutableMap.of("MyFoo", "Bar")), "abcd");
+
+ assertEquals(server.getRequestCount(), 2);
+ assertAuthentication(server);
+
+ RecordedRequest replaceRequest = server.takeRequest();
+ assertRequest(replaceRequest, "PUT",
+ "/v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer/unic%E2%82%AAde");
+ assertEquals(replaceRequest.getHeader(OBJECT_METADATA_PREFIX + "myfoo"), "Bar");
+ } finally {
+ server.shutdown();
+ }
+ }
+}