You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by he...@apache.org on 2014/11/15 01:05:22 UTC
[13/21] incubator-brooklyn git commit: fix use of jclouds blob store
wrapper when "container" contains a /, with test,
fixing errors with making backups in SL
fix use of jclouds blob store wrapper when "container" contains a /, with test, fixing errors with making backups in SL
Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/5060c4f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/5060c4f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/5060c4f0
Branch: refs/heads/master
Commit: 5060c4f06e80a1b7b8062852a276a8d32bd12237
Parents: a6891c6
Author: Alex Heneveld <al...@cloudsoftcorp.com>
Authored: Thu Nov 13 21:42:36 2014 +0000
Committer: Alex Heneveld <al...@cloudsoftcorp.com>
Committed: Thu Nov 13 23:38:55 2014 +0000
----------------------------------------------------------------------
.../JcloudsBlobStoreBasedObjectStore.java | 60 +++++++++++----
.../JcloudsObjectStoreAccessorWriterTest.java | 81 +++++++++++++++++++-
2 files changed, 125 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/5060c4f0/locations/jclouds/src/main/java/brooklyn/entity/rebind/persister/jclouds/JcloudsBlobStoreBasedObjectStore.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/brooklyn/entity/rebind/persister/jclouds/JcloudsBlobStoreBasedObjectStore.java b/locations/jclouds/src/main/java/brooklyn/entity/rebind/persister/jclouds/JcloudsBlobStoreBasedObjectStore.java
index 1e9acf8..93759d6 100644
--- a/locations/jclouds/src/main/java/brooklyn/entity/rebind/persister/jclouds/JcloudsBlobStoreBasedObjectStore.java
+++ b/locations/jclouds/src/main/java/brooklyn/entity/rebind/persister/jclouds/JcloudsBlobStoreBasedObjectStore.java
@@ -40,6 +40,7 @@ import brooklyn.location.jclouds.JcloudsUtil;
import brooklyn.management.ManagementContext;
import brooklyn.management.ha.HighAvailabilityMode;
import brooklyn.util.exceptions.FatalConfigurationRuntimeException;
+import brooklyn.util.text.Strings;
import com.google.common.base.Function;
import com.google.common.base.Objects;
@@ -53,7 +54,8 @@ public class JcloudsBlobStoreBasedObjectStore implements PersistenceObjectStore
private static final Logger log = LoggerFactory.getLogger(JcloudsBlobStoreBasedObjectStore.class);
- private final String containerName;
+ private final String containerNameFirstPart;
+ private final String containerSubPath;
private String locationSpec;
private JcloudsLocation location;
@@ -63,17 +65,28 @@ public class JcloudsBlobStoreBasedObjectStore implements PersistenceObjectStore
public JcloudsBlobStoreBasedObjectStore(String locationSpec, String containerName) {
this.locationSpec = locationSpec;
- this.containerName = containerName;
+ String[] segments = splitOnce(containerName);
+ this.containerNameFirstPart = segments[0];
+ this.containerSubPath = segments[1];
}
+ private String[] splitOnce(String path) {
+ String separator = subPathSeparator();
+ int index = path.indexOf(separator);
+ if (index<0) return new String[] { path, "" };
+ return new String[] { path.substring(0, index), path.substring(index+separator.length()) };
+ }
+
public JcloudsBlobStoreBasedObjectStore(JcloudsLocation location, String containerName) {
this.location = location;
- this.containerName = containerName;
+ String[] segments = splitOnce(containerName);
+ this.containerNameFirstPart = segments[0];
+ this.containerSubPath = segments[1];
getBlobStoreContext();
}
public String getSummaryName() {
- return (locationSpec!=null ? locationSpec : location)+":"+getContainerName();
+ return (locationSpec!=null ? locationSpec : location)+":"+getContainerNameFull();
}
public synchronized BlobStoreContext getBlobStoreContext() {
@@ -94,25 +107,40 @@ public class JcloudsBlobStoreBasedObjectStore implements PersistenceObjectStore
// TODO do we need to get location from region? can't see the jclouds API.
// doesn't matter in some places because it's already in the endpoint
// String region = location.getConfig(CloudLocationConfig.CLOUD_REGION_ID);
- context.getBlobStore().createContainerInLocation(null, getContainerName());
+ context.getBlobStore().createContainerInLocation(null, getContainerNameFirstPart());
}
return context;
}
@Override
public void prepareForMasterUse() {
- // TODO currently backups not supported here, that is all which is needed for master use
- // (we have already thrown in prepareForSharedUse if backups have been specified as required)
+ // backups not supported here, that is all which is needed for master use
+ // that's now normally done *prior* to calling in to here for writes
+ // (and we have already thrown in prepareForSharedUse if legacy backups have been specified as required)
}
public String getContainerName() {
- return containerName;
+ return getContainerNameFull();
+ }
+
+ protected String getContainerNameFull() {
+ return mergePaths(containerNameFirstPart, containerSubPath);
+ }
+
+ protected String getContainerNameFirstPart() {
+ return containerNameFirstPart;
}
+ protected String getItemInContainerSubPath(String path) {
+ if (Strings.isBlank(containerSubPath)) return path;
+ return mergePaths(containerSubPath, path);
+ }
+
@Override
public void createSubPath(String subPath) {
- // not needed, and buggy on softlayer w swift w jclouds 1.7.2:
- // throws a "not found" if we're creating an empty directory from scratch
+ // not needed - subpaths are created on demant
+ // (and buggy on softlayer w swift w jclouds 1.7.2:
+ // throws a "not found" if we're creating an empty directory from scratch)
// context.getBlobStore().createDirectory(getContainerName(), subPath);
}
@@ -124,7 +152,7 @@ public class JcloudsBlobStoreBasedObjectStore implements PersistenceObjectStore
@Override
public StoreObjectAccessor newAccessor(String path) {
checkPrepared();
- return new JcloudsStoreObjectAccessor(context.getBlobStore(), getContainerName(), path);
+ return new JcloudsStoreObjectAccessor(context.getBlobStore(), getContainerNameFirstPart(), getItemInContainerSubPath(path));
}
protected String mergePaths(String basePath, String ...subPaths) {
@@ -146,7 +174,8 @@ public class JcloudsBlobStoreBasedObjectStore implements PersistenceObjectStore
@Override
public List<String> listContentsWithSubPath(final String parentSubPath) {
checkPrepared();
- return FluentIterable.from(context.getBlobStore().list(getContainerName(), ListContainerOptions.Builder.inDirectory(parentSubPath)))
+ return FluentIterable.from(context.getBlobStore().list(getContainerNameFirstPart(),
+ ListContainerOptions.Builder.inDirectory(getItemInContainerSubPath(parentSubPath))))
.transform(new Function<StorageMetadata, String>() {
@Override
public String apply(@javax.annotation.Nullable StorageMetadata input) {
@@ -165,7 +194,7 @@ public class JcloudsBlobStoreBasedObjectStore implements PersistenceObjectStore
public String toString() {
return Objects.toStringHelper(this)
.add("blobStoreContext", context)
- .add("basedir", containerName)
+ .add("basedir", containerNameFirstPart)
.toString();
}
@@ -196,7 +225,10 @@ public class JcloudsBlobStoreBasedObjectStore implements PersistenceObjectStore
@Override
public void deleteCompletely() {
- getBlobStoreContext().getBlobStore().deleteContainer(containerName);
+ if (Strings.isBlank(containerSubPath))
+ getBlobStoreContext().getBlobStore().deleteContainer(containerNameFirstPart);
+ else
+ newAccessor(containerSubPath).delete();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/5060c4f0/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/JcloudsObjectStoreAccessorWriterTest.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/JcloudsObjectStoreAccessorWriterTest.java b/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/JcloudsObjectStoreAccessorWriterTest.java
index 8a03501..ef62dee 100644
--- a/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/JcloudsObjectStoreAccessorWriterTest.java
+++ b/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/JcloudsObjectStoreAccessorWriterTest.java
@@ -20,6 +20,8 @@ package brooklyn.entity.rebind.persister.jclouds;
import java.io.IOException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -32,12 +34,16 @@ import brooklyn.entity.rebind.persister.PersistenceStoreObjectAccessorWriterTest
import brooklyn.entity.rebind.persister.StoreObjectAccessorLocking;
import brooklyn.management.ha.HighAvailabilityMode;
import brooklyn.test.entity.LocalManagementContextForTests;
+import brooklyn.util.exceptions.Exceptions;
+import brooklyn.util.net.Urls;
import brooklyn.util.text.Identifiers;
import brooklyn.util.time.Duration;
@Test(groups={"Live", "Live-sanity"})
public class JcloudsObjectStoreAccessorWriterTest extends PersistenceStoreObjectAccessorWriterTestFixture {
+ private static final Logger log = LoggerFactory.getLogger(JcloudsObjectStoreAccessorWriterTest.class);
+
private JcloudsBlobStoreBasedObjectStore store;
private LocalManagementContextForTests mgmt;
@@ -58,7 +64,10 @@ public class JcloudsObjectStoreAccessorWriterTest extends PersistenceStoreObject
}
protected StoreObjectAccessorWithLock newPersistenceStoreObjectAccessor() throws IOException {
- return new StoreObjectAccessorLocking(store.newAccessor("sample-file-"+Identifiers.makeRandomId(4)));
+ return newPersistenceStoreObjectAccessor(store, "");
+ }
+ protected StoreObjectAccessorWithLock newPersistenceStoreObjectAccessor(JcloudsBlobStoreBasedObjectStore aStore, String prefix) throws IOException {
+ return new StoreObjectAccessorLocking(aStore.newAccessor(prefix+"sample-file-"+Identifiers.makeRandomId(4)));
}
@Override
@@ -72,7 +81,75 @@ public class JcloudsObjectStoreAccessorWriterTest extends PersistenceStoreObject
// bit smaller since it's actually uploading here!
return 10000;
}
-
+
+ /** Tests what happen when we ask the store to be in a container with a path, e.g. path1/path2
+ * and then the accessor to a file within that (path3/file) --
+ * this does it an emulated way, where the store tracks the subpath so we don't have to */
+ @Test(groups={"Live"})
+ public void testNestedPath1() throws IOException {
+ mgmt = new LocalManagementContextForTests(BrooklynProperties.Factory.newDefault());
+ String path1 = BlobStoreTest.CONTAINER_PREFIX+"-"+Identifiers.makeRandomId(4);
+ String path2 = BlobStoreTest.CONTAINER_PREFIX+"-"+Identifiers.makeRandomId(4);
+ String path3 = BlobStoreTest.CONTAINER_PREFIX+"-"+Identifiers.makeRandomId(4);
+ JcloudsBlobStoreBasedObjectStore store0 = null;
+ try {
+ store0 = new JcloudsBlobStoreBasedObjectStore(BlobStoreTest.PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC, Urls.mergePaths(path1, path2));
+ store0.injectManagementContext(mgmt);
+ store0.prepareForSharedUse(PersistMode.CLEAN, HighAvailabilityMode.DISABLED);
+
+ newPersistenceStoreObjectAccessor(store0, path3+"/").put("hello world");
+ } catch (Exception e) {
+ log.warn("Failed with: "+e, e);
+ throw Exceptions.propagate(e);
+
+ } finally {
+ store0.deleteCompletely();
+
+ JcloudsBlobStoreBasedObjectStore storeD = new JcloudsBlobStoreBasedObjectStore(BlobStoreTest.PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC, path1);
+ storeD.injectManagementContext(mgmt);
+ storeD.prepareForSharedUse(PersistMode.CLEAN, HighAvailabilityMode.DISABLED);
+ storeD.deleteCompletely();
+ }
+ }
+
+ /** Tests what happen when we ask the store to be in a container with a path, e.g. path1/path2
+ * and then the accessor to a file within that (path3/file) --
+ * this does it the "official" way, where we ask for the store's container
+ * to be the first path segment */
+ @Test(groups={"Live"})
+ public void testNestedPath2() throws IOException {
+ mgmt = new LocalManagementContextForTests(BrooklynProperties.Factory.newDefault());
+ String path1 = BlobStoreTest.CONTAINER_PREFIX+"-"+Identifiers.makeRandomId(4);
+ String path2 = BlobStoreTest.CONTAINER_PREFIX+"-"+Identifiers.makeRandomId(4);
+ String path3 = BlobStoreTest.CONTAINER_PREFIX+"-"+Identifiers.makeRandomId(4);
+ JcloudsBlobStoreBasedObjectStore store1 = null, store2 = null;
+ try {
+ store1 = new JcloudsBlobStoreBasedObjectStore(BlobStoreTest.PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC,
+ path1);
+ store1.injectManagementContext(mgmt);
+ store1.prepareForSharedUse(PersistMode.CLEAN, HighAvailabilityMode.DISABLED);
+ store1.createSubPath(path2);
+ newPersistenceStoreObjectAccessor(store1, path2+"/"+path3+"/").put("hello world");
+
+ store2 = new JcloudsBlobStoreBasedObjectStore(BlobStoreTest.PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC,
+ Urls.mergePaths(path1, path2));
+ store2.injectManagementContext(mgmt);
+ store2.prepareForSharedUse(PersistMode.CLEAN, HighAvailabilityMode.DISABLED);
+
+ newPersistenceStoreObjectAccessor(store2, path3+"/").put("hello world");
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ // this doesn't work
+// store2.deleteCompletely();
+ // this is how you have to do it:
+ store1.newAccessor(path2).delete();
+
+ store1.deleteCompletely();
+ }
+ }
+
@Test(groups={"Live", "Live-sanity"})
@Override
public void testWriteBacklogThenDeleteWillLeaveFileDeleted() throws Exception {