You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by ev...@apache.org on 2013/07/15 16:02:56 UTC

git commit: Concurrent tests with better retries

Updated Branches:
  refs/heads/master 80c730dcf -> 40a05b9e5


Concurrent tests with better retries


Project: http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/commit/40a05b9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/tree/40a05b9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/diff/40a05b9e

Branch: refs/heads/master
Commit: 40a05b9e56813d9dcc125eefbb02715ef2b6caea
Parents: 80c730d
Author: zack-shoylev <za...@rackspace.com>
Authored: Fri Jun 28 17:47:05 2013 -0500
Committer: Everett Toews <ev...@rackspace.com>
Committed: Mon Jul 15 09:00:19 2013 -0500

----------------------------------------------------------------------
 openstack-trove/pom.xml                         |   3 +
 .../trove/v1/features/DatabaseApiLiveTest.java  |  20 ++--
 .../trove/v1/features/InstanceApiLiveTest.java  |  15 +--
 .../trove/v1/features/UserApiLiveTest.java      |  22 ++--
 .../v1/internal/BaseTroveApiExpectTest.java     |   2 +-
 .../openstack/trove/v1/internal/TroveUtils.java | 102 +++++++++++++++++++
 .../trove/v1/internal/TroveUtilsExpectTest.java |  66 ++++++++++++
 .../resources/instance_get_bad_instance.json    |  37 +++++++
 rackspace-clouddatabases-uk/pom.xml             |   4 +-
 rackspace-clouddatabases-us/pom.xml             |   4 +-
 10 files changed, 241 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/40a05b9e/openstack-trove/pom.xml
----------------------------------------------------------------------
diff --git a/openstack-trove/pom.xml b/openstack-trove/pom.xml
index 6b5f4db..001ded1 100644
--- a/openstack-trove/pom.xml
+++ b/openstack-trove/pom.xml
@@ -106,6 +106,9 @@
                   <goal>test</goal>
                 </goals>
                 <configuration>
+                  <forkCount>5</forkCount>
+                  <reuseForks>true</reuseForks>
+                  <parallel>classes</parallel>
                   <systemPropertyVariables>
                     <test.openstack-trove.endpoint>${test.openstack-trove.endpoint}</test.openstack-trove.endpoint>
                     <test.openstack-trove.api-version>${test.openstack-trove.api-version}</test.openstack-trove.api-version>

http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/40a05b9e/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/DatabaseApiLiveTest.java
----------------------------------------------------------------------
diff --git a/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/DatabaseApiLiveTest.java b/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/DatabaseApiLiveTest.java
index 5859490..0bcd6a4 100644
--- a/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/DatabaseApiLiveTest.java
+++ b/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/DatabaseApiLiveTest.java
@@ -18,14 +18,13 @@ package org.jclouds.openstack.trove.v1.features;
 
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.assertEquals;
 
 import java.util.List;
 import java.util.Map;
 
 import org.jclouds.openstack.trove.v1.domain.Instance;
 import org.jclouds.openstack.trove.v1.internal.BaseTroveApiLiveTest;
-import org.jclouds.openstack.trove.v1.predicates.InstancePredicates;
+import org.jclouds.openstack.trove.v1.internal.TroveUtils;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -36,7 +35,7 @@ import com.google.common.collect.Maps;
 /**
  * @author Zack Shoylev
  */
-@Test(groups = "live", testName = "DatabaseApiLiveTest", singleThreaded = true)
+@Test(groups = "live", testName = "DatabaseApiLiveTest")
 public class DatabaseApiLiveTest extends BaseTroveApiLiveTest {
 
    // zone to instance
@@ -47,16 +46,14 @@ public class DatabaseApiLiveTest extends BaseTroveApiLiveTest {
    @BeforeClass(groups = { "integration", "live" })
    public void setup() {
       super.setup();
+      TroveUtils utils = new TroveUtils(api);
       for (String zone : api.getConfiguredZones()) {
          // create instances
          List<Instance> instanceList = Lists.newArrayList();
-         InstanceApi instanceApi = api.getInstanceApiForZone(zone);
-         Instance first = instanceApi.create("1", 1, "first_database_testing_" + zone);
-         Instance second = instanceApi.create("1", 1, "second_database_testing_" + zone);
+         Instance first = utils.getWorkingInstance(zone, "first_database_testing_" + zone, "1", 1);
+         Instance second = utils.getWorkingInstance(zone, "second_database_testing_" + zone, "1", 1);
          instanceList.add(first);
          instanceList.add(second);
-         InstancePredicates.awaitAvailable(instanceApi).apply(first);
-         InstancePredicates.awaitAvailable(instanceApi).apply(second);        
          instancesToDelete.put(zone, instanceList);
          
          DatabaseApi databaseApiFirst = api.getDatabaseApiForInstanceInZone(first.getId(), zone);
@@ -85,7 +82,7 @@ public class DatabaseApiLiveTest extends BaseTroveApiLiveTest {
       for (String zone : api.getConfiguredZones()) {
          InstanceApi instanceApi = api.getInstanceApiForZone(zone);
          assertTrue(instanceApi.list().size() >= 2);
-         for(Instance instance : instanceApi.list() ) {
+         for(Instance instance : instancesToDelete.get(zone)) {
             DatabaseApi databaseApi = api.getDatabaseApiForInstanceInZone(instance.getId(), zone);
             if(!instance.getName().contains("database_testing"))continue;
             assertTrue(databaseApi.list().size() >=1);
@@ -101,18 +98,15 @@ public class DatabaseApiLiveTest extends BaseTroveApiLiveTest {
       for (String zone : api.getConfiguredZones()) {
          InstanceApi instanceApi = api.getInstanceApiForZone(zone);
          assertTrue(instanceApi.list().size() >= 2);
-         for(Instance instance : instanceApi.list() ) {
+         for(Instance instance : instancesToDelete.get(zone)) {
             DatabaseApi databaseApi = api.getDatabaseApiForInstanceInZone(instance.getId(), zone);
             if(!instance.getName().contains("database_testing"))continue;
             assertTrue(databaseApi.list().size() >=1);
             for(String database : databaseApi.list()){
                assertNotNull(database);
                assertTrue(database.equals("livetest_db1") || database.equals("livetest_db2") || database.equals("livetest_db3") );
-               assertEquals(instanceApi.get(instance.getId()).getStatus(), Instance.Status.ACTIVE);
                assertTrue(databaseApi.delete(database));
-               assertEquals(instanceApi.get(instance.getId()).getStatus(), Instance.Status.ACTIVE);
                assertTrue(databaseApi.create(database));
-               assertEquals(instanceApi.get(instance.getId()).getStatus(), Instance.Status.ACTIVE);
             }
          }  
       }   

http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/40a05b9e/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/InstanceApiLiveTest.java
----------------------------------------------------------------------
diff --git a/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/InstanceApiLiveTest.java b/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/InstanceApiLiveTest.java
index c416fe9..d092dd7 100644
--- a/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/InstanceApiLiveTest.java
+++ b/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/InstanceApiLiveTest.java
@@ -18,19 +18,22 @@ package org.jclouds.openstack.trove.v1.features;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.assertFalse;
+
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+
 import org.jclouds.openstack.trove.v1.domain.Instance;
 import org.jclouds.openstack.trove.v1.internal.BaseTroveApiLiveTest;
-import org.jclouds.openstack.trove.v1.predicates.InstancePredicates;
+import org.jclouds.openstack.trove.v1.internal.TroveUtils;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
+
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -38,7 +41,7 @@ import com.google.common.collect.Maps;
 /**
  * @author Zack Shoylev
  */
-@Test(groups = "live", testName = "InstanceApiLiveTest", singleThreaded = true)
+@Test(groups = "live", testName = "InstanceApiLiveTest")
 public class InstanceApiLiveTest extends BaseTroveApiLiveTest {
 
     private static Map<String,List<Instance>> created = Maps.newHashMap();
@@ -47,12 +50,12 @@ public class InstanceApiLiveTest extends BaseTroveApiLiveTest {
     @BeforeClass(groups = { "integration", "live" })
     public void setup() {
         super.setup();
+        TroveUtils utils= new TroveUtils(api);
         for (String zone : api.getConfiguredZones()) {
             List<Instance> zoneList = Lists.newArrayList();
             InstanceApi instanceApi = api.getInstanceApiForZone(zone);
-            zoneList.add(instanceApi.create("1", 1, "first_instance_testing_" + zone));
-            Instance second = instanceApi.create("1", 1, "second_instance_testing_" + zone);
-            InstancePredicates.awaitAvailable(instanceApi).apply(second);
+            zoneList.add(utils.getWorkingInstance(zone, "first_instance_testing_" + zone, "1", 1));
+            Instance second = utils.getWorkingInstance(zone, "second_instance_testing_" + zone, "1", 1);
             instanceApi.enableRoot(second.getId());
             zoneList.add(second);            
             created.put(zone, zoneList);

http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/40a05b9e/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/UserApiLiveTest.java
----------------------------------------------------------------------
diff --git a/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/UserApiLiveTest.java b/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/UserApiLiveTest.java
index a4ccd3a..ae92f55 100644
--- a/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/UserApiLiveTest.java
+++ b/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/features/UserApiLiveTest.java
@@ -31,7 +31,7 @@ import java.util.UUID;
 import org.jclouds.openstack.trove.v1.domain.Instance;
 import org.jclouds.openstack.trove.v1.domain.User;
 import org.jclouds.openstack.trove.v1.internal.BaseTroveApiLiveTest;
-import org.jclouds.openstack.trove.v1.predicates.InstancePredicates;
+import org.jclouds.openstack.trove.v1.internal.TroveUtils;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -44,7 +44,7 @@ import com.google.common.collect.Maps;
 /**
  * @author Zack Shoylev
  */
-@Test(groups = "live", testName = "UserApiLiveTest", singleThreaded = true)
+@Test(groups = "live", testName = "UserApiLiveTest")
 public class UserApiLiveTest extends BaseTroveApiLiveTest {
 
    // zone to instance
@@ -55,16 +55,14 @@ public class UserApiLiveTest extends BaseTroveApiLiveTest {
    @BeforeClass(groups = { "integration", "live" })
    public void setup() {
       super.setup();
+      TroveUtils utils= new TroveUtils(api);
       for (String zone : api.getConfiguredZones()) {
          // create instances
          List<Instance> instanceList = Lists.newArrayList();
-         InstanceApi instanceApi = api.getInstanceApiForZone(zone);
-         Instance first = instanceApi.create("1", 1, "first_user_trove_live_testing_" + zone);
-         Instance second = instanceApi.create("1", 1, "second_user_trove_live_testing_" + zone);
+         Instance first = utils.getWorkingInstance(zone, "first_user_trove_live_testing_" + zone, "1", 1);
+         Instance second = utils.getWorkingInstance(zone, "second_user_trove_live_testing_" + zone, "1", 1);
          instanceList.add(first);
          instanceList.add(second);
-         InstancePredicates.awaitAvailable(instanceApi).apply(first);
-         InstancePredicates.awaitAvailable(instanceApi).apply(second);        
          instancesToDelete.put(zone, instanceList);
          // create users
          User user1 = User.builder()
@@ -117,7 +115,7 @@ public class UserApiLiveTest extends BaseTroveApiLiveTest {
       for (String zone : api.getConfiguredZones()) {
          InstanceApi instanceApi = api.getInstanceApiForZone(zone);
          assertTrue(instanceApi.list().size() >= 2);
-         for(Instance instance : instanceApi.list() ) {
+         for(Instance instance : instancesToDelete.get(zone)) {
             UserApi userApi = api.getUserApiForInstanceInZone(instance.getId(), zone);
             if(!instance.getName().contains("user_trove_live_testing"))continue;
             assertTrue(userApi.list().size() >=1);
@@ -133,7 +131,7 @@ public class UserApiLiveTest extends BaseTroveApiLiveTest {
       for (String zone : api.getConfiguredZones()) {
          InstanceApi instanceApi = api.getInstanceApiForZone(zone);
          assertTrue(instanceApi.list().size() >= 2);
-         for(Instance instance : instanceApi.list() ) {
+         for(Instance instance : instancesToDelete.get(zone)) {
             UserApi userApi = api.getUserApiForInstanceInZone(instance.getId(), zone);
             if(!instance.getName().contains("user_trove_live_testing"))continue;
             assertTrue(userApi.list().size() >=1);
@@ -154,7 +152,7 @@ public class UserApiLiveTest extends BaseTroveApiLiveTest {
       for (String zone : api.getConfiguredZones()) {
          InstanceApi instanceApi = api.getInstanceApiForZone(zone);
          assertTrue(instanceApi.list().size() >= 2 );
-         for(Instance instance : instanceApi.list() ) {
+         for(Instance instance : instancesToDelete.get(zone)) {
             UserApi userApi = api.getUserApiForInstanceInZone(instance.getId(), zone);
             if(!instance.getName().contains("user_trove_live_testing"))continue;
             assertTrue(userApi.list().size() >=1);
@@ -170,7 +168,7 @@ public class UserApiLiveTest extends BaseTroveApiLiveTest {
       for (String zone : api.getConfiguredZones()) {
          InstanceApi instanceApi = api.getInstanceApiForZone(zone);
          assertTrue(instanceApi.list().size() >= 2);
-         for(Instance instance : instanceApi.list() ) {
+         for(Instance instance : instancesToDelete.get(zone)) {
             UserApi userApi = api.getUserApiForInstanceInZone(instance.getId(), zone);
             if(!instance.getName().contains("user_trove_live_testing"))continue;
             assertTrue(userApi.list().size() >=1);
@@ -201,7 +199,7 @@ public class UserApiLiveTest extends BaseTroveApiLiveTest {
    @Test
    public void testGetUserWhenNotFound() {
       for (String zone : api.getConfiguredZones()) {
-         String instanceId = api.getInstanceApiForZone(zone).list().iterator().next().getId(); 
+         String instanceId = instancesToDelete.get(zone).iterator().next().getId();
          UserApi userApi = api.getUserApiForInstanceInZone(instanceId, zone);
          assertNull(userApi.get("9999"));
       }

http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/40a05b9e/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/BaseTroveApiExpectTest.java
----------------------------------------------------------------------
diff --git a/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/BaseTroveApiExpectTest.java b/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/BaseTroveApiExpectTest.java
index 3c9aaa7..b8c4216 100644
--- a/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/BaseTroveApiExpectTest.java
+++ b/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/BaseTroveApiExpectTest.java
@@ -19,7 +19,7 @@ package org.jclouds.openstack.trove.v1.internal;
 import org.jclouds.openstack.trove.v1.TroveApi;
 
 /**
- * Base class for writing Flavor Rest Api Expect tests
+ * Base class for writing Trove Rest Api Expect tests
  * 
  * @author Everett Toews
  */

http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/40a05b9e/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/TroveUtils.java
----------------------------------------------------------------------
diff --git a/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/TroveUtils.java b/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/TroveUtils.java
new file mode 100644
index 0000000..0a77c72
--- /dev/null
+++ b/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/TroveUtils.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.openstack.trove.v1.internal;
+
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.Resource;
+
+import org.jclouds.openstack.trove.v1.TroveApi;
+import org.jclouds.openstack.trove.v1.domain.Instance;
+import org.jclouds.openstack.trove.v1.features.InstanceApi;
+import org.jclouds.openstack.trove.v1.predicates.InstancePredicates;
+import org.jclouds.logging.Logger;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+
+/**
+ * @author Zack Shoylev
+ * 
+ *         Helper methods for dealing with instances that get created with
+ *         errors
+ */
+public class TroveUtils {
+   private final TroveApi api;
+   @Resource
+   protected Logger logger = Logger.NULL;
+
+   public TroveUtils(TroveApi api) {
+      this.api = api;
+   }
+
+   /**
+    * Create an ACTIVE operational instance
+    * 
+    * @see InstanceApi#create(String, int, String)
+    * 
+    * @param zone
+    *           The instance zone or region
+    * @param name
+    *           Instance name
+    * @param flavorId
+    *           Id of the flavor to be used when creating the instance
+    * @param size
+    *           Size of the instance
+    * @return Instance object in active state or NULL
+    */
+   public Instance getWorkingInstance(String zone, String name, String flavorId, int size) {
+      InstanceApi instanceApi = api.getInstanceApiForZone(zone);
+      for (int retries = 0; retries < 10; retries++) {
+         Instance instance = null;
+         try {
+            instance = instanceApi.create(flavorId, size, name);
+         } catch (Exception e) {
+
+            Uninterruptibles.sleepUninterruptibly(15, TimeUnit.SECONDS);
+
+            logger.error(e.getStackTrace().toString());
+            continue;
+         }
+
+         Instance updatedInstance = awaitAvailable(instance, instanceApi);
+         if (updatedInstance != null) {
+            return updatedInstance;
+         }
+         instanceApi.delete(instance.getId());
+         InstancePredicates.awaitDeleted(instanceApi).apply(instance);
+         
+      }
+      return null;
+   }
+
+   public Instance getWorkingInstance(String zone) {
+      return getWorkingInstance(zone, UUID.randomUUID().toString(), "1", 1);
+   }
+
+   private Instance awaitAvailable(Instance instance, InstanceApi iapi) {
+      for (int n = 0; n < 100; n = n + 1) {
+         Instance updatedInstance = iapi.get(instance.getId());
+         if (updatedInstance.getStatus() == Instance.Status.ACTIVE)
+            return updatedInstance;
+         if (updatedInstance.getStatus() == Instance.Status.UNRECOGNIZED)
+            return null; // fast fail
+         Uninterruptibles.sleepUninterruptibly(15, TimeUnit.SECONDS);
+      }
+      return null;
+   }
+}

http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/40a05b9e/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/TroveUtilsExpectTest.java
----------------------------------------------------------------------
diff --git a/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/TroveUtilsExpectTest.java b/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/TroveUtilsExpectTest.java
new file mode 100644
index 0000000..9ebda6f
--- /dev/null
+++ b/openstack-trove/src/test/java/org/jclouds/openstack/trove/v1/internal/TroveUtilsExpectTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.openstack.trove.v1.internal;
+
+import static org.testng.Assert.assertEquals;
+
+import java.net.URI;
+import java.util.List;
+
+import javax.ws.rs.core.MediaType;
+
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.openstack.trove.v1.TroveApi;
+import org.jclouds.openstack.trove.v1.domain.Instance;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Tests TroveUtils
+ *
+ * @author Zack Shoylev
+ */
+@Test(groups = "unit", testName = "InstanceApiExpectTest")
+public class TroveUtilsExpectTest extends BaseTroveApiExpectTest {
+    
+    public void testHelperCreateInstance() {
+        HttpRequest createInstance = authenticatedGET().endpoint(URI.create("http://172.16.0.1:8776/v1/3456/instances"))
+                                                       .method("POST")
+                                                       .payload(payloadFromResourceWithContentType("/instance_create_request.json", MediaType.APPLICATION_JSON))
+                                                       .build();
+        HttpResponse createInstanceSuccess = HttpResponse.builder().statusCode(200).payload(payloadFromResource("/instance_create.json")).build();
+        HttpResponse createInstanceFail = HttpResponse.builder().statusCode(404).payload(payloadFromResource("/instance_create.json")).build();
+        HttpRequest getInstance = authenticatedGET().endpoint(URI.create("http://172.16.0.1:8776/v1/3456/instances/44b277eb-39be-4921-be31-3d61b43651d7")).build();
+        HttpResponse badStatus = HttpResponse.builder().statusCode(200).payload(payloadFromResource("/instance_get_bad_instance.json")).build();
+        HttpResponse goodStatus = HttpResponse.builder().statusCode(200).payload(payloadFromResource("/instance_get.json")).build();
+        HttpResponse deletedStatus = HttpResponse.builder().statusCode(404).payload(payloadFromResource("/instance_get.json")).build();
+        HttpRequest deleteInstance = authenticatedGET().endpoint(URI.create("http://172.16.0.1:8776/v1/3456/instances/44b277eb-39be-4921-be31-3d61b43651d7")).method("DELETE").build();
+        HttpResponse deleteInstanceResponse = HttpResponse.builder().statusCode(202).build();
+
+        List<HttpRequest> requests = ImmutableList.of(  keystoneAuthWithUsernameAndPasswordAndTenantName, createInstance,     createInstance,     createInstance,        getInstance, deleteInstance,         getInstance,   createInstance,     createInstance,        getInstance);
+        List<HttpResponse> responses = ImmutableList.of(responseWithKeystoneAccess,                       createInstanceFail, createInstanceFail, createInstanceSuccess, badStatus,   deleteInstanceResponse, deletedStatus, createInstanceFail, createInstanceSuccess, goodStatus); 
+
+        TroveApi api = orderedRequestsSendResponses(requests, responses);
+
+        TroveUtils utils = new TroveUtils(api);
+        Instance instance = utils.getWorkingInstance("RegionOne", "json_rack_instance", "1", 2);
+        assertEquals(instance.getSize(),2);
+        assertEquals(instance.getName(), "json_rack_instance");  
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/40a05b9e/openstack-trove/src/test/resources/instance_get_bad_instance.json
----------------------------------------------------------------------
diff --git a/openstack-trove/src/test/resources/instance_get_bad_instance.json b/openstack-trove/src/test/resources/instance_get_bad_instance.json
new file mode 100644
index 0000000..99b8688
--- /dev/null
+++ b/openstack-trove/src/test/resources/instance_get_bad_instance.json
@@ -0,0 +1,37 @@
+{
+    "instance": {
+        "created": "2013-03-18T19:09:17", 
+        "flavor": {
+            "id": "1", 
+            "links": [
+                {
+                    "href": "https://ord.databases.api.rackspacecloud.com/v1.0/1234/flavors/1", 
+                    "rel": "self"
+                }, 
+                {
+                    "href": "https://ord.databases.api.rackspacecloud.com/flavors/1", 
+                    "rel": "bookmark"
+                }
+            ]
+        }, 
+        "hostname": "e09ad9a3f73309469cf1f43d11e79549caf9acf2.rackspaceclouddb.com",
+        "id": "44b277eb-39be-4921-be31-3d61b43651d7", 
+        "links": [
+            {
+                "href": "https://ord.databases.api.rackspacecloud.com/v1.0/1234/instances/44b277eb-39be-4921-be31-3d61b43651d7", 
+                "rel": "self"
+            }, 
+            {
+                "href": "https://ord.databases.api.rackspacecloud.com/instances/44b277eb-39be-4921-be31-3d61b43651d7", 
+                "rel": "bookmark"
+            }
+        ], 
+        "name": "json_rack_instance", 
+        "status": "ERROR", 
+        "updated": "2013-03-18T19:09:17", 
+        "volume": {
+            "size": 2, 
+            "used": 0.16368598397821188
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/40a05b9e/rackspace-clouddatabases-uk/pom.xml
----------------------------------------------------------------------
diff --git a/rackspace-clouddatabases-uk/pom.xml b/rackspace-clouddatabases-uk/pom.xml
index acbb0e2..474b7da 100644
--- a/rackspace-clouddatabases-uk/pom.xml
+++ b/rackspace-clouddatabases-uk/pom.xml
@@ -128,7 +128,9 @@
                   <goal>test</goal>
                 </goals>
                 <configuration>
-                  <threadCount>1</threadCount>
+                  <forkCount>5</forkCount>
+                  <reuseForks>true</reuseForks>
+                  <parallel>classes</parallel>
                   <systemPropertyVariables>
                     <test.rackspace-clouddatabases-uk.endpoint>${test.rackspace-clouddatabases-uk.endpoint}</test.rackspace-clouddatabases-uk.endpoint>
                     <test.rackspace-clouddatabases-uk.api-version>${test.rackspace-clouddatabases-uk.api-version}</test.rackspace-clouddatabases-uk.api-version>

http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/40a05b9e/rackspace-clouddatabases-us/pom.xml
----------------------------------------------------------------------
diff --git a/rackspace-clouddatabases-us/pom.xml b/rackspace-clouddatabases-us/pom.xml
index d89e790..bb8c437 100644
--- a/rackspace-clouddatabases-us/pom.xml
+++ b/rackspace-clouddatabases-us/pom.xml
@@ -128,7 +128,9 @@
                   <goal>test</goal>
                 </goals>
                 <configuration>
-                  <threadCount>1</threadCount>
+                  <forkCount>5</forkCount>
+                  <reuseForks>true</reuseForks>
+                  <parallel>classes</parallel>
                   <systemPropertyVariables>
                     <test.rackspace-clouddatabases-us.endpoint>${test.rackspace-clouddatabases-us.endpoint}</test.rackspace-clouddatabases-us.endpoint>
                     <test.rackspace-clouddatabases-us.api-version>${test.rackspace-clouddatabases-us.api-version}</test.rackspace-clouddatabases-us.api-version>